bl-odoo 0.2.6__py3-none-any.whl → 0.3.0__py3-none-any.whl

bl/freezer.py

@@ -1,13 +1,11 @@
 import asyncio
 import yaml
-from operator import countOf
 from pathlib import Path
-from typing import TextIO
 
 from rich.console import Console
 from rich.live import Live
 from rich.progress import BarColumn, MofNCompleteColumn, Progress, TaskID, TextColumn
-from bl.spec_parser import ModuleSpec, ProjectSpec
+from bl.types import RepoInfo, ProjectSpec
 from bl.utils import get_local_ref, get_module_path, run_git
 
 console = Console()
@@ -18,7 +16,7 @@ async def freeze_spec(
     progress: Progress,
     task_id: TaskID,
     module_name: str,
-    module_spec: ModuleSpec,
+    module_spec: RepoInfo,
     workdir: Path,
 ):
     result = {module_name: {}}

bl/spec_parser.py

@@ -1,18 +1,13 @@
 import re
 import warnings
+from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 import yaml
 
-
-class OriginType(Enum):
-    """Type of origin reference."""
-
-    BRANCH = "branch"
-    PR = "pr"
-    REF = "ref"
+from bl.types import RepoInfo, OriginType, ProjectSpec, RefspecInfo
 
 
 def make_remote_merge_from_src(src: str) -> tuple[dict, list]:
@@ -52,60 +47,26 @@ def get_origin_type(origin_value: str) -> OriginType:
         return OriginType.BRANCH
 
 
-class RefspecInfo:
-    """A git refspec with its remote, type and optional frozen sha."""
-
-    def __init__(
-        self,
-        remote: str,
-        ref_str: str,
-        type: OriginType,
-        ref_name: Optional[str],
-    ):
-        self.remote = remote
-        self.refspec = ref_str
-        """ The refspec string (branch name, PR ref, or commit hash). """
-        self.type = type
-        self.ref_name = ref_name
-
-    def __repr__(self) -> str:
-        return f"RefspecInfo(remote={self.remote!r}, origin={self.refspec!r}, type={self.type.value})"
-
-
-class ModuleSpec:
-    """Represents the specification for a set of modules."""
-
-    def __init__(
-        self,
-        modules: List[str],
-        remotes: Optional[Dict[str, str]] = {},
-        origins: Optional[List[RefspecInfo]] = [],
-        shell_commands: Optional[List[str]] = [],
-        patch_globs_to_apply: Optional[List[str]] = None,
-        target_folder: Optional[str] = None,
-        frozen_modules: Optional[Dict[str, Dict[str, str]]] = None,
-    ):
-        self.modules = modules
-        self.remotes = remotes
-        self.refspec_info = origins
-        self.shell_commands = shell_commands
-        self.patch_globs_to_apply = patch_globs_to_apply
-        self.frozen_modules = frozen_modules
-        self.target_folder = None
-
-    def __repr__(self) -> str:
-        return f"ModuleSpec(modules={self.modules}, remotes={self.remotes}, origins={self.refspec_info})"
-
+def parse_remote_refspec_from_parts(parts: List[str], frozen_repo: Dict[str, Dict[str, str]]):
+    if len(parts) == 2:
+        parts.insert(1, "")
+    else:
+        warnings.warn(
+            "Deprecated src format: use <url> <sha> format for the src property",
+            DeprecationWarning,
+        )
+    remote_key, _, ref_spec = parts
+    ref_type = get_origin_type(ref_spec)
 
-class ProjectSpec:
-    """Represents the overall project specification from the YAML file."""
+    ref_name = None
+    remote_freezes = frozen_repo.get(remote_key, {})
 
-    def __init__(self, specs: Dict[str, ModuleSpec], workdir: Path = Path(".")):
-        self.specs = specs
-        self.workdir = workdir
+    if ref_spec in remote_freezes:
+        ref_type = OriginType.REF
+        ref_name = ref_spec
+        ref_spec = remote_freezes.get(ref_name)
 
-    def __repr__(self) -> str:
-        return f"ProjectSpec(specs={self.specs}, workdir={self.workdir})"
+    return RefspecInfo(remote_key, ref_spec, ref_type, ref_name)
 
 
 def load_spec_file(config: Path, frozen: Path, workdir: Path) -> Optional[ProjectSpec]:
@@ -123,6 +84,7 @@ def load_spec_file(config: Path, frozen: Path, workdir: Path) -> Optional[Projec
             config = config.resolve()
             # If the file is not in the current directory, check inside the odoo subdirectory
             odoo_config = config.parent / "odoo" / config.name
+            # TODO(franz): should use rich console for prettiness
             if not odoo_config.exists():
                 print(f"Error: Neither '{config}' nor '{odoo_config}' exists.")
                 return None
@@ -151,22 +113,21 @@ def load_spec_file(config: Path, frozen: Path, workdir: Path) -> Optional[Projec
         except yaml.YAMLError as e:
             print(f"Error parsing frozen YAML file '{frozen_path}': {e}")
 
-    specs: Dict[str, ModuleSpec] = {}
-    for section_name, section_data in data.items():
-        modules = section_data.get("modules", [])
-        src = section_data.get("src")
-        remotes = section_data.get("remotes") or {}
-        merges = section_data.get("merges") or []
-        shell_commands = section_data.get("shell_command_after") or None
-        patch_globs_to_apply = section_data.get("patch_globs") or None
-
-        frozen_for_section_raw = frozen_mapping.get(section_name)
-        frozen_for_section: Optional[Dict[str, Dict[str, str]]] = (
-            frozen_for_section_raw if isinstance(frozen_for_section_raw, dict) else None
-        )
+    repos: Dict[str, RepoInfo] = {}
+    for repo_name, repo_data in data.items():
+        modules = repo_data.get("modules", [])
+        src = repo_data.get("src")
+        remotes = repo_data.get("remotes") or {}
+        merges = repo_data.get("merges") or []
+        shell_commands = repo_data.get("shell_command_after") or None
+        patch_globs_to_apply = repo_data.get("patch_globs") or None
+        target_folder = repo_data.get("target_folder") or None
+        locales = repo_data.get("locales", [])
+
+        frozen_repo = frozen_mapping.get(repo_name, {})
 
         # Parse merges into RefspecInfo objects
-        origins: List[RefspecInfo] = []
+        refspec_infos: List[RefspecInfo] = []
         if src:
             # If src is defined, create a remote and merge entry from it
             src_remotes, src_merges = make_remote_merge_from_src(src)
@@ -175,51 +136,17 @@ def load_spec_file(config: Path, frozen: Path, workdir: Path) -> Optional[Projec
 
         for merge_entry in merges:
             parts = merge_entry.split(" ", 2)
-            if len(parts) == 2:
-                remote_key, ref_spec = parts
-
-                # Determine type: PR if matches refs/pull/{pr_id}/head pattern, otherwise branch
-                ref_type = get_origin_type(ref_spec)
-
-                ref_name = None
-                if frozen_for_section:
-                    remote_freezes = frozen_for_section.get(remote_key) or {}
-                    ref_name = ref_spec
-                    ref_type = OriginType.REF
-                    frozen_ref = remote_freezes.get(ref_spec)
-                    ref_spec = frozen_ref or ref_spec
-
-                origins.append(
-                    RefspecInfo(
-                        remote_key,
-                        ref_spec,
-                        ref_type,
-                        ref_name,
-                    )
-                )
-            elif len(parts) == 3:
-                warnings.warn(
-                    "Deprecated src format: use <url> <sha> format for the src property",
-                    DeprecationWarning,
-                )
-                remote_key, _, ref_spec = parts
-                ref_type = get_origin_type(ref_spec)
-
-                ref_name = None
-                if frozen_for_section:
-                    remote_freezes = frozen_for_section.get(remote_key) or {}
-                    ref_name = ref_spec
-                    ref_spec = remote_freezes.get(ref_spec)
-
-                origins.append(RefspecInfo(remote_key, ref_spec, ref_type, ref_name))
-
-        specs[section_name] = ModuleSpec(
+            refspec_info = parse_remote_refspec_from_parts(parts, frozen_repo)
+            refspec_infos.append(refspec_info)
+
+        repos[repo_name] = RepoInfo(
             modules,
             remotes,
-            origins,
+            refspec_infos,
             shell_commands,
             patch_globs_to_apply,
-            frozen_modules=frozen_for_section or None,
+            target_folder,
+            locales,
         )
 
-    return ProjectSpec(specs, workdir)
+    return ProjectSpec(repos, workdir)

bl/spec_processor.py

@@ -1,12 +1,8 @@
 import asyncio
-import hashlib
-from logging import root
 import os
-from posix import link
 import warnings
-import shutil
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Dict, List
 
 from rich.console import Console
 from rich.live import Live
@@ -16,10 +12,17 @@ from typing_extensions import deprecated
 
 from bl.utils import english_env, get_local_ref, get_module_path, run_git
 
-from .spec_parser import ModuleSpec, OriginType, ProjectSpec, RefspecInfo
+from bl.types import CloneFlags, CloneInfo, OriginType, ProjectSpec, RefspecInfo, RepoInfo
 
 console = Console()
 
+# TODO(franz): it's a bit better now but better keep an eye on it
+# TODO(franz): Error handling should be watch carefully because if
+# we don't exit on some error code due to the fact that git resolve to
+# the parent repo we could activate sparse checkout on a parent folder
+# should probably make a function that handles the error in a unified manner
+# and crash if the error is on a vital part of the process
+
 
 def rich_warning(message, category, filename, lineno, file=None, line=None):
     console.print(f"[yellow]Warning:[/] {category.__name__}: {message}\n[dim]{filename}:{lineno}[/]")
@@ -29,24 +32,44 @@ warnings.showwarning = rich_warning
 warnings.simplefilter("default", DeprecationWarning)
 
 
-# for single branch we should clone shallow but for other we should clone
-# with tree:0 filter and because this avoid confusing fetch for git to have the history
-# before fetching
-def create_clone_args(name: str, ref_spec_info: RefspecInfo, remote_url: str, shallow: bool) -> List[str]:
+def check_path_is_repo(module_path: Path):
+    # TODO(franz): add check for .git folder
+    return not module_path.exists() or not module_path.is_dir()
+
+
+def clone_info_from_repo(name: str, repo_info: RepoInfo):
+    flags = CloneFlags.SHALLOW if name == "odoo" or len(repo_info.refspec_info) == 1 else 0
+    flags |= CloneFlags.SPARSE if name != "odoo" or len(repo_info.locales) > 0 else 0
+    root_refspec_info = repo_info.refspec_info[0]
+    remote_url = repo_info.remotes.get(root_refspec_info.remote)
+
+    return CloneInfo(
+        remote_url,
+        flags,
+        root_refspec_info,
+    )
+
+
+# for single branch we should clone shallow but for other we should clone deep
+# this allows us to get merge-base to work and git can then merge by pulling the minimum
+# amount of data
+def create_clone_args(clone_info: CloneInfo) -> List[str]:
     """Creates git clone arguments based on the base origin."""
     args = [
         "clone",
         "--filter=tree:0",
     ]
 
-    if name == "odoo" or shallow:
+    if clone_info.clone_flags & CloneFlags.SHALLOW:
         args += [
             "--depth",
             "1",
         ]
-    else:
+    if clone_info.clone_flags & CloneFlags.SPARSE:
         args += ["--sparse"]
 
+    ref_spec_info = clone_info.root_refspec_info
+
     if ref_spec_info.type == OriginType.REF:
         args += [
             "--revision",
@@ -61,7 +84,7 @@ def create_clone_args(name: str, ref_spec_info: RefspecInfo, remote_url: str, sh
         ]
 
     args += [
-        remote_url,
+        clone_info.url,
     ]
 
     return args
@@ -74,22 +97,37 @@ def normalize_merge_result(ret: int, out: str, err: str):
     return ret, err
 
 
-class SpecProcessor:
+class RepoProcessor:
     """
     Processes a ProjectSpec by concurrently cloning and merging modules.
     """
 
-    def __init__(self, workdir: Path, concurrency: int = 4):
+    def __init__(
+        self,
+        workdir: Path,
+        name: str,
+        semaphore: asyncio.Semaphore,
+        repo_info: RepoInfo,
+        progress: Progress,
+        count_progress: Progress,
+        count_task: TaskID,
+        concurrency: int,
+    ):
         self.workdir = workdir
+        self.name = name
+        self.semaphore = semaphore
+        self.repo_info = repo_info
+        self.progress = progress
+        self.count_progress = count_progress
+        self.count_task = count_task
         self.concurrency = concurrency
-        self.semaphore = asyncio.Semaphore(concurrency)
 
     @deprecated(
         "run_shell_commands is deprecated if used to apply patches. Use patch_globs properties in spec.yaml instead."
     )
-    async def run_shell_commands(self, progress: Progress, task_id: TaskID, spec: ModuleSpec, module_path: Path) -> int:
-        for cmd in spec.shell_commands:
-            progress.update(task_id, status=f"Running shell command: {cmd}...")
+    async def run_shell_commands(self, repo_info: RepoInfo, module_path: Path) -> int:
+        for cmd in repo_info.shell_commands:
+            self.progress.update(self.task_id, status=f"Running shell command: {cmd}...")
             proc = await asyncio.create_subprocess_shell(
                 cmd,
                 cwd=str(module_path),
@@ -102,128 +140,74 @@ class SpecProcessor:
                 # This is a sanity check because people usually put "git am" commands
                 # in shell_commands, so we abort any ongoing git am
                 await run_git("am", "--abort", cwd=str(module_path))
-                progress.update(
-                    task_id,
+                self.progress.update(
+                    self.task_id,
                     status=f"[red]Shell command failed: {cmd}\nError: {stderr.decode().strip()}",
                 )
                 return -1
         return 0
 
-    async def fetch_local_ref(
-        self,
-        origin: RefspecInfo,
-        local_ref: str,
-        module_path: Path,
-    ) -> tuple[int, str, str]:
-        return await run_git(
-            "fetch",
-            origin.remote,
-            f"{origin.refspec}:{local_ref}",
-            cwd=module_path,
-        )
-
-    async def clone_base_repo_ref(
-        self, name: str, ref_spec_info: RefspecInfo, remote_url: str, module_path: Path, shallow: bool
-    ) -> tuple[int, str, str]:
-        args = create_clone_args(name, ref_spec_info, remote_url, shallow)
-
-        ret, out, err = await run_git(
-            *args,
-            str(module_path),
-        )
-
-        # if it's a ref we need to manually create a base branch because we cannot
-        # merge in a detached head
-        local_ref = get_local_ref(ref_spec_info)
-        ret, out, err = await run_git(
-            "checkout",
-            "-b",
-            local_ref,
-            cwd=str(module_path),
-        )
-
-        return ret, out, err
-
-    async def try_merge(
-        self,
-        progress: Progress,
-        task_id: TaskID,
-        remote_url: str,
-        local_ref: str,
-        module_path: Path,
-        origin: RefspecInfo,
-    ) -> tuple[int, str]:
-        # Merge
-        # I think the idea would be to not fetch shallow but fetch treeless and do a merge-base
-        # then fetch the required data and then merge
-        progress.update(task_id, status=f"Merging {local_ref}", advance=0.1)
-        ret, out, err = await run_git("merge", "--no-edit", local_ref, cwd=module_path)
-        ret, err = normalize_merge_result(ret, out, err)
-
-        if "CONFLICT" in err:
-            progress.update(task_id, status=f"[red]Merge conflict in {origin.refspec}: {err}")
-            # In case of conflict, we might want to abort the merge
-            await run_git("merge", "--abort", cwd=module_path)
-        return ret, err
-
     async def setup_new_repo(
         self,
-        progress: Progress,
-        task_id: TaskID,
-        spec: ModuleSpec,
-        name: str,
-        root_refspec_info: RefspecInfo,
-        remote_url: str,
+        clone_info: CloneInfo,
         module_path: Path,
     ) -> int:
-        progress.update(
-            task_id,
-            status=(f"Cloning {root_refspec_info.remote}/{root_refspec_info.refspec}"),
+        root_refspec_info = clone_info.root_refspec_info
+        remote = root_refspec_info.remote
+        root_refspec = root_refspec_info.refspec
+
+        self.progress.update(
+            self.task_id,
+            status=(f"Cloning {remote}/{root_refspec}"),
         )
 
-        # Clone shallowly with blobless filter and no checkout
-        # We don't use the cache yet for simplicity, but we follow the optimized command
-        # User --revision for specific commit checkout if needed
-        shallow_clone = len(spec.refspec_info) == 1
-        ret, out, err = await self.clone_base_repo_ref(name, root_refspec_info, remote_url, module_path, shallow_clone)
+        clone_args = create_clone_args(clone_info)
+        ret, out, err = await run_git(*clone_args, module_path)
 
         if ret != 0:
             status_message = (
-                f"[red]Clone failed {root_refspec_info.remote}({remote_url})/{root_refspec_info.refspec}"
+                f"[red]Clone failed {root_refspec_info.remote}({clone_info.url})/{root_refspec_info.refspec}"
                 + f" -> {module_path}:\n{err}"
             )
-            progress.update(task_id, status=status_message)
+            self.progress.update(self.task_id, status=status_message)
             return ret
 
-    async def reset_repo_for_work(
-        self, progress: Progress, task_id: TaskID, spec: ModuleSpec, root_refspec_info: RefspecInfo, module_path: Path
-    ) -> int:
+        local_ref = get_local_ref(root_refspec_info)
+        ret, out, err = await run_git("checkout", "-b", local_ref, cwd=module_path)
+
+        return 0
+
+    async def reset_repo_for_work(self, module_path: Path) -> int:
+        # TODO(franz): we should test if the folder is a git repo or not
+
         ret, out, err = await run_git("status", "--porcelain", cwd=module_path)
 
         if out != "":
-            progress.update(task_id, status=f"[red]Repo is dirty:\n{out}")
+            self.progress.update(self.task_id, status=f"[red]Repo is dirty:\n{out}")
+            return ret
+        if ret != 0:
+            self.progress.update(self.task_id, status="[red]Repo does not exist")
             return ret
         # Reset all the local origin to their remote origins
-        progress.update(
-            task_id,
+        repo_info = self.repo_info
+        root_refspec_info = repo_info.refspec_info[0]
+
+        self.progress.update(
+            self.task_id,
             status=(f"Resetting existing repository for {root_refspec_info.remote}/{root_refspec_info.refspec}"),
         )
 
         s_ret, s_out, s_err = await run_git("rev-parse", "--is-shallow-repository", cwd=module_path)
-        if len(spec.refspec_info) > 1 and s_out == "true":
+        if len(repo_info.refspec_info) > 1 and s_out == "true":
             await run_git("fetch", "--unshallow", cwd=module_path)
 
-        reset_target = f"{root_refspec_info.remote}/{root_refspec_info.refspec}"
+        reset_target = get_local_ref(root_refspec_info)
         ret, out, err = await run_git("reset", "--hard", reset_target, cwd=module_path)
         if ret != 0:
-            progress.update(task_id, status=f"[red]Reset failed: {err}")
+            self.progress.update(self.task_id, status=f"[red]Reset failed: {err}")
             return ret
 
-        for refspec_info in spec.refspec_info[1:]:
-            local_ref = get_local_ref(refspec_info)
-            # This is probably the best thing but for now this works good enough
-            # TODO(franz): find something better
-            ret, out, err = await run_git("branch", "-d", local_ref, cwd=module_path)
+        return 0
 
     def link_all_modules(self, module_list: List[str], module_path: Path) -> tuple[int, str]:
         links_path = self.workdir / "links"
@@ -239,7 +223,7 @@ class SpecProcessor:
                 if path_dest_symlink.is_symlink():
                     path_dest_symlink.unlink()
 
-                os.symlink(path_src_symlink.relative_to(path_dest_symlink, walk_up=True), path_dest_symlink, True)
+                os.symlink(path_src_symlink.relative_to(links_path, walk_up=True), path_dest_symlink, True)
             except OSError as e:
                 return -1, str(e)
 
@@ -247,9 +231,7 @@ class SpecProcessor:
 
     async def merge_spec_into_tree(
         self,
-        progress: Progress,
-        task_id: TaskID,
-        spec: ModuleSpec,
+        spec: RepoInfo,
         refspec_info: RefspecInfo,
         root_refspec_info: RefspecInfo,
         module_path: Path,
@@ -260,11 +242,23 @@ class SpecProcessor:
         local_ref = get_local_ref(refspec_info)
         remote_ref = refspec_info.refspec
 
-        ret, err = await self.try_merge(progress, task_id, remote_url, local_ref, module_path, refspec_info)
+        # Merge
+        # I think the idea would be to not fetch shallow but fetch treeless and do a merge-base
+        # then fetch the required data and then merge
+        self.progress.update(self.task_id, status=f"Merging {local_ref}", advance=0.1)
+        ret, out, err = await run_git("merge", "--no-edit", local_ref, cwd=module_path)
+        ret, err = normalize_merge_result(ret, out, err)
+
+        if "CONFLICT" in err:
+            self.progress.update(self.task_id, status=f"[red]Merge conflict {local_ref} in {remote_ref}: {err}")
+            # In case of conflict, we might want to abort the merge
+            await run_git("merge", "--abort", cwd=module_path)
+            return ret, err
+
         if ret != 0:
+            self.progress.update(self.task_id, status=f"[red]Merge error {local_ref} in {remote_ref}: {err}")
             return ret, err
 
-        progress.advance(task_id)
         return 0, ""
 
     def get_refspec_by_remote(self, refspec_info_list: List[RefspecInfo]) -> Dict[str, List[RefspecInfo]]:
@@ -293,7 +287,7 @@ class SpecProcessor:
 
         return ret, out, err
 
-    def filter_non_link_module(self, spec: ModuleSpec):
+    def filter_non_link_module(self, spec: RepoInfo):
         result = []
         base_path_links = self.workdir / "links"
         for module in spec.modules:
@@ -307,153 +301,170 @@ class SpecProcessor:
                 )
         return result
 
-    async def process_module(
-        self, name: str, spec: ModuleSpec, progress: Progress, count_progress: Progress, count_task: TaskID
-    ) -> int:
-        """Processes a single ModuleSpec."""
-        total_steps = len(spec.refspec_info) + 1 if spec.refspec_info else 1
+    async def setup_odoo_sparse(self, module_spec: RepoInfo, module_path: Path):
+        list_modules = module_spec.modules
+
+        await run_git("sparse-checkout", "init", "--no-cone", cwd=module_path)
+        included_po = [f"{locale}.po" for locale in module_spec.locales]
+        included_modules = [f"/addons/{module}/*" for module in list_modules]
+        await run_git(
+            "sparse-checkout",
+            "set",
+            "/*",
+            "!/addons/*",
+            *included_modules,
+            "!*.po",
+            *included_po,
+            cwd=module_path,
+        )
 
-        symlink_modules = self.filter_non_link_module(spec)
+    async def setup_sparse_checkout(self, symlink_modules: List[str], module_path: Path):
+        # 2. Sparse Checkout setup
+        if self.name != "odoo":
+            self.progress.update(self.task_id, status="Configuring sparse checkout...")
+            await run_git("sparse-checkout", "init", "--cone", cwd=module_path)
+            if symlink_modules:
+                await run_git("sparse-checkout", "set", *self.repo_info.modules, cwd=module_path)
+        elif len(self.repo_info.locales) > 0:
+            # TODO(franz): We should still set sparse if there is no locales but there is a module list
+            self.progress.update(self.task_id, status="Configuring sparse odoo checkout...")
+            await self.setup_odoo_sparse(self.repo_info, module_path)
+
+    async def process_repo(self) -> int:
+        """Processes a single ModuleSpec."""
+        symlink_modules = self.filter_non_link_module(self.repo_info)
+        module_path = get_module_path(self.workdir, self.name, self.repo_info)
 
         async with self.semaphore:
-            task_id = progress.add_task(f"[cyan]{name}", status="Waiting...", total=total_steps)
             try:
-                if not spec.refspec_info:
-                    progress.update(task_id, status="[yellow]No origins defined", completed=1)
+                self.task_id = self.progress.add_task(
+                    f"[cyan]{self.name}", status="Waiting...", total=len(self.repo_info.refspec_info) + 1
+                )
+                if not self.repo_info.refspec_info:
+                    self.progress.update(self.task_id, status="[yellow]No origins defined", completed=1)
                     return -1
 
-                module_path = get_module_path(self.workdir, name, spec)
-
-                # 1. Initialize with first origin
-                root_refspec_info = spec.refspec_info[0]
-                remote_url = spec.remotes.get(root_refspec_info.remote) or root_refspec_info.remote
-
-                if not module_path.exists() or not module_path.is_dir():
-                    await self.setup_new_repo(progress, task_id, spec, name, root_refspec_info, remote_url, module_path)
+                # TODO(franz) the shallow and sparseness of repo should be unify
+                # so that we don't have all those stupid conditions
+                if check_path_is_repo(module_path):
+                    clone_info = clone_info_from_repo(self.name, self.repo_info)
+                    ret = await self.setup_new_repo(clone_info, module_path)
                 else:
-                    await self.reset_repo_for_work(progress, task_id, spec, root_refspec_info, module_path)
-
-                if name != "odoo":
-                    # We don't do sparse checkout for odoo because the odoo repo does not work at
-                    # all like the other repos (modules are in addons/ and src/addons/) instead of
-                    # at the root of the repo
+                    ret = await self.reset_repo_for_work(module_path)
 
-                    # TODO(franz): there is probably a way to make it work, but for now we skip it
-                    # this is probably a good way to gain performance
+                if ret != 0:
+                    return -1
 
-                    # 2. Sparse Checkout setup
-                    progress.update(task_id, status="Configuring sparse checkout...")
-                    await run_git("sparse-checkout", "init", "--cone", cwd=module_path)
-                    if symlink_modules:
-                        await run_git("sparse-checkout", "set", *spec.modules, cwd=module_path)
+                await self.setup_sparse_checkout(symlink_modules, module_path)
 
                 checkout_target = "merged"
 
                 await run_git("checkout", "-b", checkout_target, cwd=module_path)
-                progress.advance(task_id)
+                self.progress.advance(self.task_id)
 
-                for remote, remote_url in spec.remotes.items():
+                for remote, remote_url in self.repo_info.remotes.items():
                     await run_git("remote", "add", remote, remote_url, cwd=module_path)
                     await run_git("config", f"remote.{remote}.partialCloneFilter", "tree:0", cwd=module_path)
                     await run_git("config", f"remote.{remote}.promisor", "true", cwd=module_path)
 
-                # TODO(franz) fetch and merge should be done separately
-                # fetch can be done in parallel by git with -j X and putting several refspec as parameters
-                # to git fetch
-                refspec_by_remote: Dict[str, List[RefspecInfo]] = self.get_refspec_by_remote(spec.refspec_info[1:])
+                refspec_by_remote: Dict[str, List[RefspecInfo]] = self.get_refspec_by_remote(
+                    self.repo_info.refspec_info
+                )
 
+                # TODO(franz): right now we fetch everything so when the repo is just cloned
+                # we fetch the base branch twice. Since we fetch with multi this is probably not
+                # a big issue but it could be better
                 for remote, refspec_list in refspec_by_remote.items():
-                    progress.update(task_id, status=f"Fetching multi from {remote}")
+                    self.progress.update(self.task_id, status=f"Fetching multi from {remote}")
                     await self.fetch_multi(remote, refspec_list, module_path)
 
                 # 4. Fetch and Merge remaining origins
-                for refspec_info in spec.refspec_info[1:]:
+                for refspec_info in self.repo_info.refspec_info[1:]:
                     ret, err = await self.merge_spec_into_tree(
-                        progress, task_id, spec, refspec_info, root_refspec_info, module_path
+                        self.repo_info, refspec_info, self.repo_info.refspec_info[0], module_path
                     )
                     if ret != 0:
-                        progress.update(task_id, status=f"[purple]Merge failed from {refspec_info.refspec}: {err}")
                         return -1
+                    self.progress.advance(self.task_id)
 
-                if spec.shell_commands:
-                    ret = await self.run_shell_commands(progress, task_id, spec, module_path)
+                if self.repo_info.shell_commands:
+                    ret = await self.run_shell_commands(self.repo_info, module_path)
                     if ret != 0:
                         return ret
 
-                if spec.patch_globs_to_apply:
-                    for glob in spec.patch_globs_to_apply:
-                        progress.update(task_id, status=f"Applying patches: {glob}...", advance=0.1)
+                if self.repo_info.patch_globs_to_apply:
+                    for glob in self.repo_info.patch_globs_to_apply:
+                        self.progress.update(self.task_id, status=f"Applying patches: {glob}...", advance=0.1)
                         ret, out, err = await run_git("am", glob, cwd=module_path)
                         if ret != 0:
                             await run_git("am", "--abort", cwd=module_path)
-                            progress.update(task_id, status=f"[red]Applying patches failed: {err}")
+                            self.progress.update(self.task_id, status=f"[red]Applying patches failed: {err}")
                             return ret
 
-                progress.update(task_id, status="Linking directory")
-                if name != "odoo":
+                self.progress.update(self.task_id, status="Linking directory")
+                if self.name != "odoo":
                     ret, err = self.link_all_modules(symlink_modules, module_path)
                     if ret != 0:
-                        progress.update(task_id, status=f"[red]Could not link modules: {err}")
+                        self.progress.update(self.task_id, status=f"[red]Could not link modules: {err}")
                         return ret
 
-                progress.update(task_id, status="[green]Complete")
-                progress.remove_task(task_id)
-                count_progress.advance(count_task)
+                self.progress.update(self.task_id, status="[green]Complete", advance=1)
+                self.progress.remove_task(self.task_id)
+                self.count_progress.advance(self.count_task)
 
             except Exception as e:
-                progress.update(task_id, status=f"[red]Error: {str(e)}")
+                self.progress.update(self.task_id, status=f"[red]Error: {str(e)}")
+                raise e
                 return -1
 
         return 0
 
-    async def process_project(self, project_spec: ProjectSpec) -> None:
-        """Processes all modules in a ProjectSpec."""
-        (self.workdir / "external-src").mkdir(parents=True, exist_ok=True)
-
-        task_list_progress = Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-            TextColumn("{task.fields[status]}", table_column=Column(ratio=2)),
-        )
 
-        task_count_progress = Progress(
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            MofNCompleteColumn(),
-        )
-        count_task = task_count_progress.add_task("Processing Modules", total=len(project_spec.specs))
+async def process_project(project_spec: ProjectSpec, concurrency: int) -> None:
+    """Processes all modules in a ProjectSpec."""
+    (project_spec.workdir / "external-src").mkdir(parents=True, exist_ok=True)
 
-        progress_table = Table.grid()
-        progress_table.add_row(
-            task_list_progress,
-        )
-        progress_table.add_row(
-            task_count_progress,
-        )
+    task_list_progress = Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        TextColumn("{task.fields[status]}", table_column=Column(ratio=2)),
+    )
 
-        with Live(progress_table, console=console, refresh_per_second=10):
-            tasks = []
-            for name, spec in project_spec.specs.items():
-                tasks.append(
-                    self.process_module(
-                        name,
-                        spec,
-                        task_list_progress,
-                        task_count_progress,
-                        count_task,
-                    )
-                )
+    task_count_progress = Progress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+    )
+    count_task = task_count_progress.add_task("Processing Modules", total=len(project_spec.repos))
 
-            # this should error if a task crashes
-            return_codes = await asyncio.gather(*tasks)
-            if any(return_codes):
-                raise Exception()
+    progress_table = Table.grid()
+    progress_table.add_row(
+        task_list_progress,
+    )
+    progress_table.add_row(
+        task_count_progress,
+    )
 
+    semaphore = asyncio.Semaphore(concurrency)
+    with Live(progress_table, console=console, refresh_per_second=10):
+        tasks = []
+        for name, repo_info in project_spec.repos.items():
+            total_steps = len(repo_info.refspec_info) + 1
+            repo_processor = RepoProcessor(
+                project_spec.workdir,
+                name,
+                semaphore,
+                repo_info,
+                task_list_progress,
+                task_count_progress,
+                count_task,
+                concurrency,
+            )
+            tasks.append(repo_processor.process_repo())
 
-async def process_project(project_spec: ProjectSpec, concurrency: int = 4) -> None:
-    """Helper function to run the SpecProcessor."""
-    processor = SpecProcessor(project_spec.workdir, concurrency)
-    # project_spec.specs = {name: spec for name, spec in project_spec.specs.items() if name == "sale-workflow"}
-    return await processor.process_project(project_spec)
+        # this should error if a task crashes
+        return_codes = await asyncio.gather(*tasks)
+        if any(return_codes):
+            raise Exception()

bl/types.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+from dataclasses import dataclass
+from enum import Enum, IntEnum
+from typing import Dict, List, Optional
+
+
+class OriginType(Enum):
+    """Type of origin reference."""
+
+    BRANCH = "branch"
+    PR = "pr"
+    REF = "ref"
+
+
+@dataclass
+class Remote:
+    name: str
+    url: str
+
+
+class CloneFlags(IntEnum):
+    SHALLOW = 1
+    SPARSE = 2
+
+
+class RefspecInfo:
+    """A git refspec with its remote, type and optional frozen sha."""
+
+    def __init__(
+        self,
+        remote: str,
+        ref_str: str,
+        type: OriginType,
+        ref_name: Optional[str],
+    ):
+        self.remote = remote
+        self.refspec = ref_str
+        """ The refspec string (branch name, PR ref, or commit hash). """
+        self.type = type
+        self.ref_name = ref_name
+
+    def __repr__(self) -> str:
+        return f"RefspecInfo(remote={self.remote!r}, origin={self.refspec!r}, type={self.type.value})"
+
+
+@dataclass
+class CloneInfo:
+    url: str
+    clone_flags: int
+    root_refspec_info: RefspecInfo
+
+
+class RepoInfo:
+    """Represents the specification for a set of modules."""
+
+    def __init__(
+        self,
+        modules: List[str],
+        remotes: List[str] = {},
+        refspecs: List[RefspecInfo] = [],
+        shell_commands: List[str] = [],
+        patch_globs_to_apply: List[str] = [],
+        target_folder: Optional[str] = None,
+        locales: List[str] = [],
+    ):
+        self.modules = modules
+        self.remotes = remotes
+        self.refspec_info = refspecs
+        self.shell_commands = shell_commands
+        self.patch_globs_to_apply = patch_globs_to_apply
+        self.target_folder = target_folder
+        self.locales = locales
+
+    def __repr__(self) -> str:
+        return f"ModuleSpec(modules={self.modules}, remotes={self.remotes}, origins={self.refspec_info})"
+
+
+class ProjectSpec:
+    """Represents the overall project specification from the YAML file."""
+
+    def __init__(self, repos: Dict[str, RepoInfo], workdir: Path = Path(".")):
+        self.repos = repos
+        self.workdir = workdir
+
+    def __repr__(self) -> str:
+        return f"ProjectSpec(specs={self.repos}, workdir={self.workdir})"

bl/utils.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Optional
 
 import warnings
-from bl.spec_parser import ModuleSpec, OriginType, RefspecInfo
+from bl.types import RepoInfo, OriginType, RefspecInfo
 
 
 english_env = os.environ.copy()
@@ -12,7 +12,7 @@ english_env = os.environ.copy()
 english_env["LANG"] = "en_US.UTF-8"
 
 
-def get_module_path(workdir: Path, module_name: str, module_spec: ModuleSpec) -> Path:
+def get_module_path(workdir: Path, module_name: str, module_spec: RepoInfo) -> Path:
     """Returns the path to the module directory."""
     if module_name == "odoo" and module_spec.target_folder is None:
         warnings.warn(

{bl_odoo-0.2.6.dist-info → bl_odoo-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bl-odoo
-Version: 0.2.6
+Version: 0.3.0
 Summary: A command-line tool for managing Odoo dependencies.
 Author-email: Your Name <your.email@example.com>
 License-Expression: MIT

bl_odoo-0.3.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+bl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bl/__main__.py,sha256=v1d-voJ7N1QBLGJJh8JdrTxXtzf0JFHQv4RUBxlCkcg,1428
+bl/freezer.py,sha256=wUYNd0zKU-0OGdIXSLJol-_xJmxSSkXvzV_VbF2HJyg,2512
+bl/spec_parser.py,sha256=nZTwMh_ja4Mc7CKqB7-5cwECHrP15xd1S8zb_LW79BU,4953
+bl/spec_processor.py,sha256=yejkFFig_Yfbv1Ail1xTpteFv-f7dE7gasF_YU1oTXs,17867
+bl/types.py,sha256=h14FXDVCrYRxY7lYTEu8jhdrEHr1PvSNyZRIHm33CTk,2158
+bl/utils.py,sha256=JNLsxgJgWaa71Xs62gcoOwJnPjNHkrW0q9HFB0vQ60E,1600
+bl_odoo-0.3.0.dist-info/licenses/LICENSE,sha256=GTVQl3vH6ht70wJXKC0yMT8CmXKHxv_YyO_utAgm7EA,1065
+bl_odoo-0.3.0.dist-info/METADATA,sha256=VRKm91QHoxE_y5hSphmULSNRdjzvyvcSfBbQ5iRwafs,391
+bl_odoo-0.3.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+bl_odoo-0.3.0.dist-info/entry_points.txt,sha256=fmdGhYYJlP-XByamgaZdM0bo3JK4LJFswU_Nilq6SSw,39
+bl_odoo-0.3.0.dist-info/top_level.txt,sha256=1o4tN3wszdw7U5SnGgdF5P2sTYA0Schf0vKFy9_2D6A,3
+bl_odoo-0.3.0.dist-info/RECORD,,

bl_odoo-0.2.6.dist-info/RECORD

@@ -1,12 +0,0 @@
-bl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bl/__main__.py,sha256=v1d-voJ7N1QBLGJJh8JdrTxXtzf0JFHQv4RUBxlCkcg,1428
-bl/freezer.py,sha256=SXqfTXTys7tu-m94TftUjDLt7usNMkYoO10ZX62fRJg,2577
-bl/spec_parser.py,sha256=23_J5qf-20uSrKOu3drT0u2I1c9UwOGvp2Y7Ql8K5GE,7461
-bl/spec_processor.py,sha256=2_-nIlXd_M_eUpY44Q5VMtqPr-eyJn-nMZMDiDH9L0s,17579
-bl/utils.py,sha256=d6pmkwlMLU4jm94JMisd6LT31YJ_oyqgX50O3g5yzq4,1610
-bl_odoo-0.2.6.dist-info/licenses/LICENSE,sha256=GTVQl3vH6ht70wJXKC0yMT8CmXKHxv_YyO_utAgm7EA,1065
-bl_odoo-0.2.6.dist-info/METADATA,sha256=AZqNx-YmD-ERnmFVv7gUSgCcnQAbciiIem39qIDh3LU,391
-bl_odoo-0.2.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-bl_odoo-0.2.6.dist-info/entry_points.txt,sha256=fmdGhYYJlP-XByamgaZdM0bo3JK4LJFswU_Nilq6SSw,39
-bl_odoo-0.2.6.dist-info/top_level.txt,sha256=1o4tN3wszdw7U5SnGgdF5P2sTYA0Schf0vKFy9_2D6A,3
-bl_odoo-0.2.6.dist-info/RECORD,,