machineconfig 2.7__py3-none-any.whl → 2.9__py3-none-any.whl

machineconfig/scripts/python/fire_agents.py

@@ -10,20 +10,25 @@ Improved design notes:
 from __future__ import annotations
 
 from pathlib import Path
-import shlex
 from math import ceil
 from typing import Literal, TypeAlias, get_args, Iterable
 
 from machineconfig.cluster.sessions_managers.zellij_local_manager import ZellijLocalManager
-from machineconfig.utils.schemas.layouts.layout_types import TabConfig, LayoutConfig
-from machineconfig.utils.utils2 import randstr
-import random
+from machineconfig.scripts.python.fire_agents_help_launch import launch_agents
+from machineconfig.scripts.python.fire_agents_help_search import search_files_by_pattern, search_python_files
+from machineconfig.utils.schemas.layouts.layout_types import LayoutConfig
 # import time
 
 AGENTS: TypeAlias = Literal[
     "cursor-agent", "gemini", "crush", "q", "onlyPrepPromptFiles"
     # warp terminal
 ]
+
+SPLITTING_STRATEGY: TypeAlias = Literal[
+    "agent_cap",  # User decides number of agents, rows/tasks determined automatically
+    "task_rows"   # User decides number of rows/tasks, number of agents determined automatically
+]
+
 DEFAULT_AGENT_CAP = 6
 
 
@@ -43,153 +48,86 @@ def get_gemini_api_keys() -> list[str]:
     return res
 
 
-def _search_python_files(repo_root: Path, keyword: str) -> list[Path]:
-    """Return all Python files under repo_root whose text contains keyword.
-
-    Notes:
-      - Skips any paths that reside under a directory named ".venv" at any depth.
-      - Errors reading individual files are ignored (decoded with 'ignore').
-    """
-    py_files = list(repo_root.rglob("*.py"))
-    keyword_lower = keyword.lower()
-    matches: list[Path] = []
-    for f in py_files:
-        # Skip anything under a .venv directory anywhere in the path
-        if any(part == ".venv" for part in f.parts):
-            continue
-        try:
-            if keyword_lower in f.read_text(encoding="utf-8", errors="ignore").lower():
-                matches.append(f)
-        except OSError:
-            # Skip unreadable file
-            continue
-    return matches
-
-
 def _write_list_file(target: Path, files: Iterable[Path]) -> None:
     target.parent.mkdir(parents=True, exist_ok=True)
     target.write_text("\n".join(str(f) for f in files), encoding="utf-8")
 
 
-def _chunk_prompts(prompts: list[str], max_agents: int) -> list[str]:
-    prompts = [p for p in prompts if p.strip() != ""]  # drop blank entries
-    if len(prompts) <= max_agents:
-        return prompts
-    print(f"Chunking {len(prompts)} prompts into groups for up to {max_agents} agents because it exceeds the cap.")
-    chunk_size = ceil(len(prompts) / max_agents)
-    grouped: list[str] = []
-    for i in range(0, len(prompts), chunk_size):
-        grouped.append("\nTargeted Locations:\n".join(prompts[i : i + chunk_size]))
-    return grouped
-
-
-def _confirm(message: str, default_no: bool = True) -> bool:
-    suffix = "[y/N]" if default_no else "[Y/n]"
-    answer = input(f"{message} {suffix} ").strip().lower()
-    if answer in {"y", "yes"}:
-        return True
-    if not default_no and answer == "":
-        return True
-    return False
-
-
-def launch_agents(repo_root: Path, prompts: list[str], agent: AGENTS, *, max_agents: int = DEFAULT_AGENT_CAP) -> list[TabConfig]:
-    """Create tab configuration for a set of agent prompts.
-
-    If number of prompts exceeds max_agents, ask user for confirmation.
-    (Original behavior raised an error; now interactive override.)
+def _chunk_prompts(prompts: list[str], strategy: SPLITTING_STRATEGY, *, agent_cap: int | None, task_rows: int | None) -> list[str]:
+    """Chunk prompts based on splitting strategy.
+    
+    Args:
+        prompts: List of prompts to chunk
+        strategy: Either 'agent_cap' or 'task_rows'  
+        agent_cap: Maximum number of agents (used with 'agent_cap' strategy)
+        task_rows: Number of rows/tasks per agent (used with 'task_rows' strategy)
     """
-    if not prompts:
-        raise ValueError("No prompts provided")
-
-    if len(prompts) > max_agents:
-        proceed = _confirm(message=(f"You are about to launch {len(prompts)} agents which exceeds the cap ({max_agents}). Proceed?"))
-        if not proceed:
-            print("Aborting per user choice.")
-            return []
-
-    tab_config: list[TabConfig] = []
-    tmp_dir = repo_root / ".ai" / f"tmp_prompts/{randstr()}"
-    tmp_dir.mkdir(parents=True, exist_ok=True)
-
-    for idx, a_prompt in enumerate(prompts):
-        prompt_path = tmp_dir / f"agent{idx}_prompt.txt"
-        prompt_path.write_text(a_prompt, encoding="utf-8")
-        cmd_path = tmp_dir / f"agent{idx}_cmd.sh"
-        match agent:
-            case "gemini":
-                # model = "gemini-2.5-pro"
-                # model = "gemini-2.5-flash-lite"
-                model = None  # auto-select
-                if model is None:
-                    model_arg = ""
-                else:
-                    model_arg = f"--model {shlex.quote(model)}"
-                # Need a real shell for the pipeline; otherwise '| gemini ...' is passed as args to 'cat'
-                safe_path = shlex.quote(str(prompt_path))
-                api_keys = get_gemini_api_keys()
-                api_key = api_keys[idx % len(api_keys)] if api_keys else ""
-                # Export the environment variable so it's available to subshells
-                cmd = f"""
-export GEMINI_API_KEY={shlex.quote(api_key)}
-echo "Using Gemini API key $GEMINI_API_KEY"
-cat {prompt_path}
-GEMINI_API_KEY={shlex.quote(api_key)} bash -lc 'cat {safe_path} | gemini {model_arg} --yolo --prompt'
-"""
-            case "cursor-agent":
-                # As originally implemented
-                cmd = f"""
-
-cursor-agent --print --output-format text < {prompt_path}
-
-"""
-            case "crush":
-                cmd = f"""
-# cat {prompt_path} | crush run
-crush run {prompt_path}
-"""
-            case "q":
-                cmd = f"""
-q chat --no-interactive --trust-all-tools {prompt_path}
-"""
-            case "onlyPrepPromptFiles":
-                cmd = f"""
-echo "Prepared prompt file at {prompt_path}"
-"""
-            case _:
-                raise ValueError(f"Unsupported agent type: {agent}")
-        random_sleep_time = random.uniform(0, 5)
-        cmd_prefix = f"""
-echo "Sleeping for {random_sleep_time:.2f} seconds to stagger agent startups..."
-sleep {random_sleep_time:.2f}
-echo "Launching `{agent}` with prompt from {prompt_path}"
-echo "Launching `{agent}` with command from {cmd_path}"
-echo "--------START OF AGENT OUTPUT--------"
-sleep 0.1
-"""
-        cmd_postfix = """
-sleep 0.1
-echo "---------END OF AGENT OUTPUT---------"
-"""
-        cmd_path.write_text(cmd_prefix + cmd + cmd_postfix, encoding="utf-8")
-        fire_cmd = f"bash {shlex.quote(str(cmd_path))}"
-        tab_config.append(TabConfig(tabName=f"Agent{idx}", startDir=str(repo_root), command=fire_cmd))
+    prompts = [p for p in prompts if p.strip() != ""]  # drop blank entries
+    
+    if strategy == "agent_cap":
+        if agent_cap is None:
+            raise ValueError("agent_cap must be provided when using 'agent_cap' strategy")
+        
+        if len(prompts) <= agent_cap:
+            return prompts
+        
+        print(f"Chunking {len(prompts)} prompts into groups for up to {agent_cap} agents because it exceeds the cap.")
+        chunk_size = ceil(len(prompts) / agent_cap)
+        grouped: list[str] = []
+        for i in range(0, len(prompts), chunk_size):
+            grouped.append("\nTargeted Locations:\n".join(prompts[i : i + chunk_size]))
+        return grouped
+    
+    elif strategy == "task_rows":
+        if task_rows is None:
+            raise ValueError("task_rows must be provided when using 'task_rows' strategy")
+        
+        if task_rows >= len(prompts):
+            return prompts
+            
+        print(f"Chunking {len(prompts)} prompts into groups of {task_rows} rows/tasks each.")
+        grouped: list[str] = []
+        for i in range(0, len(prompts), task_rows):
+            grouped.append("\nTargeted Locations:\n".join(prompts[i : i + task_rows]))
+        return grouped
+    
+    else:
+        raise ValueError(f"Unknown splitting strategy: {strategy}")
 
-    print(f"Launching a template with #{len(tab_config)} agents")
-    return tab_config
 
 
 def main():  # noqa: C901 - (complexity acceptable for CLI glue)
     repo_root = Path.cwd()
     print(f"Operating @ {repo_root}")
 
-    file_path_input = input("Enter path to target file [press Enter to generate it from searching]: ").strip()
-    if file_path_input == "":
+    from machineconfig.utils.options import choose_one_option
+
+    # Prompt user to choose search strategy
+    search_strategies = ["file_path", "keyword_search", "filename_pattern"]
+    search_strategy = choose_one_option(
+        header="Choose search strategy:",
+        options=search_strategies
+    )
+
+    # Execute chosen search strategy
+    if search_strategy == "file_path":
+        file_path_input = input("Enter path to target file: ").strip()
+        if not file_path_input:
+            print("No file path provided. Exiting.")
+            return
+        target_file_path = Path(file_path_input).expanduser().resolve()
+        if not target_file_path.exists() or not target_file_path.is_file():
+            print(f"Invalid file path: {target_file_path}")
+            return
+        separator = input("Enter separator [\\n]: ").strip() or "\n"
+        source_text = target_file_path.read_text(encoding="utf-8", errors="ignore")
+        
+    elif search_strategy == "keyword_search":
         keyword = input("Enter keyword to search recursively for all .py files containing it: ").strip()
         if not keyword:
             print("No keyword supplied. Exiting.")
             return
-        matching_files = _search_python_files(repo_root, keyword)
+        matching_files = search_python_files(repo_root, keyword)
         if not matching_files:
             print(f"💥 No .py files found containing keyword: {keyword}")
             return
@@ -200,33 +138,52 @@ def main():  # noqa: C901 - (complexity acceptable for CLI glue)
         _write_list_file(target_list_file, matching_files)
         separator = "\n"
         source_text = target_list_file.read_text(encoding="utf-8", errors="ignore")
-    else:
-        target_file_path = Path(file_path_input).expanduser().resolve()
-        if not target_file_path.exists() or not target_file_path.is_file():
-            print(f"Invalid file path: {target_file_path}")
+        
+    elif search_strategy == "filename_pattern":
+        pattern = input("Enter filename pattern (e.g., '*.py', '*test*', 'config.*'): ").strip()
+        if not pattern:
+            print("No pattern supplied. Exiting.")
             return
-        separator = input("Enter separator [\\n]: ").strip() or "\n"
-        if not target_file_path.exists():
-            print(f"File does not exist: {target_file_path}")
+        matching_files = search_files_by_pattern(repo_root, pattern)
+        if not matching_files:
+            print(f"💥 No files found matching pattern: {pattern}")
             return
-        source_text = target_file_path.read_text(encoding="utf-8", errors="ignore")
-
+        for idx, mf in enumerate(matching_files):
+            print(f"{idx:>3}: {mf}")
+        print(f"\nFound {len(matching_files)} files matching pattern: {pattern}")
+        target_list_file = repo_root / ".ai" / "target_file.txt"
+        _write_list_file(target_list_file, matching_files)
+        separator = "\n"
+        source_text = target_list_file.read_text(encoding="utf-8", errors="ignore")        
+    else:
+        raise ValueError(f"Unknown search strategy: {search_strategy}")
     raw_prompts = source_text.split(separator)
     print(f"Loaded {len(raw_prompts)} raw prompts from source.")
     prefix = input("Enter prefix prompt: ")
-    combined_prompts = _chunk_prompts(raw_prompts, DEFAULT_AGENT_CAP)
+    # Prompt user for splitting strategy
+    splitting_strategy = choose_one_option(header="Select splitting strategy", options=get_args(SPLITTING_STRATEGY))    
+    # Get parameters based on strategy
+    if splitting_strategy == "agent_cap":
+        agent_cap_input = input(f"Enter maximum number of agents/splits [default: {DEFAULT_AGENT_CAP}]: ").strip()
+        agent_cap = int(agent_cap_input) if agent_cap_input else DEFAULT_AGENT_CAP
+        combined_prompts = _chunk_prompts(raw_prompts, splitting_strategy, agent_cap=agent_cap, task_rows=None)
+        max_agents_for_launch = agent_cap
+    elif splitting_strategy == "task_rows":
+        task_rows_input = input("Enter number of rows/tasks per agent: ").strip()
+        if not task_rows_input:
+            print("Number of rows/tasks is required for this strategy.")
+            return
+        task_rows = int(task_rows_input)
+        combined_prompts = _chunk_prompts(raw_prompts, splitting_strategy, agent_cap=None, task_rows=task_rows)
+        max_agents_for_launch = len(combined_prompts)  # Number of agents determined by chunking
+    else:
+        raise ValueError(f"Unknown splitting strategy: {splitting_strategy}")
     combined_prompts = [prefix + "\n" + p for p in combined_prompts]
-
-    from machineconfig.utils.options import choose_one_option
-
     agent_selected = choose_one_option(header="Select agent type", options=get_args(AGENTS))
-
-    tab_config = launch_agents(repo_root=repo_root, prompts=combined_prompts, agent=agent_selected, max_agents=DEFAULT_AGENT_CAP)
+    tab_config = launch_agents(repo_root=repo_root, prompts=combined_prompts, agent=agent_selected, max_agents=max_agents_for_launch)
     if not tab_config:
         return
-
     from machineconfig.utils.utils2 import randstr
-
     random_name = randstr(length=3)
     manager = ZellijLocalManager(session_layouts=[LayoutConfig(layoutName="Agents", layoutTabs=tab_config)], session_name_prefix=random_name)
     manager.start_all_sessions()

machineconfig/scripts/python/fire_agents_help_launch.py

@@ -0,0 +1,97 @@
+from machineconfig.scripts.python.fire_agents import AGENTS, get_gemini_api_keys
+from machineconfig.utils.schemas.layouts.layout_types import TabConfig
+from machineconfig.utils.utils2 import randstr
+
+import random
+import shlex
+from pathlib import Path
+
+def _confirm(message: str, default_no: bool = False) -> bool:
+    from rich.prompt import Confirm
+    return Confirm.ask(message, default=not default_no)
+
+
+def launch_agents(repo_root: Path, prompts: list[str], agent: AGENTS, *, max_agents: int) -> list[TabConfig]:
+    """Create tab configuration for a set of agent prompts.
+
+    If number of prompts exceeds max_agents, ask user for confirmation.
+    (Original behavior raised an error; now interactive override.)
+    """
+    if not prompts:
+        raise ValueError("No prompts provided")
+
+    if len(prompts) > max_agents:
+        proceed = _confirm(message=(f"You are about to launch {len(prompts)} agents which exceeds the cap ({max_agents}). Proceed?"), default_no=True)
+        if not proceed:
+            print("Aborting per user choice.")
+            return []
+
+    tab_config: list[TabConfig] = []
+    tmp_dir = repo_root / ".ai" / f"tmp_prompts/{randstr()}"
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    for idx, a_prompt in enumerate(prompts):
+        prompt_path = tmp_dir / f"agent{idx}_prompt.txt"
+        prompt_path.write_text(a_prompt, encoding="utf-8")
+        cmd_path = tmp_dir / f"agent{idx}_cmd.sh"
+        match agent:
+            case "gemini":
+                # model = "gemini-2.5-pro"
+                # model = "gemini-2.5-flash-lite"
+                model = None  # auto-select
+                if model is None:
+                    model_arg = ""
+                else:
+                    model_arg = f"--model {shlex.quote(model)}"
+                # Need a real shell for the pipeline; otherwise '| gemini ...' is passed as args to 'cat'
+                safe_path = shlex.quote(str(prompt_path))
+                api_keys = get_gemini_api_keys()
+                api_key = api_keys[idx % len(api_keys)] if api_keys else ""
+                # Export the environment variable so it's available to subshells
+                cmd = f"""
+export GEMINI_API_KEY={shlex.quote(api_key)}
+echo "Using Gemini API key $GEMINI_API_KEY"
+cat {prompt_path}
+GEMINI_API_KEY={shlex.quote(api_key)} bash -lc 'cat {safe_path} | gemini {model_arg} --yolo --prompt'
+"""
+            case "cursor-agent":
+                # As originally implemented
+                cmd = f"""
+
+cursor-agent --print --output-format text < {prompt_path}
+
+"""
+            case "crush":
+                cmd = f"""
+# cat {prompt_path} | crush run
+crush run {prompt_path}
+"""
+            case "q":
+                cmd = f"""
+q chat --no-interactive --trust-all-tools {prompt_path}
+"""
+            case "onlyPrepPromptFiles":
+                cmd = f"""
+echo "Prepared prompt file at {prompt_path}"
+"""
+            case _:
+                raise ValueError(f"Unsupported agent type: {agent}")
+        random_sleep_time = random.uniform(0, 5)
+        cmd_prefix = f"""
+echo "Sleeping for {random_sleep_time:.2f} seconds to stagger agent startups..."
+sleep {random_sleep_time:.2f}
+echo "Launching `{agent}` with prompt from {prompt_path}"
+echo "Launching `{agent}` with command from {cmd_path}"
+echo "--------START OF AGENT OUTPUT--------"
+sleep 0.1
+"""
+        cmd_postfix = """
+sleep 0.1
+echo "---------END OF AGENT OUTPUT---------"
+"""
+        cmd_path.write_text(cmd_prefix + cmd + cmd_postfix, encoding="utf-8")
+        fire_cmd = f"bash {shlex.quote(str(cmd_path))}"
+        tab_config.append(TabConfig(tabName=f"Agent{idx}", startDir=str(repo_root), command=fire_cmd))
+
+    print(f"Launching a template with #{len(tab_config)} agents")
+    return tab_config

machineconfig/scripts/python/fire_agents_help_search.py

@@ -0,0 +1,83 @@
+from machineconfig.utils.source_of_truth import EXCLUDE_DIRS
+
+
+import fnmatch
+from pathlib import Path
+
+
+def search_files_by_pattern(repo_root: Path, pattern: str) -> list[Path]:
+    """Return all files under repo_root whose filename matches the given pattern.
+
+    Notes:
+      - Uses glob-style pattern matching (e.g., "*.py", "*test*", "config.*")
+      - Skips any paths that reside under directories listed in EXCLUDE_DIRS at any depth.
+    """
+    matches: list[Path] = []
+
+    def _should_skip_dir(dir_path: Path) -> bool:
+        """Check if directory should be skipped based on EXCLUDE_DIRS."""
+        return any(part in EXCLUDE_DIRS for part in dir_path.parts)
+
+    def _walk_and_filter(current_path: Path) -> None:
+        """Recursively walk directories, filtering out excluded ones early."""
+        try:
+            if not current_path.is_dir():
+                return
+
+            # Skip if current directory is in exclude list
+            if _should_skip_dir(current_path):
+                return
+
+            for item in current_path.iterdir():
+                if item.is_dir():
+                    _walk_and_filter(item)
+                elif item.is_file() and fnmatch.fnmatch(item.name, pattern):
+                    matches.append(item)
+        except (OSError, PermissionError):
+            # Skip directories we can't read
+            return
+
+    _walk_and_filter(repo_root)
+    return matches
+
+
+def search_python_files(repo_root: Path, keyword: str) -> list[Path]:
+    """Return all Python files under repo_root whose text contains keyword.
+
+    Notes:
+      - Skips any paths that reside under directories listed in EXCLUDE_DIRS at any depth.
+      - Errors reading individual files are ignored (decoded with 'ignore').
+    """
+    keyword_lower = keyword.lower()
+    matches: list[Path] = []
+
+    def _should_skip_dir(dir_path: Path) -> bool:
+        """Check if directory should be skipped based on EXCLUDE_DIRS."""
+        return any(part in EXCLUDE_DIRS for part in dir_path.parts)
+
+    def _walk_and_filter(current_path: Path) -> None:
+        """Recursively walk directories, filtering out excluded ones early."""
+        try:
+            if not current_path.is_dir():
+                return
+
+            # Skip if current directory is in exclude list
+            if _should_skip_dir(current_path):
+                return
+
+            for item in current_path.iterdir():
+                if item.is_dir():
+                    _walk_and_filter(item)
+                elif item.is_file() and item.suffix == ".py":
+                    try:
+                        if keyword_lower in item.read_text(encoding="utf-8", errors="ignore").lower():
+                            matches.append(item)
+                    except OSError:
+                        # Skip unreadable file
+                        continue
+        except (OSError, PermissionError):
+            # Skip directories we can't read
+            return
+
+    _walk_and_filter(repo_root)
+    return matches

machineconfig/scripts/python/helpers/cloud_helpers.py

@@ -1,14 +1,12 @@
 from pathlib import Path
 from machineconfig.utils.utils2 import pprint, read_json, read_ini
-from pydantic import ConfigDict
-from pydantic.dataclasses import dataclass
 from typing import Optional
 import os
 from machineconfig.utils.source_of_truth import DEFAULTS_PATH
 from rich.console import Console
 from rich.panel import Panel
 from rich import box  # Import box
-
+from dataclasses import dataclass
 
 console = Console()
 
@@ -26,8 +24,7 @@ class ArgsDefaults:
     key = None
     pwd = None
 
-
-@dataclass(config=ConfigDict(extra="forbid", frozen=False))
+@dataclass
 class Args:
     cloud: Optional[str] = None
 

machineconfig/scripts/python/repos.py

@@ -8,7 +8,7 @@ in the event that username@github.com is not mentioned in the remote url.
 from machineconfig.utils.source_of_truth import CONFIG_PATH, DEFAULTS_PATH
 from machineconfig.utils.path_reduced import PathExtended as PathExtended
 from machineconfig.utils.utils2 import randstr, read_ini
-from machineconfig.scripts.python.devops_update_repos import update_repository
+from machineconfig.scripts.python.repos_helper_update import update_repository
 from machineconfig.scripts.python.repos_helper_record import main as record_repos
 
 import argparse

machineconfig/scripts/python/repos_helper_record.py

@@ -9,6 +9,7 @@ from machineconfig.utils.io_save import save_json
 from typing import Optional
 
 from rich import print as pprint
+from rich.progress import Progress, TaskID, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn, MofNCompleteColumn
 
 
 def build_tree_structure(repos: list[RepoRecordDict], repos_root: PathExtended) -> str:
@@ -82,7 +83,7 @@ def build_tree_structure(repos: list[RepoRecordDict], repos_root: PathExtended)
     return "\n".join(tree_lines)
 
 
-def record_a_repo(path: PathExtended, search_parent_directories: bool = False, preferred_remote: Optional[str] = None) -> RepoRecordDict:
+def record_a_repo(path: PathExtended, search_parent_directories: bool, preferred_remote: Optional[str]) -> RepoRecordDict:
     from git.repo import Repo
 
     repo = Repo(path, search_parent_directories=search_parent_directories)  # get list of remotes using git python
@@ -126,27 +127,103 @@ def record_a_repo(path: PathExtended, search_parent_directories: bool = False, p
     return res
 
 
-def record_repos_recursively(repos_root: str, r: bool = True) -> list[RepoRecordDict]:
+def count_git_repositories(repos_root: str, r: bool) -> int:
+    """Count total git repositories for accurate progress tracking."""
+    path_obj = PathExtended(repos_root).expanduser().absolute()
+    if path_obj.is_file():
+        return 0
+    
+    search_res = path_obj.search("*", files=False, folders=True)
+    count = 0
+    
+    for a_search_res in search_res:
+        if a_search_res.joinpath(".git").exists():
+            count += 1
+        elif r:
+            count += count_git_repositories(str(a_search_res), r=r)
+    
+    return count
+
+
+def count_total_directories(repos_root: str, r: bool) -> int:
+    """Count total directories to scan for accurate progress tracking."""
+    path_obj = PathExtended(repos_root).expanduser().absolute()
+    if path_obj.is_file():
+        return 0
+    
+    search_res = path_obj.search("*", files=False, folders=True)
+    count = len(search_res)
+    
+    if r:
+        for a_search_res in search_res:
+            if not a_search_res.joinpath(".git").exists():
+                count += count_total_directories(str(a_search_res), r=r)
+    
+    return count
+
+
+def record_repos_recursively(repos_root: str, r: bool, progress: Progress | None, scan_task_id: TaskID | None, process_task_id: TaskID | None) -> list[RepoRecordDict]:
     path_obj = PathExtended(repos_root).expanduser().absolute()
     if path_obj.is_file():
         return []
+    
     search_res = path_obj.search("*", files=False, folders=True)
     res: list[RepoRecordDict] = []
+    
     for a_search_res in search_res:
+        if progress and scan_task_id:
+            progress.update(scan_task_id, description=f"Scanning: {a_search_res.name}")
+        
         if a_search_res.joinpath(".git").exists():
             try:
-                res.append(record_a_repo(a_search_res))
+                if progress and process_task_id:
+                    progress.update(process_task_id, description=f"Recording: {a_search_res.name}")
+                    
+                repo_record = record_a_repo(a_search_res, search_parent_directories=False, preferred_remote=None)
+                res.append(repo_record)
+                
+                if progress and process_task_id:
+                    progress.update(process_task_id, advance=1, description=f"Recorded: {repo_record['name']}")
             except Exception as e:
                 print(f"⚠️ Failed to record {a_search_res}: {e}")
         else:
             if r:
-                res += record_repos_recursively(str(a_search_res), r=r)
+                res += record_repos_recursively(str(a_search_res), r=r, progress=progress, scan_task_id=scan_task_id, process_task_id=process_task_id)
+        
+        if progress and scan_task_id:
+            progress.update(scan_task_id, advance=1)
+    
     return res
 
 
 def main(repos_root: PathExtended):
     print("\n📝 Recording repositories...")
-    repo_records = record_repos_recursively(repos_root=str(repos_root))
+    
+    # Count total directories and repositories for accurate progress tracking
+    print("🔍 Analyzing directory structure...")
+    total_dirs = count_total_directories(str(repos_root), r=True)
+    total_repos = count_git_repositories(str(repos_root), r=True)
+    print(f"📊 Found {total_dirs} directories to scan and {total_repos} git repositories to record")
+    
+    # Setup progress bars
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TimeElapsedColumn(),
+    ) as progress:
+        scan_task = progress.add_task("Scanning directories...", total=total_dirs)
+        process_task = progress.add_task("Recording repositories...", total=total_repos)
+        
+        repo_records = record_repos_recursively(
+            repos_root=str(repos_root), 
+            r=True,
+            progress=progress, 
+            scan_task_id=scan_task, 
+            process_task_id=process_task
+        )
+    
     res: RepoRecordFile = {"version": "0.1", "repos": repo_records}
     
     # Summary with warnings