devs-webhook 2.0.6__tar.gz → 2.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/PKG-INFO +1 -1
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/container_pool.py +51 -345
- devs_webhook-2.0.7/devs_webhook/core/repository_manager.py +115 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook.egg-info/PKG-INFO +1 -1
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/pyproject.toml +1 -1
- devs_webhook-2.0.6/devs_webhook/core/repository_manager.py +0 -197
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/LICENSE +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/README.md +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/__init__.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/app.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/cli/__init__.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/cli/worker.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/config.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/__init__.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/base_dispatcher.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/claude_dispatcher.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/deduplication.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/task_processor.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/test_dispatcher.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/webhook_config.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/core/webhook_handler.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/github/__init__.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/github/app_auth.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/github/client.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/github/models.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/github/parser.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/main_cli.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/sources/__init__.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/sources/base.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/sources/sqs_source.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/sources/webhook_source.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/__init__.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/async_utils.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/container_logs.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/github.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/logging.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/s3_artifacts.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook/utils/serialization.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook.egg-info/SOURCES.txt +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook.egg-info/dependency_links.txt +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook.egg-info/entry_points.txt +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook.egg-info/requires.txt +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/devs_webhook.egg-info/top_level.txt +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/setup.cfg +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_allowlist.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_authentication.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_authorized_users.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_ci_container_pool.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_cleanup_mode.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_container_logs.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_single_queue.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_sqs_burst.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_stop_container_after_task.py +0 -0
- {devs_webhook-2.0.6 → devs_webhook-2.0.7}/tests/test_webhook_parser.py +0 -0
|
@@ -15,6 +15,7 @@ from devs_common.core.container import ContainerManager
|
|
|
15
15
|
from devs_common.core.workspace import WorkspaceManager
|
|
16
16
|
from devs_common.devs_config import DevsConfigLoader, DevsOptions
|
|
17
17
|
from devs_common.utils.config_hash import compute_env_config_hash
|
|
18
|
+
from devs_common.utils.repo_cache import RepoCache
|
|
18
19
|
|
|
19
20
|
from ..config import get_config
|
|
20
21
|
from ..github.models import WebhookEvent, IssueEvent, PullRequestEvent, CommentEvent
|
|
@@ -76,6 +77,9 @@ class ContainerPool:
|
|
|
76
77
|
# Start worker tasks for each container
|
|
77
78
|
self._start_workers()
|
|
78
79
|
|
|
80
|
+
# Shared RepoCache instance for all git clone/update operations
|
|
81
|
+
self.repo_cache = self._build_repo_cache()
|
|
82
|
+
|
|
79
83
|
# Start the idle container cleanup task (optional - disabled for burst mode)
|
|
80
84
|
self._cleanup_worker_enabled = enable_cleanup_worker
|
|
81
85
|
if enable_cleanup_worker:
|
|
@@ -129,7 +133,30 @@ class ContainerPool:
|
|
|
129
133
|
return None
|
|
130
134
|
|
|
131
135
|
return devs_options
|
|
132
|
-
|
|
136
|
+
|
|
137
|
+
def _build_repo_cache(self, default_branch: Optional[str] = None) -> RepoCache:
|
|
138
|
+
"""Create a RepoCache configured for this webhook's settings.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
default_branch: Optional explicit default branch (from DEVS.yml).
|
|
142
|
+
When set, this branch is tried first.
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
A RepoCache instance.
|
|
146
|
+
"""
|
|
147
|
+
# Build branch preference list
|
|
148
|
+
if default_branch:
|
|
149
|
+
branches = [default_branch]
|
|
150
|
+
else:
|
|
151
|
+
branches = ["dev", "main", "master"]
|
|
152
|
+
|
|
153
|
+
return RepoCache(
|
|
154
|
+
cache_dir=self.config.repo_cache_dir,
|
|
155
|
+
token=self.config.github_token or None,
|
|
156
|
+
default_branches=branches,
|
|
157
|
+
clean=True,
|
|
158
|
+
)
|
|
159
|
+
|
|
133
160
|
async def ensure_repo_config(self, repo_name: str) -> DevsOptions:
|
|
134
161
|
"""Ensure repository configuration is loaded and cached.
|
|
135
162
|
|
|
@@ -265,10 +292,12 @@ class ContainerPool:
|
|
|
265
292
|
|
|
266
293
|
async def _ensure_repository_files_available(self, repo_name: str, repo_path: Path) -> None:
|
|
267
294
|
"""Ensure repository files are available locally without re-reading config.
|
|
268
|
-
|
|
295
|
+
|
|
269
296
|
This is used when we already have the DEVS.yml config cached but need
|
|
270
297
|
to ensure the actual repository files are available for the worker.
|
|
271
|
-
|
|
298
|
+
|
|
299
|
+
Delegates to the shared RepoCache for all git operations.
|
|
300
|
+
|
|
272
301
|
Args:
|
|
273
302
|
repo_name: Repository name (owner/repo)
|
|
274
303
|
repo_path: Path where repository should be cloned
|
|
@@ -277,99 +306,15 @@ class ContainerPool:
|
|
|
277
306
|
repo=repo_name,
|
|
278
307
|
repo_path=str(repo_path),
|
|
279
308
|
exists=repo_path.exists())
|
|
280
|
-
|
|
281
|
-
if repo_path.exists():
|
|
282
|
-
# Repository already exists, try to pull latest changes
|
|
283
|
-
try:
|
|
284
|
-
logger.info("Repository exists, fetching latest changes",
|
|
285
|
-
repo=repo_name,
|
|
286
|
-
repo_path=str(repo_path))
|
|
287
|
-
|
|
288
|
-
# Set up authentication for private repos
|
|
289
|
-
if self.config.github_token:
|
|
290
|
-
set_remote_cmd = ["git", "-C", str(repo_path), "remote", "set-url", "origin",
|
|
291
|
-
f"https://x-access-token:{self.config.github_token}@github.com/{repo_name}.git"]
|
|
292
|
-
|
|
293
|
-
process = await asyncio.create_subprocess_exec(*set_remote_cmd)
|
|
294
|
-
await process.wait()
|
|
295
|
-
|
|
296
|
-
# Fetch all branches to ensure we have all commits
|
|
297
|
-
fetch_cmd = ["git", "-C", str(repo_path), "fetch", "--all"]
|
|
298
|
-
process = await asyncio.create_subprocess_exec(
|
|
299
|
-
*fetch_cmd,
|
|
300
|
-
stdout=asyncio.subprocess.PIPE,
|
|
301
|
-
stderr=asyncio.subprocess.PIPE
|
|
302
|
-
)
|
|
303
|
-
stdout, stderr = await process.communicate()
|
|
304
|
-
|
|
305
|
-
if process.returncode != 0:
|
|
306
|
-
error_msg = stderr.decode('utf-8', errors='replace') if stderr else "Unknown error"
|
|
307
|
-
logger.warning("Failed to fetch repository, will try fresh clone",
|
|
308
|
-
repo=repo_name,
|
|
309
|
-
error=error_msg)
|
|
310
|
-
|
|
311
|
-
# Remove the directory and fall through to fresh clone
|
|
312
|
-
import shutil
|
|
313
|
-
shutil.rmtree(repo_path)
|
|
314
|
-
else:
|
|
315
|
-
logger.info("Repository fetch successful",
|
|
316
|
-
repo=repo_name)
|
|
317
|
-
|
|
318
|
-
# Checkout the default branch to ensure devcontainer files are from the right branch
|
|
319
|
-
await self._checkout_default_branch(repo_name, repo_path)
|
|
320
309
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
logger.warning("Error during repository pull, will try fresh clone",
|
|
325
|
-
repo=repo_name,
|
|
326
|
-
error=str(e))
|
|
327
|
-
# Remove the directory and fall through to fresh clone
|
|
328
|
-
import shutil
|
|
329
|
-
if repo_path.exists():
|
|
330
|
-
shutil.rmtree(repo_path)
|
|
331
|
-
|
|
332
|
-
# Clone repository fresh (either first time or after failed pull)
|
|
333
|
-
try:
|
|
334
|
-
logger.info("Cloning repository",
|
|
335
|
-
repo=repo_name,
|
|
336
|
-
repo_path=str(repo_path))
|
|
337
|
-
|
|
338
|
-
# Ensure parent directory exists
|
|
339
|
-
repo_path.parent.mkdir(parents=True, exist_ok=True)
|
|
340
|
-
|
|
341
|
-
# Clone with authentication if we have a token
|
|
342
|
-
if self.config.github_token:
|
|
343
|
-
clone_url = f"https://x-access-token:{self.config.github_token}@github.com/{repo_name}.git"
|
|
344
|
-
else:
|
|
345
|
-
clone_url = f"https://github.com/{repo_name}.git"
|
|
346
|
-
|
|
347
|
-
clone_cmd = ["git", "clone", clone_url, str(repo_path)]
|
|
348
|
-
process = await asyncio.create_subprocess_exec(
|
|
349
|
-
*clone_cmd,
|
|
350
|
-
stdout=asyncio.subprocess.PIPE,
|
|
351
|
-
stderr=asyncio.subprocess.PIPE
|
|
352
|
-
)
|
|
353
|
-
stdout, stderr = await process.communicate()
|
|
354
|
-
|
|
355
|
-
if process.returncode == 0:
|
|
356
|
-
logger.info("Repository cloned successfully",
|
|
357
|
-
repo=repo_name)
|
|
310
|
+
# Determine branch preference from user config
|
|
311
|
+
user_config = self._try_load_user_config(repo_name)
|
|
312
|
+
default_branch = user_config.default_branch if user_config and user_config.default_branch else None
|
|
358
313
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
else:
|
|
362
|
-
error_msg = stderr.decode('utf-8', errors='replace') if stderr else stdout.decode('utf-8', errors='replace')
|
|
363
|
-
logger.error("Git clone failed",
|
|
364
|
-
repo=repo_name,
|
|
365
|
-
error=error_msg)
|
|
366
|
-
raise Exception(f"Git clone failed: {error_msg}")
|
|
314
|
+
cache = self._build_repo_cache(default_branch=default_branch)
|
|
315
|
+
await asyncio.to_thread(cache.ensure_repo, repo_name)
|
|
367
316
|
|
|
368
|
-
|
|
369
|
-
logger.error("Repository cloning failed",
|
|
370
|
-
repo=repo_name,
|
|
371
|
-
error=str(e))
|
|
372
|
-
raise
|
|
317
|
+
logger.info("Repository files available", repo=repo_name)
|
|
373
318
|
|
|
374
319
|
def _get_pool_for_task_type(self, task_type: str) -> list[str]:
|
|
375
320
|
"""Get the appropriate container pool for a task type.
|
|
@@ -867,275 +812,36 @@ class ContainerPool:
|
|
|
867
812
|
# Just update last_used timestamp (legacy behavior)
|
|
868
813
|
self.running_containers[dev_name]["last_used"] = datetime.now(tz=timezone.utc)
|
|
869
814
|
|
|
870
|
-
async def _checkout_default_branch(self, repo_name: str, repo_path: Path) -> None:
|
|
871
|
-
"""Checkout the default branch to ensure devcontainer files are from the right branch.
|
|
872
|
-
|
|
873
|
-
Uses the default_branch from user-specific DEVS.yml config if available,
|
|
874
|
-
otherwise tries "dev", then "main".
|
|
875
|
-
|
|
876
|
-
Args:
|
|
877
|
-
repo_name: Repository name (owner/repo)
|
|
878
|
-
repo_path: Path to the cloned repository
|
|
879
|
-
"""
|
|
880
|
-
# Try to get default_branch from user config (no need to read repo config yet)
|
|
881
|
-
user_config = self._try_load_user_config(repo_name)
|
|
882
|
-
if user_config and user_config.default_branch:
|
|
883
|
-
default_branch = user_config.default_branch
|
|
884
|
-
else:
|
|
885
|
-
default_branch = "dev" # Try dev first, fall back to main
|
|
886
|
-
|
|
887
|
-
logger.info("Checking out default branch for devcontainer",
|
|
888
|
-
repo=repo_name,
|
|
889
|
-
branch=default_branch)
|
|
890
|
-
|
|
891
|
-
# Try to checkout the branch (use -f to discard local modifications from previous runs)
|
|
892
|
-
checkout_cmd = ["git", "-C", str(repo_path), "checkout", "-f", default_branch]
|
|
893
|
-
process = await asyncio.create_subprocess_exec(
|
|
894
|
-
*checkout_cmd,
|
|
895
|
-
stdout=asyncio.subprocess.PIPE,
|
|
896
|
-
stderr=asyncio.subprocess.PIPE
|
|
897
|
-
)
|
|
898
|
-
stdout, stderr = await process.communicate()
|
|
899
|
-
|
|
900
|
-
if process.returncode == 0:
|
|
901
|
-
logger.info("Checked out default branch",
|
|
902
|
-
repo=repo_name,
|
|
903
|
-
branch=default_branch)
|
|
904
|
-
# Reset to match remote branch so we pick up fetched changes
|
|
905
|
-
await self._reset_to_remote(repo_path, default_branch)
|
|
906
|
-
# Clean untracked files after checkout
|
|
907
|
-
await self._clean_untracked_files(repo_path)
|
|
908
|
-
elif default_branch == "dev":
|
|
909
|
-
# dev branch doesn't exist, try main
|
|
910
|
-
logger.info("Branch 'dev' not found, trying 'main'",
|
|
911
|
-
repo=repo_name)
|
|
912
|
-
checkout_cmd = ["git", "-C", str(repo_path), "checkout", "-f", "main"]
|
|
913
|
-
process = await asyncio.create_subprocess_exec(
|
|
914
|
-
*checkout_cmd,
|
|
915
|
-
stdout=asyncio.subprocess.PIPE,
|
|
916
|
-
stderr=asyncio.subprocess.PIPE
|
|
917
|
-
)
|
|
918
|
-
stdout, stderr = await process.communicate()
|
|
919
|
-
|
|
920
|
-
if process.returncode == 0:
|
|
921
|
-
logger.info("Checked out main branch",
|
|
922
|
-
repo=repo_name)
|
|
923
|
-
# Reset to match remote branch so we pick up fetched changes
|
|
924
|
-
await self._reset_to_remote(repo_path, "main")
|
|
925
|
-
# Clean untracked files after checkout
|
|
926
|
-
await self._clean_untracked_files(repo_path)
|
|
927
|
-
else:
|
|
928
|
-
# Both failed, stay on current branch (probably master or main after clone)
|
|
929
|
-
logger.warning("Could not checkout dev or main branch, staying on current branch",
|
|
930
|
-
repo=repo_name,
|
|
931
|
-
stderr=stderr.decode()[:200] if stderr else "")
|
|
932
|
-
else:
|
|
933
|
-
# Specified branch doesn't exist
|
|
934
|
-
logger.warning("Could not checkout branch, staying on current branch",
|
|
935
|
-
repo=repo_name,
|
|
936
|
-
branch=default_branch,
|
|
937
|
-
stderr=stderr.decode()[:200] if stderr else "")
|
|
938
|
-
|
|
939
|
-
async def _reset_to_remote(self, repo_path: Path, branch: str) -> None:
|
|
940
|
-
"""Reset local branch to match the remote tracking branch.
|
|
941
|
-
|
|
942
|
-
After 'git fetch --all', the local branch may still be behind origin.
|
|
943
|
-
This ensures the working tree matches the latest fetched remote state.
|
|
944
|
-
|
|
945
|
-
Args:
|
|
946
|
-
repo_path: Path to the repository
|
|
947
|
-
branch: Branch name to reset (e.g. "main")
|
|
948
|
-
"""
|
|
949
|
-
reset_cmd = ["git", "-C", str(repo_path), "reset", "--hard", f"origin/{branch}"]
|
|
950
|
-
process = await asyncio.create_subprocess_exec(
|
|
951
|
-
*reset_cmd,
|
|
952
|
-
stdout=asyncio.subprocess.PIPE,
|
|
953
|
-
stderr=asyncio.subprocess.PIPE
|
|
954
|
-
)
|
|
955
|
-
stdout, stderr = await process.communicate()
|
|
956
|
-
|
|
957
|
-
if process.returncode == 0:
|
|
958
|
-
logger.info("Reset branch to match remote",
|
|
959
|
-
repo_path=str(repo_path),
|
|
960
|
-
branch=branch)
|
|
961
|
-
else:
|
|
962
|
-
logger.warning("Could not reset to remote branch",
|
|
963
|
-
repo_path=str(repo_path),
|
|
964
|
-
branch=branch,
|
|
965
|
-
stderr=stderr.decode()[:200] if stderr else "")
|
|
966
|
-
|
|
967
|
-
async def _clean_untracked_files(self, repo_path: Path) -> None:
|
|
968
|
-
"""Remove untracked files and directories from repository.
|
|
969
|
-
|
|
970
|
-
Important for reusing repocache between tasks to avoid leftover files
|
|
971
|
-
from previous runs affecting the next task.
|
|
972
|
-
|
|
973
|
-
Args:
|
|
974
|
-
repo_path: Path to the repository
|
|
975
|
-
"""
|
|
976
|
-
clean_cmd = ["git", "-C", str(repo_path), "clean", "-fd"]
|
|
977
|
-
process = await asyncio.create_subprocess_exec(
|
|
978
|
-
*clean_cmd,
|
|
979
|
-
stdout=asyncio.subprocess.PIPE,
|
|
980
|
-
stderr=asyncio.subprocess.PIPE
|
|
981
|
-
)
|
|
982
|
-
stdout, stderr = await process.communicate()
|
|
983
|
-
|
|
984
|
-
if process.returncode == 0:
|
|
985
|
-
logger.info("Cleaned untracked files from repository",
|
|
986
|
-
repo_path=str(repo_path))
|
|
987
|
-
else:
|
|
988
|
-
logger.warning("Could not clean untracked files",
|
|
989
|
-
repo_path=str(repo_path),
|
|
990
|
-
stderr=stderr.decode()[:200] if stderr else "")
|
|
991
|
-
|
|
992
815
|
async def _ensure_repository_cloned(
|
|
993
816
|
self,
|
|
994
817
|
repo_name: str,
|
|
995
818
|
repo_path: Path
|
|
996
819
|
) -> DevsOptions:
|
|
997
820
|
"""Ensure repository is cloned to the workspace directory.
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
821
|
+
|
|
822
|
+
Delegates git clone/fetch/checkout to the shared RepoCache, then reads
|
|
823
|
+
DEVS.yml configuration.
|
|
824
|
+
|
|
1002
825
|
Args:
|
|
1003
826
|
repo_name: Repository name (owner/repo)
|
|
1004
827
|
repo_path: Path where repository should be cloned
|
|
1005
|
-
|
|
828
|
+
|
|
1006
829
|
Returns:
|
|
1007
830
|
DevsOptions parsed from DEVS.yml or defaults
|
|
1008
831
|
"""
|
|
1009
|
-
logger.info("
|
|
832
|
+
logger.info("Ensuring repository is cloned",
|
|
1010
833
|
repo=repo_name,
|
|
1011
|
-
repo_path=str(repo_path)
|
|
1012
|
-
exists=repo_path.exists())
|
|
1013
|
-
|
|
1014
|
-
if repo_path.exists():
|
|
1015
|
-
# Repository already exists, try to pull latest changes
|
|
1016
|
-
try:
|
|
1017
|
-
logger.info("Repository exists, fetching latest changes",
|
|
1018
|
-
repo=repo_name,
|
|
1019
|
-
repo_path=str(repo_path))
|
|
1020
|
-
|
|
1021
|
-
# Set up authentication for private repos
|
|
1022
|
-
if self.config.github_token:
|
|
1023
|
-
# Configure the token for this specific repo
|
|
1024
|
-
remote_url = f"https://{self.config.github_token}@github.com/{repo_name}.git"
|
|
1025
|
-
set_remote_cmd = ["git", "-C", str(repo_path), "remote", "set-url", "origin", remote_url]
|
|
1026
|
-
await asyncio.create_subprocess_exec(*set_remote_cmd)
|
|
1027
|
-
|
|
1028
|
-
# Fetch all branches to ensure we have all commits
|
|
1029
|
-
cmd = ["git", "-C", str(repo_path), "fetch", "--all"]
|
|
1030
|
-
process = await asyncio.create_subprocess_exec(
|
|
1031
|
-
*cmd,
|
|
1032
|
-
stdout=asyncio.subprocess.PIPE,
|
|
1033
|
-
stderr=asyncio.subprocess.PIPE
|
|
1034
|
-
)
|
|
1035
|
-
stdout, stderr = await process.communicate()
|
|
834
|
+
repo_path=str(repo_path))
|
|
1036
835
|
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
stdout=stdout.decode()[:200] if stdout else "")
|
|
836
|
+
# Determine branch preference from user config
|
|
837
|
+
user_config = self._try_load_user_config(repo_name)
|
|
838
|
+
default_branch = user_config.default_branch if user_config and user_config.default_branch else None
|
|
1041
839
|
|
|
1042
|
-
|
|
1043
|
-
|
|
840
|
+
cache = self._build_repo_cache(default_branch=default_branch)
|
|
841
|
+
await asyncio.to_thread(cache.ensure_repo, repo_name)
|
|
1044
842
|
|
|
1045
|
-
|
|
1046
|
-
else:
|
|
1047
|
-
# Fetch failed - remove and re-clone
|
|
1048
|
-
logger.warning("Git fetch failed, removing and re-cloning",
|
|
1049
|
-
repo=repo_name,
|
|
1050
|
-
return_code=process.returncode,
|
|
1051
|
-
stderr=stderr.decode()[:200] if stderr else "")
|
|
1052
|
-
|
|
1053
|
-
# Remove the existing directory
|
|
1054
|
-
logger.info("Removing existing repository directory",
|
|
1055
|
-
repo=repo_name,
|
|
1056
|
-
repo_path=str(repo_path))
|
|
1057
|
-
shutil.rmtree(repo_path)
|
|
1058
|
-
|
|
1059
|
-
# Now fall through to clone logic
|
|
1060
|
-
|
|
1061
|
-
except Exception as e:
|
|
1062
|
-
logger.warning("Failed to update repository, removing and re-cloning",
|
|
1063
|
-
repo=repo_name,
|
|
1064
|
-
error=str(e),
|
|
1065
|
-
error_type=type(e).__name__)
|
|
1066
|
-
|
|
1067
|
-
# Remove the existing directory
|
|
1068
|
-
try:
|
|
1069
|
-
shutil.rmtree(repo_path)
|
|
1070
|
-
logger.info("Removed existing repository directory",
|
|
1071
|
-
repo=repo_name,
|
|
1072
|
-
repo_path=str(repo_path))
|
|
1073
|
-
except Exception as rm_error:
|
|
1074
|
-
logger.error("Failed to remove repository directory",
|
|
1075
|
-
repo=repo_name,
|
|
1076
|
-
repo_path=str(repo_path),
|
|
1077
|
-
error=str(rm_error))
|
|
1078
|
-
raise
|
|
1079
|
-
|
|
1080
|
-
# If we get here, either the repo didn't exist or we removed it
|
|
1081
|
-
if not repo_path.exists():
|
|
1082
|
-
# Clone the repository
|
|
1083
|
-
try:
|
|
1084
|
-
logger.info("Repository does not exist, cloning",
|
|
1085
|
-
repo=repo_name,
|
|
1086
|
-
repo_path=str(repo_path))
|
|
1087
|
-
|
|
1088
|
-
repo_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1089
|
-
|
|
1090
|
-
# Use GitHub token for authentication
|
|
1091
|
-
if self.config.github_token:
|
|
1092
|
-
clone_url = f"https://{self.config.github_token}@github.com/{repo_name}.git"
|
|
1093
|
-
else:
|
|
1094
|
-
clone_url = f"https://github.com/{repo_name}.git"
|
|
1095
|
-
|
|
1096
|
-
cmd = ["git", "clone", clone_url, str(repo_path)]
|
|
1097
|
-
|
|
1098
|
-
# Don't log the token!
|
|
1099
|
-
safe_url = f"https://github.com/{repo_name}.git"
|
|
1100
|
-
logger.info("Starting git clone",
|
|
1101
|
-
repo=repo_name,
|
|
1102
|
-
clone_url=safe_url,
|
|
1103
|
-
target_path=str(repo_path))
|
|
1104
|
-
|
|
1105
|
-
process = await asyncio.create_subprocess_exec(
|
|
1106
|
-
*cmd,
|
|
1107
|
-
stdout=asyncio.subprocess.PIPE,
|
|
1108
|
-
stderr=asyncio.subprocess.PIPE
|
|
1109
|
-
)
|
|
1110
|
-
|
|
1111
|
-
stdout, stderr = await process.communicate()
|
|
1112
|
-
|
|
1113
|
-
logger.info("Git clone completed",
|
|
1114
|
-
repo=repo_name,
|
|
1115
|
-
return_code=process.returncode,
|
|
1116
|
-
stdout=stdout.decode()[:200] if stdout else "",
|
|
1117
|
-
stderr=stderr.decode()[:200] if stderr else "")
|
|
1118
|
-
|
|
1119
|
-
if process.returncode == 0:
|
|
1120
|
-
logger.info("Repository cloned successfully",
|
|
1121
|
-
repo=repo_name,
|
|
1122
|
-
path=str(repo_path))
|
|
843
|
+
logger.info("Repository ready", repo=repo_name, path=str(repo_path))
|
|
1123
844
|
|
|
1124
|
-
# Checkout the default branch to ensure devcontainer files are from the right branch
|
|
1125
|
-
await self._checkout_default_branch(repo_name, repo_path)
|
|
1126
|
-
else:
|
|
1127
|
-
error_msg = stderr.decode('utf-8', errors='replace')
|
|
1128
|
-
logger.error("Failed to clone repository",
|
|
1129
|
-
repo=repo_name,
|
|
1130
|
-
error=error_msg)
|
|
1131
|
-
raise Exception(f"Git clone failed: {error_msg}")
|
|
1132
|
-
|
|
1133
|
-
except Exception as e:
|
|
1134
|
-
logger.error("Repository cloning failed",
|
|
1135
|
-
repo=repo_name,
|
|
1136
|
-
error=str(e))
|
|
1137
|
-
raise
|
|
1138
|
-
|
|
1139
845
|
# Read DEVS.yml configuration using shared method
|
|
1140
846
|
devs_options = self._read_devs_options(repo_path, repo_name)
|
|
1141
847
|
return devs_options
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Repository management for webhook handler."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import shutil
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional, Dict
|
|
8
|
+
import structlog
|
|
9
|
+
|
|
10
|
+
from devs_common.utils.repo_cache import RepoCache
|
|
11
|
+
|
|
12
|
+
from ..config import get_config
|
|
13
|
+
from ..github.client import GitHubClient
|
|
14
|
+
|
|
15
|
+
logger = structlog.get_logger()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class RepositoryManager:
|
|
19
|
+
"""Manages repository cloning and caching for webhook tasks.
|
|
20
|
+
|
|
21
|
+
Delegates actual git operations to the shared :class:`RepoCache` utility
|
|
22
|
+
while adding async locking and cleanup on top.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self):
|
|
26
|
+
"""Initialize repository manager."""
|
|
27
|
+
self.config = get_config()
|
|
28
|
+
|
|
29
|
+
self.github_client = GitHubClient(self.config)
|
|
30
|
+
|
|
31
|
+
self.repo_cache = RepoCache(
|
|
32
|
+
cache_dir=self.config.repo_cache_dir,
|
|
33
|
+
token=self.config.github_token or None,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
# Track repository status
|
|
37
|
+
self.repo_locks: Dict[str, asyncio.Lock] = {}
|
|
38
|
+
|
|
39
|
+
logger.info("Repository manager initialized",
|
|
40
|
+
cache_dir=str(self.config.repo_cache_dir))
|
|
41
|
+
|
|
42
|
+
async def ensure_repository(
|
|
43
|
+
self,
|
|
44
|
+
repo_name: str,
|
|
45
|
+
clone_url: str
|
|
46
|
+
) -> Optional[Path]:
|
|
47
|
+
"""Ensure repository is available locally and up to date.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
repo_name: Repository name in format "owner/repo"
|
|
51
|
+
clone_url: Repository clone URL (kept for API compatibility)
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
Path to local repository or None if failed
|
|
55
|
+
"""
|
|
56
|
+
# Get or create lock for this repository
|
|
57
|
+
if repo_name not in self.repo_locks:
|
|
58
|
+
self.repo_locks[repo_name] = asyncio.Lock()
|
|
59
|
+
|
|
60
|
+
async with self.repo_locks[repo_name]:
|
|
61
|
+
try:
|
|
62
|
+
repo_dir = await asyncio.to_thread(
|
|
63
|
+
self.repo_cache.ensure_repo, repo_name
|
|
64
|
+
)
|
|
65
|
+
logger.info("Repository ready",
|
|
66
|
+
repo=repo_name, path=str(repo_dir))
|
|
67
|
+
return repo_dir
|
|
68
|
+
except Exception as e:
|
|
69
|
+
logger.error("Failed to ensure repository",
|
|
70
|
+
repo=repo_name,
|
|
71
|
+
error=str(e))
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
async def get_repository_info(self, repo_name: str) -> Optional[Dict]:
|
|
75
|
+
"""Get information about a repository.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
repo_name: Repository name
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
Repository info dict or None if not found
|
|
82
|
+
"""
|
|
83
|
+
return await self.github_client.get_repository_info(repo_name)
|
|
84
|
+
|
|
85
|
+
async def cleanup_old_repositories(self, max_age_days: int = 7) -> None:
|
|
86
|
+
"""Clean up old repository caches.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
max_age_days: Maximum age in days before cleanup
|
|
90
|
+
"""
|
|
91
|
+
try:
|
|
92
|
+
cutoff_time = time.time() - (max_age_days * 24 * 60 * 60)
|
|
93
|
+
|
|
94
|
+
if not self.config.repo_cache_dir.exists():
|
|
95
|
+
return
|
|
96
|
+
|
|
97
|
+
for repo_dir in self.config.repo_cache_dir.iterdir():
|
|
98
|
+
if repo_dir.is_dir():
|
|
99
|
+
# Check last modification time
|
|
100
|
+
mtime = repo_dir.stat().st_mtime
|
|
101
|
+
|
|
102
|
+
if mtime < cutoff_time:
|
|
103
|
+
logger.info("Cleaning up old repository cache",
|
|
104
|
+
repo=repo_dir.name,
|
|
105
|
+
age_days=(time.time() - mtime) / (24 * 60 * 60))
|
|
106
|
+
try:
|
|
107
|
+
shutil.rmtree(repo_dir)
|
|
108
|
+
logger.info("Repository removed", path=str(repo_dir))
|
|
109
|
+
except Exception as e:
|
|
110
|
+
logger.error("Failed to remove repository",
|
|
111
|
+
path=str(repo_dir),
|
|
112
|
+
error=str(e))
|
|
113
|
+
|
|
114
|
+
except Exception as e:
|
|
115
|
+
logger.error("Error during repository cleanup", error=str(e))
|
|
@@ -1,197 +0,0 @@
|
|
|
1
|
-
"""Repository management for webhook handler."""
|
|
2
|
-
|
|
3
|
-
import asyncio
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from typing import Optional, Dict
|
|
6
|
-
import structlog
|
|
7
|
-
|
|
8
|
-
from ..config import get_config
|
|
9
|
-
from ..github.client import GitHubClient
|
|
10
|
-
from ..utils.async_utils import run_git_async
|
|
11
|
-
|
|
12
|
-
logger = structlog.get_logger()
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class RepositoryManager:
|
|
16
|
-
"""Manages repository cloning and caching for webhook tasks."""
|
|
17
|
-
|
|
18
|
-
def __init__(self):
|
|
19
|
-
"""Initialize repository manager."""
|
|
20
|
-
self.config = get_config()
|
|
21
|
-
|
|
22
|
-
self.github_client = GitHubClient(self.config)
|
|
23
|
-
|
|
24
|
-
# Track repository status
|
|
25
|
-
self.repo_locks: Dict[str, asyncio.Lock] = {}
|
|
26
|
-
|
|
27
|
-
logger.info("Repository manager initialized",
|
|
28
|
-
cache_dir=str(self.config.repo_cache_dir))
|
|
29
|
-
|
|
30
|
-
async def ensure_repository(
|
|
31
|
-
self,
|
|
32
|
-
repo_name: str,
|
|
33
|
-
clone_url: str
|
|
34
|
-
) -> Optional[Path]:
|
|
35
|
-
"""Ensure repository is available locally and up to date.
|
|
36
|
-
|
|
37
|
-
Args:
|
|
38
|
-
repo_name: Repository name in format "owner/repo"
|
|
39
|
-
clone_url: Repository clone URL
|
|
40
|
-
|
|
41
|
-
Returns:
|
|
42
|
-
Path to local repository or None if failed
|
|
43
|
-
"""
|
|
44
|
-
# Get or create lock for this repository
|
|
45
|
-
if repo_name not in self.repo_locks:
|
|
46
|
-
self.repo_locks[repo_name] = asyncio.Lock()
|
|
47
|
-
|
|
48
|
-
async with self.repo_locks[repo_name]:
|
|
49
|
-
return await self._ensure_repository_locked(repo_name, clone_url)
|
|
50
|
-
|
|
51
|
-
async def _ensure_repository_locked(
|
|
52
|
-
self,
|
|
53
|
-
repo_name: str,
|
|
54
|
-
clone_url: str
|
|
55
|
-
) -> Optional[Path]:
|
|
56
|
-
"""Ensure repository is available (called with lock held).
|
|
57
|
-
|
|
58
|
-
Args:
|
|
59
|
-
repo_name: Repository name
|
|
60
|
-
clone_url: Repository clone URL
|
|
61
|
-
|
|
62
|
-
Returns:
|
|
63
|
-
Path to local repository or None if failed
|
|
64
|
-
"""
|
|
65
|
-
# Calculate local path
|
|
66
|
-
repo_dir = self.config.repo_cache_dir / repo_name.replace("/", "-")
|
|
67
|
-
|
|
68
|
-
try:
|
|
69
|
-
if repo_dir.exists():
|
|
70
|
-
# Repository exists, update it
|
|
71
|
-
logger.info("Updating existing repository",
|
|
72
|
-
repo=repo_name, path=str(repo_dir))
|
|
73
|
-
|
|
74
|
-
success = await self._update_repository(repo_dir)
|
|
75
|
-
if success:
|
|
76
|
-
return repo_dir
|
|
77
|
-
else:
|
|
78
|
-
# Update failed, try to reclone
|
|
79
|
-
logger.warning("Update failed, recloning repository",
|
|
80
|
-
repo=repo_name)
|
|
81
|
-
await self._remove_repository(repo_dir)
|
|
82
|
-
|
|
83
|
-
# Clone repository
|
|
84
|
-
logger.info("Cloning repository",
|
|
85
|
-
repo=repo_name, path=str(repo_dir))
|
|
86
|
-
|
|
87
|
-
success = await self.github_client.clone_repository(
|
|
88
|
-
repo_name, repo_dir
|
|
89
|
-
)
|
|
90
|
-
|
|
91
|
-
if success:
|
|
92
|
-
return repo_dir
|
|
93
|
-
else:
|
|
94
|
-
return None
|
|
95
|
-
|
|
96
|
-
except Exception as e:
|
|
97
|
-
logger.error("Failed to ensure repository",
|
|
98
|
-
repo=repo_name,
|
|
99
|
-
error=str(e))
|
|
100
|
-
return None
|
|
101
|
-
|
|
102
|
-
async def _update_repository(self, repo_dir: Path) -> bool:
|
|
103
|
-
"""Update an existing repository.
|
|
104
|
-
|
|
105
|
-
Args:
|
|
106
|
-
repo_dir: Path to repository directory
|
|
107
|
-
|
|
108
|
-
Returns:
|
|
109
|
-
True if update successful
|
|
110
|
-
"""
|
|
111
|
-
try:
|
|
112
|
-
# Fetch all remotes using async git
|
|
113
|
-
success, _, stderr = await run_git_async(
|
|
114
|
-
["fetch", "--all"],
|
|
115
|
-
str(repo_dir)
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
if not success:
|
|
119
|
-
logger.warning("Git fetch failed",
|
|
120
|
-
path=str(repo_dir),
|
|
121
|
-
error=stderr)
|
|
122
|
-
return False
|
|
123
|
-
|
|
124
|
-
# Reset to origin/main or origin/master
|
|
125
|
-
for branch in ["main", "master"]:
|
|
126
|
-
success, _, _ = await run_git_async(
|
|
127
|
-
["reset", "--hard", f"origin/{branch}"],
|
|
128
|
-
str(repo_dir)
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
if success:
|
|
132
|
-
logger.info("Repository updated",
|
|
133
|
-
path=str(repo_dir),
|
|
134
|
-
branch=branch)
|
|
135
|
-
return True
|
|
136
|
-
|
|
137
|
-
logger.warning("Could not reset to main or master branch",
|
|
138
|
-
path=str(repo_dir))
|
|
139
|
-
return False
|
|
140
|
-
|
|
141
|
-
except Exception as e:
|
|
142
|
-
logger.error("Error updating repository",
|
|
143
|
-
path=str(repo_dir),
|
|
144
|
-
error=str(e))
|
|
145
|
-
return False
|
|
146
|
-
|
|
147
|
-
async def _remove_repository(self, repo_dir: Path) -> None:
|
|
148
|
-
"""Remove a repository directory.
|
|
149
|
-
|
|
150
|
-
Args:
|
|
151
|
-
repo_dir: Path to repository directory
|
|
152
|
-
"""
|
|
153
|
-
try:
|
|
154
|
-
import shutil
|
|
155
|
-
shutil.rmtree(repo_dir)
|
|
156
|
-
logger.info("Repository removed", path=str(repo_dir))
|
|
157
|
-
except Exception as e:
|
|
158
|
-
logger.error("Failed to remove repository",
|
|
159
|
-
path=str(repo_dir),
|
|
160
|
-
error=str(e))
|
|
161
|
-
|
|
162
|
-
async def get_repository_info(self, repo_name: str) -> Optional[Dict]:
|
|
163
|
-
"""Get information about a repository.
|
|
164
|
-
|
|
165
|
-
Args:
|
|
166
|
-
repo_name: Repository name
|
|
167
|
-
|
|
168
|
-
Returns:
|
|
169
|
-
Repository info dict or None if not found
|
|
170
|
-
"""
|
|
171
|
-
return await self.github_client.get_repository_info(repo_name)
|
|
172
|
-
|
|
173
|
-
async def cleanup_old_repositories(self, max_age_days: int = 7) -> None:
|
|
174
|
-
"""Clean up old repository caches.
|
|
175
|
-
|
|
176
|
-
Args:
|
|
177
|
-
max_age_days: Maximum age in days before cleanup
|
|
178
|
-
"""
|
|
179
|
-
try:
|
|
180
|
-
import time
|
|
181
|
-
from datetime import datetime, timedelta
|
|
182
|
-
|
|
183
|
-
cutoff_time = time.time() - (max_age_days * 24 * 60 * 60)
|
|
184
|
-
|
|
185
|
-
for repo_dir in self.config.repo_cache_dir.iterdir():
|
|
186
|
-
if repo_dir.is_dir():
|
|
187
|
-
# Check last modification time
|
|
188
|
-
mtime = repo_dir.stat().st_mtime
|
|
189
|
-
|
|
190
|
-
if mtime < cutoff_time:
|
|
191
|
-
logger.info("Cleaning up old repository cache",
|
|
192
|
-
repo=repo_dir.name,
|
|
193
|
-
age_days=(time.time() - mtime) / (24 * 60 * 60))
|
|
194
|
-
await self._remove_repository(repo_dir)
|
|
195
|
-
|
|
196
|
-
except Exception as e:
|
|
197
|
-
logger.error("Error during repository cleanup", error=str(e))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|