atlas-init 0.4.4__py3-none-any.whl → 0.6.0__py3-none-any.whl

atlas_init/cli_tf/github_logs.py

@@ -1,12 +1,8 @@
 import logging
 import os
-from collections import defaultdict
 from collections.abc import Callable
-from concurrent.futures import Future, ThreadPoolExecutor, wait
-from datetime import datetime
 from functools import lru_cache
 from pathlib import Path
-from typing import NamedTuple
 
 import requests
 from github import Auth, Github
@@ -16,7 +12,6 @@ from github.WorkflowRun import WorkflowRun
 from github.WorkflowStep import WorkflowStep
 from zero_3rdparty import datetime_utils, file_utils
 
-from atlas_init.cli_tf.go_test_run import GoTestRun, parse
 from atlas_init.repos.path import (
     GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS,
 )
@@ -25,7 +20,6 @@ from atlas_init.settings.env_vars import init_settings
 logger = logging.getLogger(__name__)
 
 GH_TOKEN_ENV_NAME = "GH_TOKEN"  # noqa: S105 #nosec
-GITHUB_CI_SUMMARY_DIR_ENV_NAME = "GITHUB_CI_SUMMARY_DIR_ENV_NAME"
 REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME]
 MAX_DOWNLOADS = 5
 
@@ -44,101 +38,20 @@ def get_repo(repo_id: str) -> Repository:
     return g.get_repo(repo_id)
 
 
-_DEFAULT_FILESTEMS = {
-    "test-suite",
-    "terraform-compatibility-matrix",
-    # "acceptance-tests",
-}
-
-
-def include_filestems(stems: set[str]) -> Callable[[WorkflowRun], bool]:
-    def inner(run: WorkflowRun) -> bool:
-        workflow_stem = stem_name(run.path)
-        return workflow_stem in stems
-
-    return inner
-
-
-def stem_name(workflow_path: str) -> str:
-    return Path(workflow_path).stem
-
-
 def tf_repo() -> Repository:
     return get_repo(GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS)
 
 
-class WorkflowJobId(NamedTuple):
-    workflow_id: int
-    job_id: int
-
-
-def find_test_runs(
-    since: datetime,
-    include_workflow: Callable[[WorkflowRun], bool] | None = None,
-    include_job: Callable[[WorkflowJob], bool] | None = None,
-    branch: str = "master",
-) -> dict[WorkflowJobId, list[GoTestRun]]:
-    include_workflow = include_workflow or include_filestems(_DEFAULT_FILESTEMS)
-    include_job = include_job or include_test_jobs()
-    jobs_found = defaultdict(list)
-    repository = tf_repo()
-    for workflow in repository.get_workflow_runs(
-        created=f">{since.strftime('%Y-%m-%d')}",
-        branch=branch,  # type: ignore
-        exclude_pull_requests=True,  # type: ignore
-    ):
-        if not include_workflow(workflow):
-            continue
-        workflow_dir = workflow_logs_dir(workflow)
-        paginated_jobs = workflow.jobs("all")
-        worker_count = min(paginated_jobs.totalCount, 10) or 1
-        with ThreadPoolExecutor(max_workers=worker_count) as pool:
-            futures: dict[Future[list[GoTestRun]], WorkflowJob] = {}
-            for job in paginated_jobs:
-                if not include_job(job):
-                    continue
-                future = pool.submit(find_job_test_runs, workflow_dir, job)
-                futures[future] = job
-            done, not_done = wait(futures.keys(), timeout=300)
-            for f in not_done:
-                logger.warning(f"timeout to find go tests for job = {futures[f].html_url}")
-        workflow_id = workflow.id
-        for f in done:
-            job = futures[f]
-            try:
-                go_test_runs: list[GoTestRun] = f.result()
-            except Exception:
-                job_log_path = logs_file(workflow_dir, job)
-                logger.exception(
-                    f"failed to find go tests for job: {job.html_url}, error 👆, local_path: {job_log_path}"
-                )
-                continue
-            jobs_found[WorkflowJobId(workflow_id, job.id)].extend(go_test_runs)
-    return jobs_found
-
-
-def find_job_test_runs(workflow_dir: Path, job: WorkflowJob) -> list[GoTestRun]:
-    jobs_log_path = download_job_safely(workflow_dir, job)
-    return [] if jobs_log_path is None else parse_job_logs(job, jobs_log_path)
-
-
-def parse_job_logs(job: WorkflowJob, logs_path: Path) -> list[GoTestRun]:
-    if job.conclusion in {"skipped", "cancelled", None}:
-        return []
-    step, logs_lines = select_step_and_log_content(job, logs_path)
-    test_runs = list(parse(logs_lines, job, step))
-    for run in test_runs:
-        run.log_path = logs_path
-    return test_runs
-
-
 def download_job_safely(workflow_dir: Path, job: WorkflowJob) -> Path | None:
+    if job.conclusion in {"skipped", "cancelled", None}:
+        logger.debug(f"not downloading job: {job.html_url}, conclusion: {job.conclusion}")
+        return None
     path = logs_file(workflow_dir, job)
     job_summary = f"found test job: {job.name}, attempt {job.run_attempt}, {job.created_at}, url: {job.html_url}"
     if path.exists():
-        logger.info(f"{job_summary} exist @ {path}")
+        logger.debug(f"{job_summary} exist @ {path}")
         return path
-    logger.info(f"{job_summary}\n\t\t downloading to {path}")
+    logger.debug(f"{job_summary}\n\t\t downloading to {path}")
     try:
         logs_response = requests.get(job.logs_url(), timeout=60)
         logs_response.raise_for_status()
@@ -160,14 +73,14 @@ def summary_dir(summary_name: str) -> Path:
 def workflow_logs_dir(workflow: WorkflowRun) -> Path:
     dt = workflow.created_at
     date_str = datetime_utils.get_date_as_rfc3339_without_time(dt)
-    workflow_name = stem_name(workflow.path)
+    workflow_name = Path(workflow.path).stem
     return logs_dir() / f"{date_str}/{workflow.id}_{workflow_name}"
 
 
 def logs_file(workflow_dir: Path, job: WorkflowJob) -> Path:
     if job.run_attempt != 1:
         workflow_dir = workflow_dir.with_name(f"{workflow_dir.name}_attempt{job.run_attempt}")
-    filename = f"{job.id}_" + job.name.replace(" ", "").replace("/", "_").replace("__", "_") + ".txt"
+    filename = f"{job.id}_" + job.name.replace(" ", "").replace("/", "_").replace("__", "_") + ".log"
     return workflow_dir / filename