atlas-init 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

atlas_init/cli_tf/go_test_run.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Iterable
+from enum import StrEnum
+from functools import total_ordering
+
+import humanize
+from github.WorkflowJob import WorkflowJob
+from model_lib import Entity, Event, utc_datetime
+from pydantic import Field
+
+logger = logging.getLogger(__name__)
+
+
+class GoTestStatus(StrEnum):
+    RUN = "RUN"
+    PASS = "PASS"  # noqa: S105
+    FAIL = "FAIL"
+    SKIP = "SKIP"
+
+
+class Classification(StrEnum):
+    OUT_OF_CAPACITY = "OUT_OF_CAPACITY"
+    # DANGLING_RESOURCES = "DANGLING_RESOURCES"
+    # PERFORMANCE_REGRESSION = "PERFORMANCE_REGRESSION"
+    FIRST_TIME_ERROR = "FIRST_TIME_ERROR"
+    LEGIT_ERROR = "LEGIT_ERROR"
+    PANIC = "PANIC"
+
+
+class LineInfo(Event):
+    number: int
+    text: str
+
+
+@total_ordering
+class GoTestRun(Entity):
+    name: str
+    status: GoTestStatus = GoTestStatus.RUN
+    start_line: LineInfo
+    ts: utc_datetime
+    finish_ts: utc_datetime | None = None
+    job: WorkflowJob
+    test_step: int
+
+    finish_line: LineInfo | None = None
+    context_lines: list[str] = Field(default_factory=list)
+    run_seconds: float | None = None
+
+    classifications: set[Classification] = Field(default_factory=set)
+
+    def finish_summary(self) -> str:
+        finish_line = self.finish_line
+        lines = [
+            self.start_line.text if finish_line is None else finish_line.text,
+            self.url,
+        ]
+        return "\n".join(lines + self.context_lines)
+
+    def __lt__(self, other) -> bool:
+        if not isinstance(other, GoTestRun):
+            raise TypeError
+        return (self.ts, self.name) < (other.ts, other.name)
+
+    @property
+    def when(self) -> str:
+        return humanize.naturaltime(self.ts)
+
+    @property
+    def runtime_human(self) -> str:
+        if seconds := self.run_seconds:
+            return humanize.naturaldelta(seconds)
+        return "unknown"
+
+    @property
+    def context_lines_str(self) -> str:
+        return "\n".join(self.context_lines)
+
+    @property
+    def url(self) -> str:
+        line = self.finish_line or self.start_line
+        return f"{self.job.html_url}#step:{self.test_step}:{line.number}"
+
+    @property
+    def is_failure(self) -> bool:
+        return self.status == GoTestStatus.FAIL
+
+    def add_line_match(self, match: LineMatch, line: str, line_number: int) -> None:
+        self.run_seconds = match.run_seconds or self.run_seconds
+        self.finish_line = LineInfo(number=line_number, text=line)
+        self.status = match.status
+        self.finish_ts = match.ts
+
+    @classmethod
+    def from_line_match(
+        cls,
+        match: LineMatch,
+        line: str,
+        line_number: int,
+        job: WorkflowJob,
+        test_step_nr: int,
+    ) -> GoTestRun:
+        start_line = LineInfo(number=line_number, text=line)
+        return cls(
+            name=match.name,
+            status=match.status,
+            ts=match.ts,
+            run_seconds=match.run_seconds,
+            start_line=start_line,
+            job=job,
+            test_step=test_step_nr,
+        )
+
+
+class LineMatch(Event):
+    ts: utc_datetime
+    status: GoTestStatus
+    name: str
+    run_seconds: float | None = None
+
+
+_status_options = "|".join(list(GoTestStatus))
+line_result = re.compile(
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)\s[-=]+\s"
+    + r"(?P<status>%s):?\s+" % _status_options  # noqa: UP031
+    + r"(?P<name>[\w_]+)"
+    + r"\s*\(?(?P<run_seconds>[\d\.]+)?s?\)?"
+)
+
+
+def _test_name_is_nested(name: str, line: str) -> bool:
+    return f"{name}/" in line
+
+
+def match_line(line: str) -> LineMatch | None:
+    """
+    2024-06-26T04:41:47.7209465Z === RUN   TestAccNetworkDSPrivateLinkEndpoint_basic
+    2024-06-26T04:41:47.7228652Z --- PASS: TestAccNetworkRSPrivateLinkEndpointGCP_basic (424.50s)
+    """
+    if match := line_result.match(line):
+        line_match = LineMatch(**match.groupdict())
+        return None if _test_name_is_nested(line_match.name, line) else line_match
+    return None
+
+
+context_start_pattern = re.compile(
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)\s[-=]+\s" r"NAME\s+" r"(?P<name>[\w_]+)"
+)
+
+
+def context_start_match(line: str) -> str:
+    if match := context_start_pattern.match(line):
+        return match.groupdict()["name"]
+    return ""
+
+
+context_line_pattern = re.compile(
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)" r"\s{5}" r"(?P<indent>\s*)" r"(?P<relevant_line>.*)"
+)
+
+
+def extract_context(line: str) -> str:
+    if match := context_line_pattern.match(line):
+        match_vars = match.groupdict()
+        return match_vars["indent"] + match_vars["relevant_line"].strip()
+    return ""
+
+
+def parse(test_lines: list[str], job: WorkflowJob, test_step_nr: int) -> Iterable[GoTestRun]:
+    tests: dict[str, GoTestRun] = {}
+    context_lines: list[str] = []
+    current_context_test = ""
+    for line_nr, line in enumerate(test_lines, start=0):  # possibly an extra line in the log files we download
+        if current_context_test:
+            if more_context := extract_context(line):
+                context_lines.append(more_context)
+                continue
+            else:
+                tests[current_context_test].context_lines.extend(context_lines)
+                context_lines.clear()
+                current_context_test = ""
+        if new_context_test := context_start_match(line):
+            current_context_test = new_context_test
+            continue
+        if line_match := match_line(line):
+            if existing := tests.pop(line_match.name, None):
+                existing.add_line_match(line_match, line, line_nr)
+                yield existing
+            else:
+                tests[line_match.name] = GoTestRun.from_line_match(line_match, line, line_nr, job, test_step_nr)
+    if tests:
+        logger.warning(f"unfinished tests: {sorted(tests.keys())}")

atlas_init/cli_tf/go_test_run_format.py

@@ -0,0 +1,31 @@
+from collections import Counter
+
+from github.WorkflowJob import WorkflowJob
+from zero_3rdparty import datetime_utils
+
+from atlas_init.cli_tf.go_test_run import GoTestRun, GoTestStatus
+
+
+def format_job(job: WorkflowJob) -> str:
+    date = datetime_utils.date_filename(job.created_at)
+    exec_time = "0s"
+    if complete_ts := job.completed_at:
+        exec_time = f"{(complete_ts - job.created_at).total_seconds()}s"
+    return f"{date}_{job.workflow_name}_attempt{job.run_attempt}_ ({exec_time})"
+
+
+def job_summary(runs: list[GoTestRun]) -> tuple[WorkflowJob, str]:
+    status_counts: dict[GoTestStatus, int] = Counter()
+    for run in runs:
+        status_counts[run.status] += 1
+    line = [f"{key}={status_counts[key]}" for key in sorted(status_counts.keys())]
+    job = runs[0].job
+    return job, f"{format_job(job)}:" + ",".join(line)
+
+
+def fail_test_summary(runs: list[GoTestRun]) -> str:
+    failed_runs = [r for r in runs if r.is_failure]
+    failed_details: list[str] = [run.finish_summary() for run in failed_runs]
+    failed_names = [f"- {run.name}" for run in failed_runs]
+    delimiter = "\n" + "-" * 40 + "\n"
+    return "\n".join(failed_details) + delimiter + "\n".join(failed_names)

atlas_init/cli_tf/go_test_summary.py

@@ -0,0 +1,144 @@
+import logging
+from datetime import date, datetime, timedelta
+from functools import total_ordering
+
+from model_lib import Entity
+from pydantic import Field, model_validator
+from zero_3rdparty import datetime_utils, file_utils
+
+from atlas_init.cli_tf.github_logs import summary_dir
+from atlas_init.cli_tf.go_test_run import GoTestRun, GoTestStatus
+
+logger = logging.getLogger(__name__)
+_COMPLETE_STATUSES = {GoTestStatus.PASS, GoTestStatus.FAIL}
+
+
+@total_ordering
+class GoTestSummary(Entity):
+    name: str
+    results: list[GoTestRun] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def sort_results(self):
+        self.results.sort()
+        return self
+
+    @property
+    def total_completed(self) -> int:
+        return sum((r.status in _COMPLETE_STATUSES for r in self.results), 0)
+
+    @property
+    def success_rate(self) -> float:
+        total = self.total_completed
+        if total == 0:
+            logger.warning(f"No results to calculate success rate for {self.name}")
+            return 0
+        return sum(r.status == "PASS" for r in self.results) / total
+
+    @property
+    def is_skipped(self) -> bool:
+        return all(r.status == GoTestStatus.SKIP for r in self.results)
+
+    @property
+    def success_rate_human(self) -> str:
+        return f"{self.success_rate:.2%}"
+
+    def last_pass_human(self) -> str:
+        return next(
+            (f"Passed {test.when}" for test in reversed(self.results) if test.status == GoTestStatus.PASS),
+            "never passed",
+        )
+
+    def __lt__(self, other) -> bool:
+        if not isinstance(other, GoTestSummary):
+            raise TypeError
+        return (self.success_rate, self.name) < (other.success_rate, other.name)
+
+    def select_tests(self, date: date) -> list[GoTestRun]:
+        return [r for r in self.results if r.ts.date() == date]
+
+
+def summary_str(summary: GoTestSummary, start_date: datetime, end_date: datetime) -> str:
+    return "\n".join(
+        [
+            f"## {summary.name}",
+            f"Success rate: {summary.success_rate_human}",
+            "",
+            "### Timeline",
+            *timeline_lines(summary, start_date, end_date),
+            "",
+            *failure_details(summary),
+        ]
+    )
+
+
+def timeline_lines(summary: GoTestSummary, start_date: datetime, end_date: datetime) -> list[str]:
+    lines = []
+    one_day = timedelta(days=1)
+    for active_date in datetime_utils.day_range(start_date.date(), (end_date + one_day).date(), one_day):
+        active_tests = summary.select_tests(active_date)
+        if not active_tests:
+            lines.append(f"{active_date:%Y-%m-%d}: MISSING")
+            continue
+
+        tests_str = ", ".join(format_test_oneline(t) for t in active_tests)
+        lines.append(f"{active_date:%Y-%m-%d}: {tests_str}")
+    return lines
+
+
+def failure_details(summary: GoTestSummary) -> list[str]:
+    lines = ["## Failures"]
+    for test in summary.results:
+        if test.status == GoTestStatus.FAIL:
+            lines.extend(
+                (
+                    f"### {test.when} {format_test_oneline(test)}",
+                    test.finish_summary(),
+                    "",
+                )
+            )
+    return lines
+
+
+def format_test_oneline(test: GoTestRun) -> str:
+    return f"[{test.status} {test.runtime_human}]({test.url})"
+
+
+def create_detailed_summary(
+    summary_name: str,
+    end_test_date: datetime,
+    start_test_date: datetime,
+    test_results: dict[str, list[GoTestRun]],
+    expected_names: set[str] | None = None,
+) -> list[str]:
+    summary_dir_path = summary_dir(summary_name)
+    if summary_dir_path.exists():
+        file_utils.clean_dir(summary_dir_path)
+    summaries = [GoTestSummary(name=name, results=runs) for name, runs in test_results.items()]
+    top_level_summary = ["# SUMMARY OF ALL TESTS name (success rate)"]
+    summaries = [summary for summary in summaries if summary.results and not summary.is_skipped]
+    if expected_names and (skipped_names := expected_names - {summary.name for summary in summaries}):
+        logger.warning(f"skipped test names: {'\n'.join(skipped_names)}")
+        top_level_summary.append(f"Skipped tests: {', '.join(skipped_names)}")
+    for summary in sorted(summaries):
+        test_summary_path = summary_dir_path / f"{summary.success_rate_human}_{summary.name}.md"
+        test_summary_md = summary_str(summary, start_test_date, end_test_date)
+        file_utils.ensure_parents_write_text(test_summary_path, test_summary_md)
+        top_level_summary.append(f"- {summary.name} ({summary.success_rate_human}) ({summary.last_pass_human()})")
+    return top_level_summary
+
+
+def create_short_summary(test_results: dict[str, list[GoTestRun]], failing_names: list[str]) -> list[str]:
+    summary = ["# SUMMARY OF FAILING TESTS"]
+    summary_fail_details: list[str] = ["# FAIL DETAILS"]
+
+    for fail_name in failing_names:
+        fail_tests = test_results[fail_name]
+        summary.append(f"- {fail_name} has {len(fail_tests)} failures:")
+        summary.extend(
+            f"  - [{fail_run.when} failed in {fail_run.runtime_human}]({fail_run.url})" for fail_run in fail_tests
+        )
+        summary_fail_details.append(f"\n\n ## {fail_name} details:")
+        summary_fail_details.extend(f"```\n{fail_run.finish_summary()}\n```" for fail_run in fail_tests)
+    logger.info("\n".join(summary_fail_details))
+    return summary

atlas_init/cli_tf/hcl/cli.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+from collections.abc import Iterable
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from atlas_init.cli_tf.hcl.cluster_mig import (
+    LEGACY_CLUSTER_TYPE,
+    NEW_CLUSTER_TYPE,
+    convert_clusters,
+)
+from atlas_init.cli_tf.hcl.parser import (
+    ResourceBlock,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def should_continue(is_interactive: bool, question: str):
+    for h in logger.handlers:
+        h.flush()
+    return input(f"{question} [y/N]") == "y" if is_interactive else True
+
+
+def tf(cmd: str, tf_dir: Path, err_msg: str, err_msg_code_2: str = "") -> str:
+    err_codes = {
+        1: err_msg,
+        2: err_msg_code_2,
+    }
+    with TemporaryDirectory() as temp_dir:
+        result_file = Path(temp_dir) / "file"
+        with open(result_file, "w") as file:
+            exit_code = subprocess.call(
+                f"terraform {cmd}".split(),
+                stdin=sys.stdin,
+                stderr=sys.stderr,
+                stdout=file,
+                cwd=tf_dir,
+            )
+            cmd_output = result_file.read_text().strip()
+            if exit_code == 0:
+                return cmd_output
+            err_msg = err_codes.get(exit_code, err_msg) or err_msg
+            logger.error(cmd_output)
+            logger.error(err_msg)
+            sys.exit(exit_code)
+
+
+def convert_and_validate(tf_dir: Path, *, is_interactive: bool = False):
+    out_path = tf_dir / "conversion_cluster_adv_cluster.tf"
+    if out_path.exists() and should_continue(is_interactive, f"{out_path} already exists, should it be overwritten?"):
+        logger.info(f"removing existing conversion file @ {out_path}")
+        out_path.unlink()
+    if adv_clusters := read_import_id_addresses(tf_dir, NEW_CLUSTER_TYPE):
+        existing_addresses = ", ".join(adv_clusters.values())
+        if should_continue(
+            is_interactive,
+            f"found existing advanced clusters: {existing_addresses}, should they be removed?",
+        ):
+            remove_from_state(tf_dir, adv_clusters.values())
+    ensure_no_plan_changes(tf_dir)
+    new_clusters_path = convert_clusters(tf_dir, out_path)
+    logger.info(
+        f"found a total of {len(new_clusters_path)} clusters to convert and generated their config to {out_path}"
+    )
+    if should_continue(is_interactive, f"should import the new clusters in {out_path}?"):
+        import_new_clusters(tf_dir)
+        ensure_no_plan_changes(tf_dir)
+    else:
+        logger.info("skipping import")
+    if should_continue(is_interactive, "should replace the old cluster resources with the new ones?"):
+        replace_old_clusters(tf_dir, out_path, new_clusters_path)
+        ensure_no_plan_changes(tf_dir)
+        logger.info(f"migration successful, migrated {len(new_clusters_path)} clusters!")
+    else:
+        logger.info("skipping replacment")
+
+
+def remove_from_state(tf_dir, addresses: Iterable[str]) -> None:
+    for address in addresses:
+        logger.info(f"removing {address} from state")
+        tf(f"state rm {address}", tf_dir, f"failed to remove {address}")
+
+
+def ensure_no_plan_changes(tf_dir):
+    logger.info("running plan to ensure there are no changes")
+    tf(
+        "plan -detailed-exitcode",
+        tf_dir,
+        "error running terraform plan",
+        "plan had changes",
+    )
+
+
+def import_new_clusters(tf_dir: Path) -> None:
+    cluster_import_ids = read_import_id_addresses(tf_dir)
+    for import_id, resource_address in cluster_import_ids.items():
+        new_resource_address = resource_address.replace(LEGACY_CLUSTER_TYPE, NEW_CLUSTER_TYPE)
+        logger.info(f"importing {import_id} to {new_resource_address}")
+        tf(
+            f"import {new_resource_address} {import_id}",
+            tf_dir,
+            f"failed to import {new_resource_address}",
+        )
+
+
+def read_import_id_addresses(tf_dir: Path, resource_type: str = "") -> dict[str, str]:
+    current_state = tf("show -json", tf_dir, "failed to read terraform state")
+    return read_cluster_import_ids(current_state, resource_type)
+
+
+def replace_old_clusters(
+    tf_dir: Path,
+    out_path: Path,
+    new_clusters_path: dict[tuple[Path, ResourceBlock], str],
+) -> None:
+    out_path.unlink()
+    for (path, block), new_config in new_clusters_path.items():
+        old_resource_id = block.resource_id
+        logger.info(f"replacing {old_resource_id} @ {path}")
+        old_text = path.read_text()
+        new_text = old_text.replace(block.hcl, new_config)
+        path.write_text(new_text)
+    remove_from_state(tf_dir, read_import_id_addresses(tf_dir).values())
+
+
+def read_cluster_import_ids(state: str, resource_type: str = "") -> dict[str, str]:
+    resource_type = resource_type or LEGACY_CLUSTER_TYPE
+    try:
+        json_state = json.loads(state)
+    except json.JSONDecodeError:
+        logger.exception("unable to decode state")
+        sys.exit(1)
+    resources = json_state["values"]["root_module"]["resources"]
+    assert isinstance(resources, list)
+    import_ids = {}
+    for resource in resources:
+        if resource["type"] == resource_type:
+            project_id = resource["values"]["project_id"]
+            name = resource["values"]["name"]
+            import_id = f"{project_id}-{name}"
+            import_ids[import_id] = resource["address"]
+    return import_ids
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    *_, tf_dir_str = sys.argv
+    tf_dir_path = Path(tf_dir_str)
+    assert tf_dir_path.is_dir(), f"not a directory: {tf_dir_path}"
+    fast_forward = os.environ.get("FAST_FORWARD", "false").lower() in {
+        "yes",
+        "true",
+        "1",
+    }
+    convert_and_validate(tf_dir_path, is_interactive=not fast_forward)