qontract-reconcile 0.10.1rc216__py3-none-any.whl → 0.10.1rc218__py3-none-any.whl

{qontract_reconcile-0.10.1rc216.dist-info → qontract_reconcile-0.10.1rc218.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc216
+Version: 0.10.1rc218
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc216.dist-info → qontract_reconcile-0.10.1rc218.dist-info}/RECORD

@@ -23,12 +23,12 @@ reconcile/github_repo_invites.py,sha256=n5_gUcGotBBnvx3F41auCqX_gbZKut_XRTWGlsrK
 reconcile/github_repo_permissions_validator.py,sha256=dcbXdUx6imjNchjp3pg9-z1i7lFEGOr_28GvsiwO5Xw,1734
 reconcile/github_users.py,sha256=nfTq78QRONIfDVj-5O3bD6psllJjzWFnog-EJ1WqFPU,3672
 reconcile/github_validator.py,sha256=cVTVxJIGR4a1Jz8wrdXEAb_CMpXUzvykVmUURX4cook,917
-reconcile/gitlab_fork_compliance.py,sha256=XEwm5tlTvoY95py4MsBHgGcw6_8tWtJ9hLyUKa6y2_k,4246
-reconcile/gitlab_housekeeping.py,sha256=UesYfhhQMzXkL9oWTr2yh8r0wZKIEe9PshTAYTXsqk4,22169
+reconcile/gitlab_fork_compliance.py,sha256=nLrwtoj5bTlaMutpKUc4R6lqZtpqB0otcIOMNfIxUsU,4223
+reconcile/gitlab_housekeeping.py,sha256=9rZZ97R0eMHW67YksHM2ATmHLQoq62bmimfUSeI_M0Q,21276
 reconcile/gitlab_labeler.py,sha256=avNifNROyGhko6WDAaTH3ixA86qzxdBsSk1IisRoxao,4635
 reconcile/gitlab_members.py,sha256=M6LwFOrwgvl1NNdOJa1mrQFUon-bEVv1AyhGeLed454,8443
 reconcile/gitlab_mr_sqs_consumer.py,sha256=O46mdziPgGOndbU-0_UJKJVUaiEoVzJPEgKm4_UvYoI,2571
-reconcile/gitlab_owners.py,sha256=3OwGjPsKzKBtp0l2wWfh1RwnWYDeLejykCq5SHof0vk,13922
+reconcile/gitlab_owners.py,sha256=n5_pX841OST3dGfMUEXtqIcyqV4NjzSEvh2E0up7Y1c,13876
 reconcile/gitlab_permissions.py,sha256=ciEKj_wnRbS_vs_ZwcUeD6HkWVe3osAuotFqJSmvd94,1638
 reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
 reconcile/integrations_manager.py,sha256=zUCh1bYrnNoT_6SSQO-yYA2QdDxfCuzwb1tjcByIOaE,8885
@@ -126,7 +126,7 @@ reconcile/aws_ami_cleanup/integration.py,sha256=E--71oG3HhrUgBieAf51wNE1N1xtsuOa
 reconcile/change_owners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/change_owners/approver.py,sha256=L7XJWJ-rgn8BOmeMb6lBDV8lHFCUaNoHGDSD7OH03vA,2244
 reconcile/change_owners/bundle.py,sha256=Bh9v08tIw8TieG4EE9kVI9iFN1sW7Z6Zyb8DTbsv6F0,5371
-reconcile/change_owners/change_owners.py,sha256=L3Glirkd5a6TnPyseANq8THhnV6oJP_2PcqAfxhkjxE,13257
+reconcile/change_owners/change_owners.py,sha256=2Z182TKDpcvPwLtyTDbHdA11IMXuWrBSQxUoeUmdXbU,13268
 reconcile/change_owners/change_types.py,sha256=3UFjyuTW0-si9pvaKfqGCWGwTiBVi7n9HZnfHk_x5iY,31486
 reconcile/change_owners/changes.py,sha256=KrtaDTM_jxyYsHPUEjJCWJr186L4a-NsxCnztdSUry4,16912
 reconcile/change_owners/decision.py,sha256=JBMhNG8UqaKzTZgaKQLTPn8KW8CSEvq8clP76ZfQT6s,6381
@@ -330,7 +330,7 @@ reconcile/test/test_closedbox_endpoint_monitoring.py,sha256=isMHYwRWMFARU2nbJgbl
 reconcile/test/test_gabi_authorized_users.py,sha256=vyO9-1w6OtGQJvJuTazJMVAjQbUd90sgsHcMyIYkUC8,2483
 reconcile/test/test_github_org.py,sha256=j3KeB4OnSln1gm2hidce49xdMru-j75NS3cM-AEgzZc,4511
 reconcile/test/test_github_repo_invites.py,sha256=QJ0VFk5B59rx4XtHoT6XOGWw9xRIZMen_cgtviN_Vi8,3419
-reconcile/test/test_gitlab_housekeeping.py,sha256=ScFp3ztym2Q4lJwuTclq9HIHaT8JAymGoP1t-ynRFCQ,10049
+reconcile/test/test_gitlab_housekeeping.py,sha256=7EpWikaWJH52IlA2PZs7vz4GEgID-_dfZBpJNlwhMSw,10018
 reconcile/test/test_gitlab_labeler.py,sha256=vFLUJXSIaCduj6wSqgw7Fg7FhlopaDnYI5SLzNHtLoY,4362
 reconcile/test/test_gitlab_members.py,sha256=dP_dm-1THba9Vyzcq-EX1tdmBoX2hq8R-MY4Uqknq5s,9896
 reconcile/test/test_instrumented_wrappers.py,sha256=CZzhnQH0c4i7-Rxjg7-0dfFMvVPegLHL46z5NHOOCwo,608
@@ -464,7 +464,7 @@ reconcile/utils/filtering.py,sha256=dw7Ok7HXjZb0ruvCWHFh194rtunX1COLDTRnNfOpwQU,
 reconcile/utils/git.py,sha256=kgjN93MMB5mnkuNb1n53f5kldGGf5u0pBHj9YJbiE_c,1455
 reconcile/utils/git_secrets.py,sha256=897nRs7tycA3m7YYeVEbzOhI8RFrI9IJT2E0di1eJhc,1956
 reconcile/utils/github_api.py,sha256=6gdlKK0W3vZpxbbtOcohRgvZ4YkiSki7Gxdb16goHPo,2316
-reconcile/utils/gitlab_api.py,sha256=6udOgddf0qcTSgJTJcSHL5YJrUKiAFnTg7ZrHXKbhso,24939
+reconcile/utils/gitlab_api.py,sha256=v_cf2xt7trUlYFRYsuJ9ZkCEtFiO_0coAjlxBjjHedc,24839
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=KogegLFsvjiTWqPBDSb4qJToISrdsDeLJ3gkHwi1DQ8,11672
 reconcile/utils/helm.py,sha256=IWlB_LrBK6ydwNQuZP2aMJGrtQw0lW7qdFqSrd3r8lg,1321
@@ -508,7 +508,7 @@ reconcile/utils/sqs_gateway.py,sha256=gFl9DM4DmGnptuxTOe4lS3YTyE80eSAvK42ljS8h4d
 reconcile/utils/state.py,sha256=_SmE7fOEReET3iy9jRQ1pyuaJebg5962Zs9Iy1dzTJk,9530
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
-reconcile/utils/terraform_client.py,sha256=DUUuqKiTOWGQ3tj8zCyJP_7v45VWi1o5iGvqe3Dunco,30544
+reconcile/utils/terraform_client.py,sha256=Sy125P0LybUUkI9_BTK00m_Q4-Oc6GOAPBAQ6aRirCQ,33329
 reconcile/utils/terrascript_aws_client.py,sha256=FP4LSlKynxAjdEqfotsML7fg6lFyKktXcWQrLghFuU0,260201
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/unleash.py,sha256=QGANGA8BHG7oC_bt39c2M7uRa2ycjzmahN8_m7Zovos,3094
@@ -577,8 +577,8 @@ tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/test_qontract_cli.py,sha256=awwTHEc2DWlykuqGIYM0WOBoSL0KRnOraCLk3C7izis,1401
 tools/test/test_sd_app_sre_alert_report.py,sha256=JeLhgzpKCPgLvptwg_4ZvJHLVWKNG1T5845HXTkMBxA,1826
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc216.dist-info/METADATA,sha256=ftHv5boIaF1JALKk5Ee5kSTEQVvbQSluaTFltsAmlsQ,2328
-qontract_reconcile-0.10.1rc216.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
-qontract_reconcile-0.10.1rc216.dist-info/entry_points.txt,sha256=ErVY2Jp-0Rtuq5KOtMlW5yvna4nIEuc_1YbEdEdcy9o,301
-qontract_reconcile-0.10.1rc216.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc216.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc218.dist-info/METADATA,sha256=NYrL4Jw0x9csqevY8_axCXnpuJXmobKSqauptV79qqY,2328
+qontract_reconcile-0.10.1rc218.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
+qontract_reconcile-0.10.1rc218.dist-info/entry_points.txt,sha256=ErVY2Jp-0Rtuq5KOtMlW5yvna4nIEuc_1YbEdEdcy9o,301
+qontract_reconcile-0.10.1rc218.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc218.dist-info/RECORD,,

reconcile/change_owners/change_owners.py

@@ -372,9 +372,8 @@ def run(
             # e.g. gitlab-housekeeper rejects direct lgtm labels and the review-queue
             # skips MRs with this label
             if SELF_SERVICEABLE in labels:
-                gl.remove_label_from_merge_request(
-                    gitlab_merge_request_id, SELF_SERVICEABLE
-                )
+                merge_request = gl.get_merge_request(gitlab_merge_request_id)
+                gl.remove_label(merge_request, SELF_SERVICEABLE)
 
     except BaseException:
         logging.error(traceback.format_exc())

reconcile/gitlab_fork_compliance.py

@@ -80,7 +80,7 @@ class GitlabForkCompliance:
         # it is set
         mr_labels = self.gl_cli.get_merge_request_labels(self.mr.iid)
         if BLOCKED_BOT_ACCESS in mr_labels:
-            self.gl_cli.remove_label_from_merge_request(self.mr.iid, BLOCKED_BOT_ACCESS)
+            self.gl_cli.remove_label(self.mr, BLOCKED_BOT_ACCESS)
 
         sys.exit(self.exit_code)
 

reconcile/gitlab_housekeeping.py

@@ -214,8 +214,7 @@ def handle_stale_items(
     now = datetime.utcnow()
     for item in items:
         item_iid = item.attributes.get("iid")
-        item_labels = get_labels(item, gl)
-        if AUTO_MERGE in item_labels:
+        if AUTO_MERGE in item.labels:
             if item.merge_status == MRStatus.UNCHECKED:
                 # this call triggers a status recheck
                 item = gl.get_merge_request(item_iid)
@@ -227,16 +226,16 @@ def handle_stale_items(
         current_interval = now.date() - update_date.date()
         if current_interval > timedelta(days=days_interval):
             # if item does not have 'stale' label - add it
-            if LABEL not in item_labels:
+            if LABEL not in item.labels:
                 logging.info(["add_label", gl.project.name, item_type, item_iid, LABEL])
                 if not dry_run:
-                    gl.add_label(item, item_type, LABEL)
+                    gl.add_label(item, LABEL)
             # if item has 'stale' label - close it
             else:
                 close_item(dry_run, gl, enable_closing, item_type, item)
         # if item is under days_interval
         else:
-            if LABEL not in item_labels:
+            if LABEL not in item.labels:
                 continue
 
             # if item has 'stale' label - check the notes
@@ -261,7 +260,7 @@ def handle_stale_items(
                     ["remove_label", gl.project.name, item_type, item_iid, LABEL]
                 )
                 if not dry_run:
-                    gl.remove_label(item, item_type, LABEL)
+                    gl.remove_label(item, LABEL)
 
 
 def is_good_to_merge(labels):
@@ -277,21 +276,6 @@ def is_rebased(mr, gl: GitLabApi) -> bool:
     return len(result["commits"]) == 0
 
 
-def get_labels(mr: ProjectMergeRequest, gl: GitLabApi) -> list[str]:
-    """
-    This function used to contain logic for checking if labels were empty and calling
-    gl.get_merge_request_labels() if they were missing because there were reports of the
-    label attribute being empty sometimes when it shouldn't be. This was an expensive
-    approach, increasing the runtime of the integration by something around 20-30%.
-    Through investigation in APPSRE-6653 it was determined this no longer appears to be
-    an issue.
-
-    This is being left to continue to abstract the way that labels are pulled in case
-    this becomes an issue again in the future.
-    """
-    return mr.attributes.get("labels")
-
-
 def get_merge_requests(
     dry_run: bool,
     gl: GitLabApi,
@@ -324,7 +308,7 @@ def preprocess_merge_requests(
         if len(mr.commits()) == 0:
             continue
 
-        labels = get_labels(mr, gl)
+        labels = mr.labels
         if not labels:
             continue
 
@@ -336,7 +320,7 @@ def preprocess_merge_requests(
                 + "suitable for self serviceable MRs. removing 'lgtm' label"
             )
             if not dry_run:
-                gl.remove_label_from_merge_request(mr.iid, LGTM)
+                gl.remove_label(mr, LGTM)
             continue
 
         label_events = gl.get_merge_request_label_events(mr)
@@ -366,18 +350,17 @@ def preprocess_merge_requests(
                     approved_by = added_by
 
         for bad_label in labels_by_unauthorized_users - labels_by_authorized_users:
-            if bad_label not in labels:
+            if bad_label not in mr.labels:
                 continue
             logging.warning(
                 f"[{gl.project.name}/{mr.iid}] someone added a label who "
                 f"isn't allowed. removing label {bad_label}"
             )
-            # Remove bad_label from the cached labels list. Otherwise, we may face a caching bug
-            labels.remove(bad_label)
             if not dry_run:
-                gl.remove_label_from_merge_request(mr.iid, bad_label)
+                # TODO: optimize this to remove all bad labels at once
+                gl.remove_label(mr, bad_label)
 
-        if not is_good_to_merge(labels):
+        if not is_good_to_merge(mr.labels):
             continue
 
         label_priority = min(
@@ -540,7 +523,7 @@ def merge_merge_requests(
         if not dry_run and merges < merge_limit:
             try:
                 mr.merge()
-                labels = get_labels(mr, gl)
+                labels = mr.labels
                 merged_merge_requests.labels(
                     project_id=mr.target_project_id,
                     self_service=SELF_SERVICEABLE in labels,

reconcile/gitlab_owners.py

@@ -370,7 +370,7 @@ def act(repo, dry_run, instance, settings, defer=None):
                         f"- removing approval"
                     ]
                 )
-                gitlab_cli.remove_label_from_merge_request(mr.iid, APPROVED)
+                gitlab_cli.remove_label(mr, APPROVED)
 
         if approval_status["report"] is not None:
             _LOG.info(
@@ -382,7 +382,7 @@ def act(repo, dry_run, instance, settings, defer=None):
             )
 
             if not dry_run:
-                gitlab_cli.remove_label_from_merge_request(mr.iid, APPROVED)
+                gitlab_cli.remove_label(mr, APPROVED)
                 mr.notes.create({"body": approval_status["report"]})
             continue
 

reconcile/test/test_gitlab_housekeeping.py

@@ -177,9 +177,7 @@ def can_be_merged_merge_request() -> ProjectMergeRequest:
     mr.merge_status = "can_be_merged"
     mr.work_in_progress = False
     mr.commits.return_value = [create_autospec(ProjectCommit)]
-    mr.attributes = {
-        "labels": ["lgtm"],
-    }
+    mr.labels = ["lgtm"]
     mr.iid = 1
     mr.target_project_id = 3
     mr.author = {"username": "user"}

reconcile/utils/gitlab_api.py

@@ -14,6 +14,7 @@ import gitlab
 import urllib3
 from gitlab.v4.objects import (
     CurrentUser,
+    ProjectIssue,
     ProjectMergeRequest,
     ProjectMergeRequestNote,
 )
@@ -449,14 +450,6 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
         merge_request = self.project.mergerequests.get(mr_id)
         self.update_labels(merge_request, "merge-request", labels)
 
-    def remove_label_from_merge_request(self, mr_id, label):
-        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-        merge_request = self.project.mergerequests.get(mr_id)
-        labels = merge_request.attributes.get("labels")
-        if label in labels:
-            labels.remove(label)
-        self.update_labels(merge_request, "merge-request", labels)
-
     def add_comment_to_merge_request(self, mr_id, body):
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
         merge_request = self.project.mergerequests.get(mr_id)
@@ -480,20 +473,34 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
         self.project.labels.create({"name": label_text, "color": label_color})
 
-    def add_label(self, item, item_type, label):
-        note_body = (
-            "item has been marked as {0}. " "to remove say `/{0} cancel`"
-        ).format(label)
-        labels = item.attributes.get("labels")
+    @staticmethod
+    def add_label(
+        item: ProjectMergeRequest | ProjectIssue,
+        label: str,
+    ):
+        labels = item.labels
+        if label in labels:
+            return
         labels.append(label)
+        note_body = (
+            f"item has been marked as {label}. " f"to remove say `/{label} cancel`"
+        )
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
         item.notes.create({"body": note_body})
-        self.update_labels(item, item_type, labels)
+        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
+        item.save()
 
-    def remove_label(self, item, item_type, label):
-        labels = item.attributes.get("labels")
+    @staticmethod
+    def remove_label(
+        item: ProjectMergeRequest | ProjectIssue,
+        label: str,
+    ):
+        labels = item.labels
+        if label not in labels:
+            return
         labels.remove(label)
-        self.update_labels(item, item_type, labels)
+        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
+        item.save()
 
     def update_labels(self, item, item_type, labels):
         if item_type == "issue":

reconcile/utils/terraform_client.py

@@ -1,11 +1,17 @@
 import json
 import logging
+import os
+import re
 import shutil
+import tempfile
+import typing
 from collections import defaultdict
 from collections.abc import (
+    Generator,
     Iterable,
     Mapping,
 )
+from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import (
     datetime,
@@ -18,6 +24,7 @@ from typing import (
     cast,
 )
 
+import python_terraform
 from botocore.errorfactory import ClientError
 from python_terraform import (
     IsFlagged,
@@ -39,6 +46,8 @@ from reconcile.utils.external_resource_spec import (
 
 ALLOWED_TF_SHOW_FORMAT_VERSION = "0.1"
 DATE_FORMAT = "%Y-%m-%d"
+PROVIDER_LOG_REGEX = r""".*(?:\[INFO]|\[WARN]|\[ERROR]).+(?:\[WARN]|\[ERROR]).*"""
+TERRAFORM_LOG_LEVEL = "TRACE"  # can change to INFO after tf 0.15
 
 
 @dataclass
@@ -72,6 +81,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         self.thread_pool_size = thread_pool_size
         self._aws_api = aws_api
         self._log_lock = Lock()
+        self._tf_env_lock = Lock()
         self.should_apply = False
 
         self.init_specs()
@@ -110,30 +120,76 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
 
     def init_specs(self):
         wd_specs = [{"name": name, "wd": wd} for name, wd in self.working_dirs.items()]
-        results = threaded.run(self.terraform_init, wd_specs, self.thread_pool_size)
+        with self._monkey_patch_terraform_env():
+            results = threaded.run(self.terraform_init, wd_specs, self.thread_pool_size)
         self.specs = [{"name": name, "tf": tf} for name, tf in results]
 
+    @contextmanager
+    def _monkey_patch_terraform_env(self) -> Generator[None, None, None]:
+        # https://github.com/beelit94/python-terraform/blob/release/0.10.1/python_terraform/__init__.py#L290
+        old_copy = python_terraform.os.environ.copy
+
+        def new_copy():
+            result = old_copy()
+            self._tf_env_lock.release()
+            return result
+
+        python_terraform.os.environ.copy = new_copy
+
+        try:
+            yield
+        finally:
+            python_terraform.os.environ.copy = old_copy
+
+    @contextmanager
+    def _terraform_log_file(self, working_dir: str) -> Generator[typing.IO, None, None]:
+        with tempfile.NamedTemporaryFile(dir=working_dir) as f:
+            # lock is released in terraform method monkey patched in _monkey_patch_terraform_env
+            self._tf_env_lock.acquire()
+            os.environ["TF_LOG"] = TERRAFORM_LOG_LEVEL
+            os.environ["TF_LOG_PATH"] = f.name
+            try:
+                yield f
+            except Exception:
+                # release lock if exception raised before monkey patched os.environ.copy called
+                if self._tf_env_lock.locked():
+                    self._tf_env_lock.release()
+                raise
+            finally:
+                with self._tf_env_lock:
+                    if "TF_LOG" in os.environ:
+                        del os.environ["TF_LOG"]
+                    if "TF_LOG_PATH" in os.environ:
+                        del os.environ["TF_LOG_PATH"]
+
     @retry(exceptions=TerraformCommandError)
     def terraform_init(self, init_spec):
         name = init_spec["name"]
         wd = init_spec["wd"]
-        tf = Terraform(working_dir=wd)
-        return_code, stdout, stderr = tf.init()
-        error = self.check_output(name, "init", return_code, stdout, stderr)
+        tf = Terraform(working_dir=wd, is_env_vars_included=True)
+        with self._terraform_log_file(tf.working_dir) as f:
+            return_code, stdout, stderr = tf.init()
+            log = f.read().decode("utf-8")
+        error = self.check_output(name, "init", return_code, stdout, stderr, log)
         if error:
             raise TerraformCommandError(return_code, "init", out=stdout, err=stderr)
         return name, tf
 
     def init_outputs(self):
-        results = threaded.run(self.terraform_output, self.specs, self.thread_pool_size)
+        with self._monkey_patch_terraform_env():
+            results = threaded.run(
+                self.terraform_output, self.specs, self.thread_pool_size
+            )
         self.outputs = dict(results)
 
     @retry(exceptions=TerraformCommandError)
     def terraform_output(self, spec):
         name = spec["name"]
         tf = spec["tf"]
-        return_code, stdout, stderr = tf.output_cmd(json=IsFlagged)
-        error = self.check_output(name, "output", return_code, stdout, stderr)
+        with self._terraform_log_file(tf.working_dir) as f:
+            return_code, stdout, stderr = tf.output_cmd(json=IsFlagged)
+            log = f.read().decode("utf-8")
+        error = self.check_output(name, "output", return_code, stdout, stderr, log)
         no_output_error = (
             "The module root could not be found. There is nothing to output."
         )
@@ -150,12 +206,13 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
     def plan(self, enable_deletion):
         errors = False
         disabled_deletions_detected = False
-        results = threaded.run(
-            self.terraform_plan,
-            self.specs,
-            self.thread_pool_size,
-            enable_deletion=enable_deletion,
-        )
+        with self._monkey_patch_terraform_env():
+            results = threaded.run(
+                self.terraform_plan,
+                self.specs,
+                self.thread_pool_size,
+                enable_deletion=enable_deletion,
+            )
 
         self.created_users = []
         for disabled_deletion_detected, created_users, error in results:
@@ -172,10 +229,12 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
     ) -> tuple[bool, list[AccountUser], bool]:
         name = plan_spec["name"]
         tf = plan_spec["tf"]
-        return_code, stdout, stderr = tf.plan(
-            detailed_exitcode=False, parallelism=self.parallelism, out=name
-        )
-        error = self.check_output(name, "plan", return_code, stdout, stderr)
+        with self._terraform_log_file(tf.working_dir) as f:
+            return_code, stdout, stderr = tf.plan(
+                detailed_exitcode=False, parallelism=self.parallelism, out=name
+            )
+            log = f.read().decode("utf-8")
+        error = self.check_output(name, "plan", return_code, stdout, stderr, log)
         disabled_deletion_detected, created_users = self.log_plan_diff(
             name, tf, enable_deletion
         )
@@ -370,7 +429,10 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
     def apply(self):
         errors = False
 
-        results = threaded.run(self.terraform_apply, self.specs, self.thread_pool_size)
+        with self._monkey_patch_terraform_env():
+            results = threaded.run(
+                self.terraform_apply, self.specs, self.thread_pool_size
+            )
 
         for error in results:
             if error:
@@ -380,10 +442,12 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
     def terraform_apply(self, apply_spec):
         name = apply_spec["name"]
         tf = apply_spec["tf"]
-        # adding var=None to allow applying the saved plan
-        # https://github.com/beelit94/python-terraform/issues/67
-        return_code, stdout, stderr = tf.apply(dir_or_plan=name, var=None)
-        error = self.check_output(name, "apply", return_code, stdout, stderr)
+        with self._terraform_log_file(tf.working_dir) as f:
+            # adding var=None to allow applying the saved plan
+            # https://github.com/beelit94/python-terraform/issues/67
+            return_code, stdout, stderr = tf.apply(dir_or_plan=name, var=None)
+            log = f.read().decode("utf-8")
+        error = self.check_output(name, "apply", return_code, stdout, stderr, log)
         return error
 
     def get_terraform_output_secrets(self) -> dict[str, dict[str, dict[str, str]]]:
@@ -553,22 +617,26 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         name: str,
         cmd: str,
         return_code: int,
-        stdout: list[str],
-        stderr: list[str],
+        stdout: str,
+        stderr: str,
+        log: str,
     ) -> bool:
-        error_occured = False
+        error_occured = return_code != 0
         line_format = "[{} - {}] {}"
-        stdout, stderr = self.split_to_lines(stdout, stderr)
+        stdout, stderr, log = self.split_to_lines(stdout, stderr, log)
+        provider_log_re = re.compile(PROVIDER_LOG_REGEX)
         with self._log_lock:
             for line in stdout:
                 logging.debug(line_format.format(name, cmd, line))
-            if return_code == 0:
+            if error_occured:
                 for line in stderr:
-                    logging.warning(line_format.format(name, cmd, line))
+                    logging.error(line_format.format(name, cmd, line))
             else:
                 for line in stderr:
-                    logging.error(line_format.format(name, cmd, line))
-                error_occured = True
+                    logging.warning(line_format.format(name, cmd, line))
+            for line in log:
+                if provider_log_re.match(line):
+                    logging.warning(line_format.format(name, cmd, line))
         return error_occured
 
     @staticmethod