qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/state.py

@@ -1,29 +1,38 @@
+import contextlib
 import json
+import logging
 import os
-from collections.abc import (
-    Iterable,
-    Mapping,
-)
+from abc import abstractmethod
+from collections.abc import Callable, Generator, Mapping
+from dataclasses import dataclass, field
 from typing import (
+    TYPE_CHECKING,
     Any,
-    Optional,
+    Self,
 )
 
+import boto3
 from botocore.errorfactory import ClientError
-from jinja2 import Template
-from mypy_boto3_s3 import S3Client
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client
+else:
+    S3Client = object
+
+from pydantic import BaseModel
 
 from reconcile.gql_definitions.common.app_interface_state_settings import (
     AppInterfaceStateConfigurationS3V1,
 )
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
 from reconcile.typed_queries.app_interface_state_settings import (
     get_app_interface_state_settings,
 )
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
-from reconcile.utils import gql
-from reconcile.utils.aws_api import AWSApi
+from reconcile.typed_queries.get_state_aws_account import get_state_aws_account
+from reconcile.utils.aws_api import aws_config_file_path
 from reconcile.utils.secret_reader import (
     SecretReaderBase,
     create_secret_reader,
@@ -34,124 +43,185 @@ class StateInaccessibleException(Exception):
     pass
 
 
-STATE_ACCOUNT_QUERY = """
-{
-  accounts: awsaccounts_v1 (name: "{{ name }}")
-  {
-    name
-    resourcesDefaultRegion
-    automationToken {
-      path
-      field
-      version
-      format
-    }
-  }
-}
-"""
-
-
 def init_state(
     integration: str,
-    secret_reader: Optional[SecretReaderBase] = None,
+    secret_reader: SecretReaderBase | None = None,
 ) -> "State":
     if not secret_reader:
         vault_settings = get_app_interface_vault_settings()
         secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
-    # if state settings are present via env variables, use them
-    state_bucket_name = os.environ.get("APP_INTERFACE_STATE_BUCKET")
-    state_bucket_account_name = os.environ.get("APP_INTERFACE_STATE_BUCKET_ACCOUNT")
-    if state_bucket_name and state_bucket_account_name:
-        query = Template(STATE_ACCOUNT_QUERY).render(name=state_bucket_account_name)
-        aws_accounts = gql.get_api().query(query)["accounts"]
-        return _init_state_from_accounts(
-            integration=integration,
-            secret_reader=secret_reader,
-            bucket_name=state_bucket_name,
-            account_name=state_bucket_account_name,
-            accounts=aws_accounts,
-        )
+    s3_settings = acquire_state_settings(secret_reader)
+
+    return State(
+        integration=integration,
+        bucket=s3_settings.bucket,
+        client=s3_settings.build_client(),
+    )
 
-    # ... otherwise have a look if state settings are present in app-interface-settings-1.yml
-    state_settings = get_app_interface_state_settings()
-    if not state_settings:
-        raise StateInaccessibleException(
-            "app-interface state must be configured in order to use stateful integrations. "
-            "use one of the following options to provide state config: "
-            "* env vars APP_INTERFACE_STATE_BUCKET and APP_INTERFACE_STATE_BUCKET_ACCOUNT "
-            "* state settings in app-interface-settings-1.yml"
-        )
 
-    if isinstance(state_settings, AppInterfaceStateConfigurationS3V1):
-        return _init_state_from_settings(
-            integration=integration,
-            secret_reader=secret_reader,
-            state_settings=state_settings,
+class S3StateConfiguration(BaseModel):
+    bucket: str
+    region: str
+
+    @abstractmethod
+    def build_client(self) -> S3Client:
+        pass
+
+
+class S3CredsBasedStateConfiguration(S3StateConfiguration):
+    access_key_id: str
+    secret_access_key: str
+
+    def build_client(self) -> S3Client:
+        session = boto3.Session(
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+            region_name=self.region,
         )
+        return session.client("s3")
 
-    raise StateInaccessibleException(
-        f"app-interface-settings-1.yml state provider {state_settings.provider} is not supported."
-    )
 
+class S3ProfileBasedStateConfiguration(S3StateConfiguration):
+    profile: str
 
-def _init_state_from_settings(
-    integration: str,
-    secret_reader: SecretReaderBase,
-    state_settings: AppInterfaceStateConfigurationS3V1,
-) -> "State":
+    def build_client(self) -> S3Client:
+        session = boto3.Session(profile_name=self.profile, region_name=self.region)
+        return session.client("s3")
+
+
+def acquire_state_settings(
+    secret_reader: SecretReaderBase, query_func: Callable | None = None
+) -> S3StateConfiguration:
     """
-    Initializes a state object from the app-interface settings.
+    Finds the settings for the app-interface state provider in the following order:
 
-    :raises StateInaccessibleException: if the bucket is missing
-    or not accessible
+    * env variables pointing to a bucket and an AWS profile
+    * env variables pointing to static credentials
+    * env variables pointing to a bucket and a vault secret for creds
+    * env variables pointing to a bucket and ann AWS account from app-interface for creds
+    * state settings in app-interface-settings-1.yml
+
+    If no settings can be found, a StateInaccessibleException is raised.
     """
-    return _init_state_from_accounts(
-        integration=integration,
-        secret_reader=secret_reader,
-        bucket_name=state_settings.bucket,
-        account_name="settings-account",
-        accounts=[
-            {
-                "name": "settings-account",
-                "resourcesDefaultRegion": state_settings.region,
-                "automationToken": state_settings.credentials.dict(by_alias=True),
-            }
-        ],
+    state_bucket_name = os.environ.get("APP_INTERFACE_STATE_BUCKET")
+    state_bucket_region = os.environ.get("APP_INTERFACE_STATE_BUCKET_REGION")
+    state_bucket_account_name = os.environ.get("APP_INTERFACE_STATE_BUCKET_ACCOUNT")
+    state_bucket_access_key_id = os.environ.get(
+        "APP_INTERFACE_STATE_BUCKET_ACCESS_KEY_ID"
+    )
+    state_bucket_secret_access_key = os.environ.get(
+        "APP_INTERFACE_STATE_BUCKET_SECRET_ACCESS_KEY"
+    )
+    state_bucket_vault_secret = os.environ.get("APP_INTERFACE_STATE_VAULT_SECRET")
+    state_bucket_vault_secret_version = os.environ.get(
+        "APP_INTERFACE_STATE_VAULT_SECRET_VERSION"
     )
 
+    state_bucket_aws_profile = os.environ.get("APP_INTERFACE_STATE_AWS_PROFILE")
+
+    # if an AWS config file can be found and a profile for state usage is set ...
+    if (
+        state_bucket_name
+        and state_bucket_region
+        and aws_config_file_path()
+        and state_bucket_aws_profile
+    ):
+        logging.debug(f"access state via AWS profile {state_bucket_aws_profile}")
+        return S3ProfileBasedStateConfiguration(
+            bucket=state_bucket_name,
+            region=state_bucket_region,
+            profile=state_bucket_aws_profile,
+        )
 
-def _init_state_from_accounts(
-    integration: str,
-    secret_reader: SecretReaderBase,
-    bucket_name: str,
-    account_name: str,
-    accounts: Iterable[Mapping[str, Any]],
-) -> "State":
-    aws_account = next(
-        (a for a in accounts if a["name"] == account_name),
-        None,
-    )
-    if not aws_account:
+    # if the env vars point towards a vault secret that contains the credentials ...
+    if state_bucket_name and state_bucket_region and state_bucket_vault_secret:
+        logging.debug(
+            f"access state via access credentials from vault secret {state_bucket_vault_secret} version {state_bucket_vault_secret_version or 'latest'})"
+        )
+        secret = secret_reader.read_all_secret(
+            VaultSecret(
+                path=state_bucket_vault_secret,
+                field="all",
+                format=None,
+                version=int(state_bucket_vault_secret_version)
+                if state_bucket_vault_secret_version
+                else None,
+            )
+        )
+        return S3CredsBasedStateConfiguration(
+            bucket=state_bucket_name,
+            region=state_bucket_region,
+            access_key_id=secret["aws_access_key_id"],
+            secret_access_key=secret["aws_secret_access_key"],
+        )
+
+    # if the env vars contain actual AWS credentials, lets use them ...
+    if (
+        state_bucket_name
+        and state_bucket_region
+        and state_bucket_access_key_id
+        and state_bucket_secret_access_key
+    ):
+        logging.debug("access state via static credentials from env variables :(")
+        return S3CredsBasedStateConfiguration(
+            bucket=state_bucket_name,
+            region=state_bucket_region,
+            access_key_id=state_bucket_access_key_id,
+            secret_access_key=state_bucket_secret_access_key,
+        )
+
+    # if the env vars point towards an AWS account mentioned in app-interface ...
+    if state_bucket_name and state_bucket_account_name:
+        logging.debug(
+            f"access state via {state_bucket_account_name} automation token from app-interface"
+        )
+        account = get_state_aws_account(
+            state_bucket_account_name, query_func=query_func
+        )
+        if not account:
+            raise StateInaccessibleException(
+                f"The AWS account {state_bucket_account_name} that holds the state bucket can't be found in app-interface."
+            )
+        secret = secret_reader.read_all_secret(account.automation_token)
+        return S3CredsBasedStateConfiguration(
+            bucket=state_bucket_name,
+            region=state_bucket_region or account.resources_default_region,
+            access_key_id=secret["aws_access_key_id"],
+            secret_access_key=secret["aws_secret_access_key"],
+        )
+
+    # ... otherwise have a look if state settings are present in app-interface-settings-1.yml
+    ai_settings = get_app_interface_state_settings()
+    if ai_settings:
+        logging.debug("access state via app-interface settings")
+        if isinstance(ai_settings, AppInterfaceStateConfigurationS3V1):
+            secret = secret_reader.read_all_secret(ai_settings.credentials)
+            return S3CredsBasedStateConfiguration(
+                bucket=ai_settings.bucket,
+                region=ai_settings.region,
+                access_key_id=secret["aws_access_key_id"],
+                secret_access_key=secret["aws_secret_access_key"],
+            )
         raise StateInaccessibleException(
-            f"Can initialize state. app-interface does not define an AWS account named {account_name}"
+            f"The app-interface state provider {ai_settings.provider} is not supported."
         )
 
-    aws_api = AWSApi(
-        1,
-        [aws_account],
-        settings=None,
-        secret_reader=secret_reader,
-        init_users=False,
-    )
-    session = aws_api.get_session(account_name)
-    return State(
-        integration=integration,
-        bucket=bucket_name,
-        client=aws_api.get_session_client(session, "s3"),
+    raise StateInaccessibleException(
+        "app-interface state must be configured to use stateful integrations. "
+        "use one of the following options to provide state config: "
+        "* env vars APP_INTERFACE_STATE_BUCKET, APP_INTERFACE_STATE_BUCKET_REGION, APP_INTERFACE_STATE_AWS_PROFILE and AWS_CONFIG (hosting the requested profile) \n"
+        "* env vars APP_INTERFACE_STATE_BUCKET, APP_INTERFACE_STATE_BUCKET_REGION, APP_INTERFACE_STATE_VAULT_SECRET (and optionally APP_INTERFACE_STATE_VAULT_SECRET_VERSION) \n"
+        "* env vars APP_INTERFACE_STATE_BUCKET, APP_INTERFACE_STATE_BUCKET_REGION, APP_INTERFACE_STATE_BUCKET_ACCESS_KEY_ID, APP_INTERFACE_STATE_BUCKET_SECRET_ACCESS_KEY \n"
+        "* env vars APP_INTERFACE_STATE_BUCKET, APP_INTERFACE_STATE_BUCKET_REGION and APP_INTERFACE_STATE_BUCKET_ACCOUNT if the mentioned AWS account is present in app-interface \n"
+        "* state settings in app-interface-settings-1.yml"
     )
 
 
+class AbortStateTransaction(Exception):
+    """Raise to abort a state transaction."""
+
+
 class State:
     """
     A state object to be used by stateful integrations.
@@ -180,22 +250,22 @@ class State:
             self.client.head_bucket(Bucket=self.bucket)
         except ClientError as details:
             raise StateInaccessibleException(
-                f"Bucket {self.bucket} is not accessible - {str(details)}"
-            )
+                f"Bucket {self.bucket} is not accessible - {details!s}"
+            ) from None
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, *exc):
+    def __exit__(self, *exc: Any) -> None:
         self.cleanup()
 
-    def cleanup(self):
+    def cleanup(self) -> None:
         """
         Closes the S3 client
         """
         self.client.close()
 
-    def exists(self, key):
+    def exists(self, key: str) -> bool:
         """
         Checks if a key exists in the state.
 
@@ -203,24 +273,38 @@ class State:
 
         :type key: string
 
+        :raises StateInaccessibleException: if the bucket is missing or
+        permissions are insufficient or a general AWS error occurred
+        """
+        exists, _ = self.head(key)
+        return exists
+
+    def head(self, key: str) -> tuple[bool, dict[str, str]]:
+        """
+        Checks if a key exists in the state. Returns the metadata of a key in the state.
+
+        :param key: key to check
+
+        :return: tuple of (exists, metadata)
+
         :raises StateInaccessibleException: if the bucket is missing or
         permissions are insufficient or a general AWS error occurred
         """
         key_path = f"{self.state_path}/{key}"
         try:
-            self.client.head_object(Bucket=self.bucket, Key=key_path)
-            return True
+            response = self.client.head_object(Bucket=self.bucket, Key=key_path)
+            return True, response["Metadata"]
         except ClientError as details:
             error_code = details.response.get("Error", {}).get("Code", None)
             if error_code == "404":
-                return False
+                return False, {}
 
             raise StateInaccessibleException(
                 f"Can not access state key {key_path} "
-                f"in bucket {self.bucket} - {str(details)}"
-            )
+                f"in bucket {self.bucket} - {details!s}"
+            ) from None
 
-    def ls(self):
+    def ls(self) -> list[str]:
         """
         Returns a list of keys in the state
         """
@@ -244,20 +328,38 @@ class State:
 
         return [c["Key"].replace(self.state_path, "") for c in contents]
 
-    def add(self, key, value=None, force=False):
+    def add(
+        self,
+        key: str,
+        value: Any = None,
+        metadata: Mapping[str, str] | None = None,
+        force: bool = False,
+    ) -> None:
         """
         Adds a key/value to the state and fails if the key already exists
 
         :param key: key to add
         :param value: (optional) value of the state, defaults to None
+        :param metadata: (optional) metadata of the state, Mapping[str, str], defaults to None
+        :param force: (optional) if True, overrides the key if it exists,
 
         :type key: string
         """
-        if self.exists(key) and not force:
+        if not force and self.exists(key):
             raise KeyError(f"[state] key {key} already " f"exists in {self.state_path}")
-        self[key] = value
+        self._set(key, value, metadata=metadata)
+
+    def _set(
+        self, key: str, value: Any, metadata: Mapping[str, str] | None = None
+    ) -> None:
+        self.client.put_object(
+            Bucket=self.bucket,
+            Key=f"{self.state_path}/{key}",
+            Body=json.dumps(value),
+            Metadata=metadata or {},
+        )
 
-    def rm(self, key):
+    def rm(self, key: str) -> None:
         """
         Removes a key from the state and fails if the key does not exists
 
@@ -269,7 +371,7 @@ class State:
             raise KeyError(f"[state] key {key} does not exists in {self.state_path}")
         self.client.delete_object(Bucket=self.bucket, Key=f"{self.state_path}/{key}")
 
-    def get(self, key, *args):
+    def get(self, key: str, *args: Any) -> Any:
         """
         Gets a key value from the state and return the default
         value or raises and exception if the key does not exist.
@@ -287,7 +389,7 @@ class State:
                 return args[0]
             raise
 
-    def get_all(self, path):
+    def get_all(self, path: str) -> dict[str, Any]:
         """
         Gets all keys and values from the state in the specified path.
         """
@@ -297,7 +399,7 @@ class State:
             if k.startswith(f"/{path}")
         }
 
-    def __getitem__(self, item):
+    def __getitem__(self, item: str) -> Any:
         try:
             response = self.client.get_object(
                 Bucket=self.bucket, Key=f"{self.state_path}/{item}"
@@ -305,12 +407,59 @@ class State:
             return json.loads(response["Body"].read())
         except ClientError as details:
             if details.response["Error"]["Code"] == "NoSuchKey":
-                raise KeyError(item)
+                raise KeyError(item) from None
             raise
         except json.decoder.JSONDecodeError:
-            raise KeyError(item)
+            raise KeyError(item) from None
 
-    def __setitem__(self, key, value):
-        self.client.put_object(
-            Bucket=self.bucket, Key=f"{self.state_path}/{key}", Body=json.dumps(value)
-        )
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._set(key, value)
+
+    @contextlib.contextmanager
+    def transaction(
+        self, key: str, value: Any = None
+    ) -> Generator["TransactionStateObj", None, None]:
+        """Get a context manager to set the key in the state if no exception occurs.
+
+        You can set the value either via the value parameter or by setting the value attribute of the returned object.
+        If both are provided, the value attribute of the state object will take precedence.
+
+        Attention!
+
+        This is not a locking mechanism. It is a way to ensure that a key is set in the state if no exception occurs.
+        This method is not thread-safe nor multi-process-safe! There is no locking mechanism in place.
+        """
+        try:
+            _current_value = self[key]
+        except KeyError:
+            _current_value = None
+        state_obj = TransactionStateObj(key, value=_current_value)
+        try:
+            yield state_obj
+        except AbortStateTransaction:
+            return
+        else:
+            if state_obj.changed and state_obj.value != _current_value:
+                self[state_obj.key] = state_obj.value
+            elif value is not None and state_obj.value != value:
+                self[state_obj.key] = value
+
+
+@dataclass
+class TransactionStateObj:
+    """Represents a transistion state object with a key and a value."""
+
+    key: str
+    value: Any = None
+    _init_value: Any = field(init=False)
+
+    def __post_init__(self) -> None:
+        self._init_value = self.value
+
+    @property
+    def changed(self) -> bool:
+        return self.value != self._init_value
+
+    @property
+    def exists(self) -> bool:
+        return self._init_value is not None

reconcile/utils/terraform/config_client.py

@@ -4,7 +4,6 @@ from abc import (
     abstractmethod,
 )
 from collections.abc import Iterable
-from typing import Optional
 
 from reconcile.utils.exceptions import PrintToFileInGitRepositoryError
 from reconcile.utils.external_resource_spec import (
@@ -34,7 +33,7 @@ class TerraformConfigClient(ABC):
     @abstractmethod
     def dump(
         self,
-        existing_dir: Optional[str] = None,
+        existing_dir: str | None = None,
     ) -> str:
         """Dump the Terraform JSON configuration to the filesystem."""
 
@@ -65,7 +64,7 @@ class TerraformConfigClientCollection:
     def add_specs(
         self,
         specs: Iterable[ExternalResourceSpec],
-        account_filter: Optional[str] = None,
+        account_filter: str | None = None,
     ) -> None:
         """
         Add external resource specs
@@ -85,7 +84,7 @@ class TerraformConfigClientCollection:
             except KeyError:
                 raise ClientNotRegisteredError(
                     f"There aren't any clients registered with the account name: {spec.provisioner_name}"
-                )
+                ) from None
             self.resource_spec_inventory[spec.id_object()] = spec
 
     def populate_resources(self) -> None:
@@ -94,8 +93,8 @@ class TerraformConfigClientCollection:
 
     def dump(
         self,
-        print_to_file: Optional[str] = None,
-        existing_dirs: Optional[dict[str, str]] = None,
+        print_to_file: str | None = None,
+        existing_dirs: dict[str, str] | None = None,
     ) -> dict[str, str]:
         """
         Dump the Terraform JSON config to the filesystem.
@@ -122,7 +121,7 @@ class TerraformConfigClientCollection:
             working_dirs[account_name] = client.dump()
 
             if print_to_file:
-                with open(print_to_file, "a") as f:
+                with open(print_to_file, "a", encoding="locale") as f:
                     f.write(f"##### {account_name} #####\n")
                     f.write(client.dumps())
                     f.write("\n")