qontract-reconcile 0.10.2.dev427__py3-none-any.whl → 0.10.2.dev456__py3-none-any.whl

reconcile/typed_queries/saas_files.py

@@ -42,6 +42,7 @@ from reconcile.gql_definitions.fragments.saas_target_namespace import (
     SaasTargetNamespace,
 )
 from reconcile.utils import gql
+from reconcile.utils.environ import used_for_security_is_enabled
 from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
     ParameterError,
@@ -78,10 +79,14 @@ class SaasResourceTemplateTarget(
         self, parent_saas_file_name: str, parent_resource_template_name: str
     ) -> str:
         """Returns a unique identifier for a target."""
-        return hashlib.blake2s(
-            f"{parent_saas_file_name}:{parent_resource_template_name}:{self.name or 'default'}:{self.namespace.cluster.name}:{self.namespace.name}".encode(),
-            digest_size=20,
-        ).hexdigest()
+        data = f"{parent_saas_file_name}:{parent_resource_template_name}:{self.name or 'default'}:{self.namespace.cluster.name}:{self.namespace.name}".encode()
+        if used_for_security_is_enabled():
+            # When USED_FOR_SECURITY is enabled, use blake2s without digest_size and truncate to 20 bytes
+            # This is needed for FIPS compliance where digest_size parameter is not supported
+            return hashlib.blake2s(data).digest()[:20].hex()
+        else:
+            # Default behavior: use blake2s with digest_size=20
+            return hashlib.blake2s(data, digest_size=20).hexdigest()
 
 
 class SaasResourceTemplate(ConfiguredBaseModel, validate_by_alias=True):

reconcile/utils/environ.py

@@ -4,6 +4,11 @@ from functools import wraps
 from typing import Any
 
 
+def used_for_security_is_enabled() -> bool:
+    used_for_security_env = os.getenv("USED_FOR_SECURITY", "false")
+    return used_for_security_env.lower() == "true"
+
+
 def environ(variables: Iterable[str] | None = None) -> Callable:
     """Check that environment variables are set before execution."""
     if variables is None:

reconcile/utils/gitlab_api.py

@@ -444,6 +444,8 @@ class GitLabApi:
     def get_merge_request_comments(
         merge_request: ProjectMergeRequest,
         include_description: bool = False,
+        include_approvals: bool = False,
+        approval_body: str = "",
     ) -> list[Comment]:
         comments = []
         if include_description:
@@ -455,6 +457,16 @@ class GitLabApi:
                     created_at=merge_request.created_at,
                 )
             )
+        if include_approvals:
+            comments.extend(
+                Comment(
+                    id=approval["user"]["id"],
+                    username=approval["user"]["username"],
+                    body=approval_body,
+                    created_at=approval["approved_at"],
+                )
+                for approval in merge_request.approvals.get().approved_by
+            )
         comments.extend(
             Comment(
                 id=note.id,

reconcile/utils/jjb_client.py

@@ -33,6 +33,10 @@ from reconcile.utils.vcs import GITHUB_BASE_URL
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
 
+class MissingJobUrlError(Exception):
+    pass
+
+
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
@@ -335,7 +339,7 @@ class JJB:
                 job_name = job["name"]
                 try:
                     repos.add(self.get_repo_url(job))
-                except KeyError:
+                except MissingJobUrlError:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
@@ -355,7 +359,19 @@ class JJB:
 
     @staticmethod
     def get_repo_url(job: Mapping[str, Any]) -> str:
-        repo_url_raw = job["properties"][0]["github"]["url"]
+        repo_url_raw = job.get("properties", [{}])[0].get("github", {}).get("url")
+
+        # we may be in a Github Branch Source type of job
+        if not repo_url_raw:
+            gh_org = job.get("scm", [{}])[0].get("github", {}).get("repo-owner")
+            gh_repo = job.get("scm", [{}])[0].get("github", {}).get("repo")
+            if gh_org and gh_repo:
+                repo_url_raw = f"https://github.com/{gh_org}/{gh_repo}/"
+            else:
+                raise MissingJobUrlError(
+                    f"Cannot find job url for {job['display-name']}"
+                )
+
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
@@ -404,7 +420,7 @@ class JJB:
                 try:
                     if self.get_repo_url(job).lower() == repo_url.rstrip("/").lower():
                         return job
-                except KeyError:
+                except MissingJobUrlError:
                     # something wrong here. ignore this job
                     pass
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")

reconcile/utils/oc.py

@@ -651,9 +651,15 @@ class OCCli:
             raise e
         return True
 
+    def _use_oc_project(self, namespace: str) -> bool:
+        # Note, that openshift-* namespaces cannot be created via new-project
+        return self.is_kind_supported(PROJECT_KIND) and not namespace.startswith(
+            "openshift-"
+        )
+
     @OCDecorators.process_reconcile_time
     def new_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported(PROJECT_KIND):
+        if self._use_oc_project(namespace=namespace):
             cmd = ["new-project", namespace]
         else:
             cmd = ["create", "namespace", namespace]
@@ -669,7 +675,7 @@ class OCCli:
 
     @OCDecorators.process_reconcile_time
     def delete_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported(PROJECT_KIND):
+        if self._use_oc_project(namespace=namespace):
             cmd = ["delete", "project", namespace]
         else:
             cmd = ["delete", "namespace", namespace]

reconcile/utils/rhcsv2_certs.py

@@ -1,21 +1,35 @@
+import base64
 import re
 from datetime import UTC
+from enum import StrEnum
 
 import requests
 from cryptography import x509
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives.serialization import pkcs12
 from cryptography.x509.oid import NameOID
 from pydantic import BaseModel, Field
 
 
-class RhcsV2Cert(BaseModel, validate_by_name=True, validate_by_alias=True):
+class CertificateFormat(StrEnum):
+    PEM = "PEM"
+    PKCS12 = "PKCS12"
+
+
+class RhcsV2CertPem(BaseModel, validate_by_name=True, validate_by_alias=True):
     certificate: str = Field(alias="tls.crt")
     private_key: str = Field(alias="tls.key")
     ca_cert: str = Field(alias="ca.crt")
     expiration_timestamp: int
 
 
+class RhcsV2CertPkcs12(BaseModel, validate_by_name=True, validate_by_alias=True):
+    pkcs12_keystore: str = Field(alias="keystore.pkcs12.b64")
+    pkcs12_truststore: str = Field(alias="truststore.pkcs12.b64")
+    expiration_timestamp: int
+
+
 def extract_cert(text: str) -> re.Match:
     # The CA webform returns an HTML page with inline JS that builds an array of “outputList”
     # objects. Each object looks roughly like:
@@ -67,7 +81,66 @@ def get_cert_expiry_timestamp(js_escaped_pem: str) -> int:
     return int(dt_expiry.timestamp())
 
 
-def generate_cert(issuer_url: str, uid: str, pwd: str, ca_url: str) -> RhcsV2Cert:
+def _format_pem(
+    private_key: rsa.RSAPrivateKey,
+    cert_pem: str,
+    ca_pem: str,
+    cert_expiry_timestamp: int,
+) -> RhcsV2CertPem:
+    """Generate RhcsV2Cert with PEM components."""
+    private_key_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    ).decode()
+    return RhcsV2CertPem(
+        private_key=private_key_pem,
+        certificate=cert_pem.encode().decode("unicode_escape").replace("\\/", "/"),
+        ca_cert=ca_pem,
+        expiration_timestamp=cert_expiry_timestamp,
+    )
+
+
+def _format_pkcs12(
+    private_key: rsa.RSAPrivateKey,
+    cert_pem: str,
+    ca_pem: str,
+    uid: str,
+    pwd: str,
+    cert_expiry_timestamp: int,
+) -> RhcsV2CertPkcs12:
+    """Generate PKCS#12 keystore and truststore components, returns base64-encoded strings."""
+    clean_cert_pem = cert_pem.encode().decode("unicode_escape").replace("\\/", "/")
+    cert_obj = x509.load_pem_x509_certificate(clean_cert_pem.encode())
+    ca_obj = x509.load_pem_x509_certificate(ca_pem.encode())
+    keystore_p12 = pkcs12.serialize_key_and_certificates(
+        name=uid.encode("utf-8"),
+        key=private_key,
+        cert=cert_obj,
+        cas=[ca_obj],
+        encryption_algorithm=serialization.BestAvailableEncryption(pwd.encode("utf-8")),
+    )
+    truststore_p12 = pkcs12.serialize_key_and_certificates(
+        name=b"ca-trust",
+        key=None,
+        cert=None,
+        cas=[ca_obj],
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    return RhcsV2CertPkcs12(
+        pkcs12_keystore=base64.b64encode(keystore_p12).decode("utf-8"),
+        pkcs12_truststore=base64.b64encode(truststore_p12).decode("utf-8"),
+        expiration_timestamp=cert_expiry_timestamp,
+    )
+
+
+def generate_cert(
+    issuer_url: str,
+    uid: str,
+    pwd: str,
+    ca_url: str,
+    cert_format: CertificateFormat = CertificateFormat.PEM,
+) -> RhcsV2CertPem | RhcsV2CertPkcs12:
     private_key = rsa.generate_private_key(65537, 4096)
     csr = (
         x509.CertificateSigningRequestBuilder()
@@ -78,6 +151,7 @@ def generate_cert(issuer_url: str, uid: str, pwd: str, ca_url: str) -> RhcsV2Cer
         )
         .sign(private_key, hashes.SHA256())
     )
+
     data = {
         "uid": uid,
         "pwd": pwd,
@@ -87,27 +161,19 @@ def generate_cert(issuer_url: str, uid: str, pwd: str, ca_url: str) -> RhcsV2Cer
         "renewal": "false",
         "xmlOutput": "false",
     }
-    response = requests.post(issuer_url, data=data)
+    response = requests.post(issuer_url, data=data, timeout=30)
     response.raise_for_status()
+    cert_pem = extract_cert(response.text).group(1)
+    cert_expiry_timestamp = get_cert_expiry_timestamp(cert_pem)
 
-    cert_pem = extract_cert(response.text)
-    cert_expiry_timestamp = get_cert_expiry_timestamp(cert_pem.group(1))
-    private_key_pem = private_key.private_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PrivateFormat.TraditionalOpenSSL,
-        encryption_algorithm=serialization.NoEncryption(),
-    ).decode()
-
-    response = requests.get(ca_url)
+    response = requests.get(ca_url, timeout=30)
     response.raise_for_status()
     ca_pem = response.text
 
-    return RhcsV2Cert(
-        private_key=private_key_pem,
-        certificate=cert_pem.group(1)
-        .encode()
-        .decode("unicode_escape")
-        .replace("\\/", "/"),
-        ca_cert=ca_pem,
-        expiration_timestamp=cert_expiry_timestamp,
-    )
+    match cert_format:
+        case CertificateFormat.PKCS12:
+            return _format_pkcs12(
+                private_key, cert_pem, ca_pem, uid, pwd, cert_expiry_timestamp
+            )
+        case CertificateFormat.PEM:
+            return _format_pem(private_key, cert_pem, ca_pem, cert_expiry_timestamp)

reconcile/utils/terrascript_aws_client.py

@@ -272,6 +272,7 @@ VARIABLE_KEYS = [
     "lifecycle",
     "max_session_duration",
     "secret_format",
+    "policy",
 ]
 
 EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$")
@@ -373,6 +374,10 @@ class aws_s3_bucket_logging(Resource):
     pass
 
 
+class aws_kinesis_resource_policy(Resource):
+    pass
+
+
 class aws_cloudfront_log_delivery_canonical_user_id(Data):
     pass
 
@@ -2249,14 +2254,22 @@ class TerrascriptClient:
 
         return lifecycle_rules
 
-    def populate_tf_resource_s3(self, spec: ExternalResourceSpec) -> aws_s3_bucket:
+    def _populate_tf_resource_s3_bucket(
+        self,
+        spec: ExternalResourceSpec,
+        common_values: dict[str, Any],
+    ) -> tuple[aws_s3_bucket, list[TFResource]]:
+        """Create S3 bucket with configuration and notifications.
+
+        Creates aws_s3_bucket with versioning, encryption, lifecycle rules,
+        CORS, logging, and replication. Also creates aws_s3_bucket_notification
+        for SQS/SNS event notifications if configured.
+        """
         account = spec.provisioner_name
         identifier = spec.identifier
-        common_values = self.init_values(spec)
         output_prefix = spec.output_prefix
 
         tf_resources: list[TFResource] = []
-        self.init_common_outputs(tf_resources, spec)
 
         # s3 bucket
         # Terraform resource reference:
@@ -2433,8 +2446,7 @@ class TerrascriptClient:
         output_name = output_prefix + "__endpoint"
         tf_resources.append(Output(output_name, value=endpoint))
 
-        sqs_identifier = common_values.get("sqs_identifier", None)
-        if sqs_identifier is not None:
+        if sqs_identifier := common_values.get("sqs_identifier"):
             sqs_values = {"name": sqs_identifier}
             sqs_provider = values.get("provider")
             if sqs_provider:
@@ -2453,11 +2465,9 @@ class TerrascriptClient:
                     }
                 ],
             }
-            filter_prefix = common_values.get("filter_prefix", None)
-            if filter_prefix is not None:
+            if filter_prefix := common_values.get("filter_prefix"):
                 notification_values["queue"][0]["filter_prefix"] = filter_prefix
-            filter_suffix = common_values.get("filter_suffix", None)
-            if filter_suffix is not None:
+            if filter_suffix := common_values.get("filter_suffix"):
                 notification_values["queue"][0]["filter_suffix"] = filter_suffix
 
             notification_tf_resource = aws_s3_bucket_notification(
@@ -2537,21 +2547,48 @@ class TerrascriptClient:
             )
             tf_resources.append(notification_tf_resource)
 
-        bucket_policy = common_values.get("bucket_policy")
-        if bucket_policy:
-            values = {
-                "bucket": identifier,
-                "policy": bucket_policy,
-                "depends_on": self.get_dependencies([bucket_tf_resource]),
-            }
-            if self._multiregion_account(account):
-                values["provider"] = "aws." + region
-            bucket_policy_tf_resource = aws_s3_bucket_policy(identifier, **values)
-            tf_resources.append(bucket_policy_tf_resource)
+        return bucket_tf_resource, tf_resources
 
-        # iam resources
-        # Terraform resource reference:
-        # https://www.terraform.io/docs/providers/aws/r/iam_access_key.html
+    def _populate_tf_resource_s3_bucket_policy(
+        self,
+        spec: ExternalResourceSpec,
+        bucket_tf_resource: aws_s3_bucket,
+        policy: str,
+        common_values: dict[str, Any],
+    ) -> list[TFResource]:
+        """Create S3 bucket policy resource.
+
+        Creates aws_s3_bucket_policy with the provided policy document.
+        """
+        account = spec.provisioner_name
+        identifier = spec.identifier
+        region = common_values.get("region") or self.default_regions.get(account)
+        assert region  # make mypy happy
+
+        values: dict[str, Any] = {
+            "bucket": identifier,
+            "policy": policy,
+            "depends_on": self.get_dependencies([bucket_tf_resource]),
+        }
+        if self._multiregion_account(account):
+            values["provider"] = "aws." + region
+        bucket_policy_tf_resource = aws_s3_bucket_policy(identifier, **values)
+        return [bucket_policy_tf_resource]
+
+    def _populate_tf_resource_s3_iam(
+        self,
+        spec: ExternalResourceSpec,
+        bucket_tf_resource: aws_s3_bucket,
+        common_values: dict[str, Any],
+    ) -> list[TFResource]:
+        """Create IAM resources for S3 bucket access.
+
+        Creates aws_iam_user, aws_iam_access_key, aws_iam_policy,
+        and aws_iam_user_policy_attachment for bucket access.
+        """
+        identifier = spec.identifier
+        output_prefix = spec.output_prefix
+        tf_resources: list[TFResource] = []
 
         # iam user for bucket
         values = {
@@ -2609,6 +2646,32 @@ class TerrascriptClient:
         )
         tf_resources.append(tf_user_policy_attachment)
 
+        return tf_resources
+
+    def populate_tf_resource_s3(self, spec: ExternalResourceSpec) -> aws_s3_bucket:
+        account = spec.provisioner_name
+        common_values = self.init_values(spec)
+
+        tf_resources: list[TFResource] = []
+        self.init_common_outputs(tf_resources, spec)
+
+        bucket_tf_resource, bucket_resources = self._populate_tf_resource_s3_bucket(
+            spec, common_values
+        )
+        tf_resources.extend(bucket_resources)
+
+        bucket_policy = common_values.get("bucket_policy")
+        if bucket_policy:
+            tf_resources.extend(
+                self._populate_tf_resource_s3_bucket_policy(
+                    spec, bucket_tf_resource, bucket_policy, common_values
+                )
+            )
+
+        tf_resources.extend(
+            self._populate_tf_resource_s3_iam(spec, bucket_tf_resource, common_values)
+        )
+
         self.add_resources(account, tf_resources)
 
         return bucket_tf_resource
@@ -3383,42 +3446,53 @@ class TerrascriptClient:
         common_values = self.init_values(spec)
         output_prefix = spec.output_prefix
 
-        bucket_tf_resource = self.populate_tf_resource_s3(spec)
-
         tf_resources: list[TFResource] = []
+        self.init_common_outputs(tf_resources, spec)
+
+        bucket_tf_resource, bucket_resources = self._populate_tf_resource_s3_bucket(
+            spec, common_values
+        )
+        tf_resources.extend(bucket_resources)
+
+        tf_resources.extend(
+            self._populate_tf_resource_s3_iam(spec, bucket_tf_resource, common_values)
+        )
 
         # cloudfront origin access identity
         values = {"comment": f"{identifier}-cf-identity"}
         cf_oai_tf_resource = aws_cloudfront_origin_access_identity(identifier, **values)
         tf_resources.append(cf_oai_tf_resource)
 
-        # bucket policy for cloudfront
-        values_policy: dict[str, Any] = {"bucket": identifier}
-        policy = {
-            "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Sid": "Grant access to CloudFront Origin Identity",
-                    "Effect": "Allow",
-                    "Principal": {"AWS": "${" + cf_oai_tf_resource.iam_arn + "}"},
-                    "Action": "s3:GetObject",
-                    "Resource": [
-                        f"arn:aws:s3:::{identifier}/{enable_dir}/*"
-                        for enable_dir in common_values.get(
-                            "get_object_enable_dirs", []
-                        )
-                    ],
-                }
+        # bucket policy for cloudfront - merge custom policy with CloudFront access statement
+        cf_statement = {
+            "Sid": "Grant access to CloudFront Origin Identity",
+            "Effect": "Allow",
+            "Principal": {"AWS": "${" + cf_oai_tf_resource.iam_arn + "}"},
+            "Action": "s3:GetObject",
+            "Resource": [
+                f"arn:aws:s3:::{identifier}/{enable_dir}/*"
+                for enable_dir in common_values.get("get_object_enable_dirs", [])
             ],
         }
-        values_policy["policy"] = json_dumps(policy)
-        values_policy["depends_on"] = self.get_dependencies([bucket_tf_resource])
-        region = common_values.get("region") or self.default_regions.get(account)
-        assert region  # make mypy happy
-        if self._multiregion_account(account):
-            values_policy["provider"] = "aws." + region
-        bucket_policy_tf_resource = aws_s3_bucket_policy(identifier, **values_policy)
-        tf_resources.append(bucket_policy_tf_resource)
+
+        custom_bucket_policy = common_values.get("bucket_policy")
+        if custom_bucket_policy:
+            # if the user specifies a custom bucket policy then we merge their statements with the cloudfront origin identity policy
+            if isinstance(custom_bucket_policy, str):
+                custom_bucket_policy = json.loads(custom_bucket_policy)
+            custom_bucket_policy.setdefault("Statement", []).append(cf_statement)
+            policy = custom_bucket_policy
+        else:
+            policy = {
+                "Version": "2012-10-17",
+                "Statement": [cf_statement],
+            }
+
+        tf_resources.extend(
+            self._populate_tf_resource_s3_bucket_policy(
+                spec, bucket_tf_resource, json_dumps(policy), common_values
+            )
+        )
 
         distribution_config = common_values.get("distribution_config", {})
         # aws_s3_bucket_acl
@@ -4019,6 +4093,22 @@ class TerrascriptClient:
         kinesis_tf_resource = aws_kinesis_stream(identifier, **kinesis_values)
         tf_resources.append(kinesis_tf_resource)
 
+        # kinesis resource policy (optional)
+        # Terraform resource reference:
+        # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_resource_policy
+        if policy := common_values.get("policy"):
+            policy_identifier = f"{identifier}-policy"
+            policy_values: dict[str, Any] = {
+                "resource_arn": "${" + kinesis_tf_resource.arn + "}",
+                "policy": policy,
+            }
+            if provider:
+                policy_values["provider"] = provider
+            kinesis_policy_tf_resource = aws_kinesis_resource_policy(
+                policy_identifier, **policy_values
+            )
+            tf_resources.append(kinesis_policy_tf_resource)
+
         es_identifier = common_values.get("es_identifier", None)
         if es_identifier:
             es_resource = self._find_resource_spec(

reconcile/vpc_peerings_validator.py

@@ -159,6 +159,19 @@ def validate_no_public_to_public_peerings(
             if peer.internal or (peer.spec and peer.spec.private):
                 continue
 
+            # If both sides are allowed to override this check, then we can
+            # allow the peering.
+            if (
+                cluster.allowed_to_bypass_public_peering_restriction
+                and peer.allowed_to_bypass_public_peering_restriction
+            ):
+                logging.debug(
+                    f"{cluster.name} and {peer.name} are both allowed to skip \
+                    the check 'no peering with public clusters' check, so their \
+                    peering is allowed"
+                )
+                continue
+
             valid = False
             pair = {cluster.name, peer.name}
             if pair in found_pairs: