huaweicloud-encryption-sdk-python 1.2.1__py3-none-any.whl

huaweicloud_encryptionsdk/kms/kms_http_client.py

@@ -0,0 +1,116 @@
+import json
+import logging
+
+import requests
+
+from huaweicloud_encryptionsdk.exception.errors import HuaweicloudException, ErrorMessage
+from huaweicloud_encryptionsdk.model.kms_config import KMSConfig
+
+logger = logging.getLogger(__name__)
+
+SIGNATURE_ALGORITHM = "SDK-HMAC-SHA256"
+CONTENT_TYPE_JSON = "application/json"
+CONNECT_TIMEOUT_SECONDS = 10
+SOCKET_TIMEOUT_SECONDS = 10
+
+
+class KmsHttpClient:
+
+    def __init__(self, ak: str, sk: str, kms_config: KMSConfig, security_token: str = ""):
+        self._ak = ak
+        self._sk = sk
+        self._kms_config = kms_config
+        self._security_token = security_token
+        self._signer = self._create_signer()
+
+    def _create_signer(self):
+        from huaweicloud_encryptionsdk._apig_sdk import signer
+        sig = signer.Signer()
+        sig.Key = self._ak
+        sig.Secret = self._sk
+        return sig
+
+    def post(self, uri, body, response_type=None):
+        return self._execute("POST", uri, body, response_type)
+
+    def get(self, uri, response_type=None):
+        return self._execute("GET", uri, None, response_type)
+
+    def put(self, uri, body, response_type=None):
+        return self._execute("PUT", uri, body, response_type)
+
+    def delete(self, uri, body, response_type=None):
+        return self._execute("DELETE", uri, body, response_type)
+
+    def _execute(self, method, uri, body, response_type):
+        from huaweicloud_encryptionsdk._apig_sdk.signer import HttpRequest
+        url = self._build_url(uri)
+        json_body = json.dumps(body, default=self._json_default) if body is not None else None
+        try:
+            r = HttpRequest(method, url)
+            r.headers["Content-Type"] = CONTENT_TYPE_JSON
+            if self._security_token:
+                r.headers["X-Security-Token"] = self._security_token
+            if json_body is not None:
+                r.body = json_body
+
+            self._signer.Sign(r)
+
+            logger.info("[EncryptionSDK] KMS Request: %s %s", method, url)
+            if json_body is not None:
+                logger.debug("[EncryptionSDK] KMS Request Body: %s", json_body)
+
+            resp = requests.request(
+                method=r.method,
+                url=r.scheme + "://" + r.host + r.uri,
+                headers=r.headers,
+                data=r.body,
+                timeout=(CONNECT_TIMEOUT_SECONDS, SOCKET_TIMEOUT_SECONDS)
+            )
+            request_id = resp.headers.get("X-Request-Id", "")
+            logger.info("[EncryptionSDK] X-Request-Id is %s", request_id)
+            logger.info("[EncryptionSDK] KMS Response Status: %s", resp.status_code)
+            logger.debug("[EncryptionSDK] KMS Response Body: %s", resp.text[:1024] if resp.text else "")
+            if resp.status_code != 200:
+                raise HuaweicloudException(
+                    "KMS API call failed, status=" + str(resp.status_code)
+                    + ", body=" + resp.text
+                )
+            response_data = resp.json()
+            if response_type is not None:
+                return self._deserialize_response(response_data, response_type)
+            return response_data
+        except HuaweicloudException:
+            raise
+        except Exception as e:
+            raise HuaweicloudException(ErrorMessage.KMS_CREATE_KEY_EXCEPTION.message, e)
+
+    def _build_url(self, uri):
+        endpoint = self._kms_config.end_point
+        if not endpoint:
+            raise HuaweicloudException(ErrorMessage.END_POINT_NOT_NULL.message)
+        if not endpoint.startswith("http://") and not endpoint.startswith("https://"):
+            endpoint = "https://" + endpoint
+        return endpoint + "/v1.0/" + self._kms_config.project_id + uri
+
+    @staticmethod
+    def _deserialize_response(data, response_type):
+        if hasattr(response_type, 'from_dict'):
+            return response_type.from_dict(data)
+        obj = response_type()
+        for key, value in data.items():
+            attr_name = key.replace("-", "_").replace(" ", "_")
+            if hasattr(obj, attr_name):
+                setattr(obj, attr_name, value)
+        return obj
+
+    @staticmethod
+    def _json_default(obj):
+        if hasattr(obj, '__dict__'):
+            result = {}
+            for key, value in obj.__dict__.items():
+                if value is not None:
+                    json_key = key.lstrip('_')
+                    result[json_key] = value
+            return result
+        return str(obj)

huaweicloud_encryptionsdk/log/log_config.py

@@ -0,0 +1,30 @@
+import os
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class LogConfig:
+    log_group_id: str = ""
+    log_stream_id: str = ""
+    log_group_name: str = "DSC"
+    log_stream_name: str = "crypto_operation_log"
+    region: Optional[str] = None
+    project_id: Optional[str] = None
+    lts_endpoint: Optional[str] = None
+    batch_count_threshold: int = 1000
+    batch_size_threshold_in_bytes: int = 524288
+    linger_ms: int = 30000
+    total_size_in_bytes: int = 104857600
+    max_block_ms: int = 0
+    io_thread_count: int = field(default_factory=lambda: os.cpu_count() or 1)
+    retries: int = 5
+    base_retry_backoff_ms: int = 100
+    max_retry_backoff_ms: int = 500
+    enable_local_test: bool = False
+    backup_dir: str = field(default_factory=lambda: os.path.join(os.path.expanduser("~"), "lts_log_backup"))
+    max_backup_file_size: int = 10 * 1024 * 1024
+    enable_recovery_on_startup: bool = True
+    recovery_batch_size: int = 1000
+    recovery_batch_interval: int = 1000
+    fail_on_backup_dir_missing: bool = False

huaweicloud_encryptionsdk/log/log_file_backup.py

@@ -0,0 +1,144 @@
+import json
+import os
+import time
+import threading
+from typing import List, Optional
+
+from huaweicloud_encryptionsdk.exception.errors import HuaweicloudException
+
+
+class LogFileBackup:
+
+    DEFAULT_BACKUP_DIR = "lts_log_backup"
+    BACKUP_FILE_PREFIX = "lts_backup_"
+    BACKUP_FILE_SUFFIX = ".json"
+    MAX_BACKUP_FILE_SIZE = 10 * 1024 * 1024
+    _save_lock = threading.Lock()
+
+    def __init__(self, backup_dir_path: Optional[str] = None, fail_on_missing: bool = False):
+        if backup_dir_path is None or backup_dir_path.strip() == "":
+            backup_dir_path = os.path.join(os.path.expanduser("~"), self.DEFAULT_BACKUP_DIR)
+        self._backup_dir_path = backup_dir_path.strip()
+        self._fail_on_missing = fail_on_missing
+        self._init_backup_dir()
+
+    def _init_backup_dir(self) -> None:
+        backup_dir = self._backup_dir_path
+        if os.path.exists(backup_dir):
+            if not os.path.isdir(backup_dir):
+                if self._fail_on_missing:
+                    raise HuaweicloudException(f"Backup path exists but is not a directory: {backup_dir}")
+                return
+            if not os.access(backup_dir, os.W_OK):
+                if self._fail_on_missing:
+                    raise HuaweicloudException(f"No write permission for backup directory: {backup_dir}")
+                return
+            test_file = os.path.join(backup_dir, f".write_test_{int(time.time() * 1000)}")
+            try:
+                with open(test_file, "w") as f:
+                    f.write("test")
+                os.remove(test_file)
+            except Exception as e:
+                if self._fail_on_missing:
+                    raise HuaweicloudException(f"Cannot write to backup directory: {backup_dir}", e)
+                return
+        else:
+            try:
+                os.makedirs(backup_dir, exist_ok=True)
+            except Exception:
+                if self._fail_on_missing:
+                    raise HuaweicloudException(f"Failed to create backup directory: {backup_dir}")
+                return
+
+    def save_failed_logs(self, logs: List[dict], timestamp: int) -> Optional[str]:
+        if logs is None or len(logs) == 0:
+            return None
+        try:
+            date_str = time.strftime("%Y%m%d", time.localtime(timestamp / 1000))
+            file_name = f"{self.BACKUP_FILE_PREFIX}{date_str}{self.BACKUP_FILE_SUFFIX}"
+            backup_file = os.path.join(self._backup_dir_path, file_name)
+
+            with self._save_lock:
+                existing_logs = []
+                if os.path.exists(backup_file):
+                    try:
+                        if os.path.getsize(backup_file) < self.MAX_BACKUP_FILE_SIZE:
+                            with open(backup_file, "r", encoding="utf-8") as f:
+                                existing_logs = json.loads(f.read())
+                        else:
+                            seq = 1
+                            while True:
+                                file_name = f"{self.BACKUP_FILE_PREFIX}{date_str}_{seq}{self.BACKUP_FILE_SUFFIX}"
+                                backup_file = os.path.join(self._backup_dir_path, file_name)
+                                if not os.path.exists(backup_file) or os.path.getsize(backup_file) < self.MAX_BACKUP_FILE_SIZE:
+                                    if os.path.exists(backup_file):
+                                        with open(backup_file, "r", encoding="utf-8") as f:
+                                            existing_logs = json.loads(f.read())
+                                    break
+                                seq += 1
+                    except Exception:
+                        existing_logs = []
+
+                existing_logs.extend(logs)
+                json_content = json.dumps(existing_logs, ensure_ascii=False, default=str)
+                with open(backup_file, "w", encoding="utf-8") as f:
+                    f.write(json_content)
+
+            return os.path.abspath(backup_file)
+        except Exception:
+            return None
+
+    def recover_failed_logs(self) -> List[dict]:
+        recovered_logs = []
+        try:
+            if not os.path.exists(self._backup_dir_path) or not os.path.isdir(self._backup_dir_path):
+                return recovered_logs
+            backup_files = []
+            for name in os.listdir(self._backup_dir_path):
+                if name.startswith(self.BACKUP_FILE_PREFIX) and name.endswith(self.BACKUP_FILE_SUFFIX):
+                    backup_files.append(name)
+            if not backup_files:
+                return recovered_logs
+            backup_files.sort(key=lambda f: self._extract_timestamp_from_file(f))
+            for file_name in backup_files:
+                backup_file = os.path.join(self._backup_dir_path, file_name)
+                try:
+                    file_size = os.path.getsize(backup_file)
+                    if file_size > self.MAX_BACKUP_FILE_SIZE:
+                        continue
+                    with open(backup_file, "r", encoding="utf-8") as f:
+                        content = f.read()
+                    logs = json.loads(content)
+                    if logs and len(logs) > 0:
+                        recovered_logs.extend(logs)
+                        try:
+                            os.remove(backup_file)
+                        except Exception:
+                            pass
+                except Exception:
+                    pass
+        except Exception:
+            pass
+        return recovered_logs
+
+    def _extract_timestamp_from_file(self, file_name: str) -> int:
+        try:
+            ts_str = file_name[len(self.BACKUP_FILE_PREFIX):-len(self.BACKUP_FILE_SUFFIX)]
+            ts_str = ts_str.split("_")[0]
+            return int(ts_str)
+        except Exception:
+            return 0
+
+    def clear_all_backup_files(self) -> None:
+        if not os.path.exists(self._backup_dir_path):
+            return
+        for name in os.listdir(self._backup_dir_path):
+            if name.startswith(self.BACKUP_FILE_PREFIX) and name.endswith(self.BACKUP_FILE_SUFFIX):
+                try:
+                    os.remove(os.path.join(self._backup_dir_path, name))
+                except Exception:
+                    pass
+
+    @property
+    def backup_dir_path(self) -> str:
+        return self._backup_dir_path

huaweicloud_encryptionsdk/log/log_metrics.py

@@ -0,0 +1,132 @@
+import threading
+from typing import Optional
+
+
+class LogMetrics:
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._last_report_time: int = 0
+        self._success_count: int = 0
+        self._failure_count: int = 0
+        self._success_log_count: int = 0
+        self._failure_log_count: int = 0
+        self._buffer_log_count: int = 0
+        self._last_error_message: Optional[str] = None
+        self._last_error_time: int = 0
+
+    @property
+    def last_report_time(self) -> int:
+        with self._lock:
+            return self._last_report_time
+
+    @last_report_time.setter
+    def last_report_time(self, value: int) -> None:
+        with self._lock:
+            self._last_report_time = value
+
+    @property
+    def success_count(self) -> int:
+        with self._lock:
+            return self._success_count
+
+    @success_count.setter
+    def success_count(self, value: int) -> None:
+        with self._lock:
+            self._success_count = value
+
+    @property
+    def failure_count(self) -> int:
+        with self._lock:
+            return self._failure_count
+
+    @failure_count.setter
+    def failure_count(self, value: int) -> None:
+        with self._lock:
+            self._failure_count = value
+
+    @property
+    def success_log_count(self) -> int:
+        with self._lock:
+            return self._success_log_count
+
+    @success_log_count.setter
+    def success_log_count(self, value: int) -> None:
+        with self._lock:
+            self._success_log_count = value
+
+    @property
+    def failure_log_count(self) -> int:
+        with self._lock:
+            return self._failure_log_count
+
+    @failure_log_count.setter
+    def failure_log_count(self, value: int) -> None:
+        with self._lock:
+            self._failure_log_count = value
+
+    @property
+    def buffer_log_count(self) -> int:
+        with self._lock:
+            return self._buffer_log_count
+
+    @buffer_log_count.setter
+    def buffer_log_count(self, value: int) -> None:
+        with self._lock:
+            self._buffer_log_count = value
+
+    @property
+    def last_error_message(self) -> Optional[str]:
+        with self._lock:
+            return self._last_error_message
+
+    @last_error_message.setter
+    def last_error_message(self, value: Optional[str]) -> None:
+        with self._lock:
+            self._last_error_message = value
+
+    @property
+    def last_error_time(self) -> int:
+        with self._lock:
+            return self._last_error_time
+
+    @last_error_time.setter
+    def last_error_time(self, value: int) -> None:
+        with self._lock:
+            self._last_error_time = value
+
+    def increment_success_count(self) -> int:
+        with self._lock:
+            self._success_count += 1
+            return self._success_count
+
+    def increment_failure_count(self) -> int:
+        with self._lock:
+            self._failure_count += 1
+            return self._failure_count
+
+    def add_success_log_count(self, delta: int) -> int:
+        with self._lock:
+            self._success_log_count += delta
+            return self._success_log_count
+
+    def add_failure_log_count(self, delta: int) -> int:
+        with self._lock:
+            self._failure_log_count += delta
+            return self._failure_log_count
+
+    def add_buffer_log_count(self, delta: int) -> int:
+        with self._lock:
+            self._buffer_log_count += delta
+            return self._buffer_log_count
+
+    def reset(self) -> None:
+        with self._lock:
+            self._last_report_time = 0
+            self._success_count = 0
+            self._failure_count = 0
+            self._success_log_count = 0
+            self._failure_log_count = 0
+            self._buffer_log_count = 0
+            self._last_error_message = None
+            self._last_error_time = 0

huaweicloud_encryptionsdk/log/log_reporter_manager.py

@@ -0,0 +1,163 @@
+import logging
+import threading
+from typing import Optional
+
+from huaweicloud_encryptionsdk.log.log_config import LogConfig
+from huaweicloud_encryptionsdk.log.lts_sdk_producer import LTSSDKProducer
+from huaweicloud_encryptionsdk.log.lts_struct_log_builder import LtsStructLogBuilder
+
+logger = logging.getLogger(__name__)
+
+MAX_INIT_RETRY_COUNT = 3
+
+
+def desensitize(value: Optional[str]) -> str:
+    if value is None or len(value) <= 4:
+        return "****"
+    return value[:2] + "****" + value[-2:]
+
+
+class LogReporterManager:
+
+    def __init__(self):
+        self._lts_producer: Optional[LTSSDKProducer] = None
+        self._log_config: Optional[LogConfig] = None
+        self._local_log_only: bool = False
+        self._initialized: bool = False
+        self._init_retry_count: int = 0
+        self._lock = threading.Lock()
+
+    def init(self, config) -> None:
+        with self._lock:
+            if self._initialized:
+                logger.debug("LogReporterManager is already initialized")
+                return
+
+            if self._init_retry_count >= MAX_INIT_RETRY_COUNT:
+                logger.warn("LogReporterManager init has exceeded max retry count (%d), giving up", MAX_INIT_RETRY_COUNT)
+                return
+
+            if config.log_config is None and not config.local_log_only:
+                logger.info("Log config is not provided and localLogOnly is false, logging is disabled")
+                return
+
+            self._log_config = config.log_config if config.log_config is not None else LogConfig()
+
+            if config.local_log_only:
+                self._local_log_only = True
+                self._initialized = True
+                logger.info("LogReporterManager initialized with local log only mode")
+                return
+
+            try:
+                if self._log_config.project_id is None:
+                    if config.kms_config_list and len(config.kms_config_list) > 0:
+                        self._log_config.project_id = config.kms_config_list[0].project_id
+                    else:
+                        self._log_config.project_id = "default"
+                self._lts_producer = LTSSDKProducer(
+                    config.get_ak(),
+                    config.get_sk(),
+                    self._log_config.project_id,
+                    self._log_config.region,
+                    self._log_config,
+                    security_token=config.security_token or "",
+                )
+                self._initialized = True
+                logger.info("LogReporterManager initialized successfully with LTS mode")
+                logger.info("Log config: batchCountThreshold=%s, lingerMs=%s, ioThreadCount=%s",
+                            self._log_config.batch_count_threshold,
+                            self._log_config.linger_ms,
+                            self._log_config.io_thread_count)
+            except Exception as e:
+                self._init_retry_count += 1
+                logger.error("Failed to initialize LogReporterManager (attempt %d/%d): %s",
+                             self._init_retry_count, MAX_INIT_RETRY_COUNT, str(e), exc_info=e)
+                if self._log_config.fail_on_backup_dir_missing:
+                    raise
+
+    def record_encrypt_log(
+        self,
+        encrypt_file_name: Optional[str],
+        encrypt_data_bytes: Optional[int],
+        encrypt_output_file_name: Optional[str],
+        encrypt_output_bytes: Optional[int],
+        key_id: Optional[str],
+        access_point_type: Optional[str],
+        instance_id: Optional[str],
+    ) -> None:
+        self._record_log(
+            "encrypt", encrypt_file_name, encrypt_data_bytes,
+            encrypt_output_file_name, encrypt_output_bytes,
+            key_id, access_point_type, instance_id,
+        )
+
+    def record_decrypt_log(
+        self,
+        decrypt_file_name: Optional[str],
+        decrypt_data_bytes: Optional[int],
+        decrypt_output_file_name: Optional[str],
+        decrypt_output_bytes: Optional[int],
+        key_id: Optional[str],
+        access_point_type: Optional[str],
+        instance_id: Optional[str],
+    ) -> None:
+        self._record_log(
+            "decrypt", decrypt_file_name, decrypt_data_bytes,
+            decrypt_output_file_name, decrypt_output_bytes,
+            key_id, access_point_type, instance_id,
+        )
+
+    def _record_log(
+        self,
+        operation: str,
+        file_name: Optional[str],
+        data_bytes: Optional[int],
+        output_file_name: Optional[str],
+        output_bytes: Optional[int],
+        key_id: Optional[str],
+        access_point_type: Optional[str],
+        instance_id: Optional[str],
+    ) -> None:
+        if not self._initialized:
+            return
+        try:
+            struct_log = LtsStructLogBuilder.build_struct_log(
+                operation, file_name, data_bytes,
+                output_file_name, output_bytes,
+                key_id, access_point_type, instance_id,
+            )
+
+            if self._local_log_only:
+                logger.info("[LOCAL_LOG] %s", struct_log)
+                return
+
+            if self._lts_producer is not None:
+                self._lts_producer.send_struct_log(struct_log)
+            logger.info("Recorded %s log for keyId: %s, dataBytes: %s, accessPointType: %s",
+                        operation, desensitize(key_id), data_bytes, access_point_type)
+        except Exception as e:
+            logger.error("Failed to record %s operation log: %s", operation, str(e))
+
+    def shutdown(self) -> None:
+        with self._lock:
+            if not self._initialized:
+                return
+            if self._lts_producer is not None:
+                try:
+                    self._lts_producer.close()
+                except Exception as e:
+                    logger.error("Failed to close LTS producer during shutdown, will retry on next shutdown call: %s", str(e))
+                    return
+                self._lts_producer = None
+            self._initialized = False
+            self._init_retry_count = 0
+            logger.info("LogReporterManager shutdown successfully")
+
+    @property
+    def is_initialized(self) -> bool:
+        return self._initialized
+
+    @property
+    def lts_producer(self) -> Optional[LTSSDKProducer]:
+        return self._lts_producer