huaweicloud-encryption-sdk-python 1.2.1__py3-none-any.whl

huaweicloud_encryptionsdk/_lts_sdk/serial_main.py

@@ -0,0 +1,77 @@
+import argparse
+import random
+
+from loguru import logger
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.core.producer import Producer
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import lts_store
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.config import Config
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+def generate_random_str(str_len=16):
+    """
+    生成一个指定长度的随机字符串
+    """
+    random_str = ''
+    base_str = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789'
+    length = len(base_str) - 1
+    for i in range(str_len):
+        random_str += base_str[random.randint(0, length)]
+    return random_str
+
+
+def send_log_single_producer(endpoint, ak, sk, region_id, project_id, group_id, stream_id, log_count, log_size):
+    log_num = log_count
+    config = Config()
+    config.endpoint = endpoint
+    config.access_key = ak
+    config.access_secret = sk
+    config.region_id = region_id
+    config.project_id = project_id
+    producer = Producer.init_producer(config)
+    producer.start_producer()
+
+    group_id = group_id
+    stream_id = stream_id
+
+    batch_num = 20
+    count = int(log_num/batch_num)
+    leave = log_num % batch_num
+
+    logger.info("send count is {}, leave is {}", count, leave)
+
+    log_content = generate_random_str(log_size)
+    labels = {"keyA": "valueA"}
+    log = []
+    for i in range(batch_num):
+        log.append(log_content)
+    log_p = lts_store.generate_log(log, labels)
+
+    begin_ms = common.current_time_ms()
+    for i in range(count):
+        producer.send_log(group_id, stream_id, log_p)
+    log.clear()
+
+    if leave > 0:
+        for i in range(leave):
+            log.append(log_content)
+        log_p = lts_store.generate_log(log, labels)
+        producer.send_log(group_id, stream_id, log_p)
+    end_ms = common.current_time_ms()
+    logger.info("end send log, cost [{}]ms", end_ms - begin_ms)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='lts python sdk send log help tester information')
+    parser.add_argument('--endpoint', dest='endpoint', type=str, help='lts endpoint')
+    parser.add_argument('--ak', dest='ak', type=str, help='huaweicloud ak')
+    parser.add_argument('--sk', dest='sk', type=str, help='huaweicloud sk')
+    parser.add_argument('--region', dest='region', type=str, help='huaweicloud region')
+    parser.add_argument('--project_id', dest='project_id', type=str, help='huaweicloud project_id')
+    parser.add_argument('--group_id', dest='group_id', type=str, help='send log group_id')
+    parser.add_argument('--stream_id', dest='stream_id', type=str, help='send log stream_id')
+    parser.add_argument('--log_count', dest='log_count', type=int, help='send log log_count')
+    parser.add_argument('--log_size', dest='log_size', type=int, help='send log log_size(byte)')
+    args = parser.parse_args()
+    send_log_single_producer(args.endpoint, args.ak, args.sk, args.region, args.project_id, args.group_id, args.stream_id, args.log_count, args.log_size)

huaweicloud_encryptionsdk/_lts_sdk/utils/common.py

@@ -0,0 +1,57 @@
+import json
+import time
+
+
+def solv_log(x):
+    n = 0
+    while True:
+        n += 1
+        x >>= 7
+        if x == 0:
+            break
+    return n
+
+
+def _encode_obj(obj) -> dict:
+    obj_dict = {}
+    obj_dict.update(obj.__dict__)
+    return obj_dict
+
+
+def convert_obj2_json_str(obj):
+    if obj is None:
+        return ""
+    return json.dumps(obj, default=_encode_obj, indent=None, sort_keys=True)
+
+
+def convert_json_str2_obj(json_str, objClass):
+    parse_data = json.loads(json_str.strip('\t\r\n'))
+    result = objClass()
+    result.__dict__ = parse_data
+    return result
+
+
+def convert_dict2_obj(parse_data, objClass):
+    result = objClass()
+    result.__dict__ = parse_data
+    return result
+
+
+def convert_json_str2_list_obj(json_str, objClass):
+    lists = json.loads(json_str.strip('\t\r\n'))
+    result = []
+    for each_item in lists:
+        result.append(convert_dict2_obj(each_item, objClass))
+    return result
+
+
+def current_time_ms() -> int:
+    return int((time.time() * 1000))
+
+
+def current_time_ns() -> int:
+    return time.time_ns()
+
+
+def sleep_ms(mill):
+    time.sleep(mill / 1000)

huaweicloud_encryptionsdk/_lts_sdk/utils/lts_threading.py

@@ -0,0 +1,32 @@
+import threading
+
+
+class ReadWriteLock:
+
+    def __init__(self):
+        self._cond = threading.Condition(threading.Lock())
+        self._active_readers = 0
+
+    def acquire_read(self):
+        self._cond.acquire()
+        try:
+            self._active_readers += 1
+        finally:
+            self._cond.release()
+
+    def release_read(self):
+        self._cond.acquire()
+        try:
+            self._active_readers -= 1
+            if not self._active_readers:
+                self._cond.notify_all()
+        finally:
+            self._cond.release()
+
+    def acquire_write(self):
+        self._cond.acquire()
+        while self._active_readers > 0:
+            self._cond.wait()
+
+    def release_write(self):
+        self._cond.release()

huaweicloud_encryptionsdk/cache/data_key_cache.py

@@ -0,0 +1,79 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from huaweicloud_encryptionsdk.model.data_key_materials import DataKeyMaterials
+from huaweicloud_encryptionsdk.common.utils import add_prevent_overflow
+
+
+class UsageStatus:
+
+    def __init__(self, bytes_encrypt_count: int, message_encrypt_count: int):
+        self._bytes_encrypt_count = bytes_encrypt_count
+        self._message_encrypt_count = message_encrypt_count
+
+    @property
+    def bytes_encrypt_count(self) -> int:
+        return self._bytes_encrypt_count
+
+    @property
+    def message_encrypt_count(self) -> int:
+        return self._message_encrypt_count
+
+    def add(self, other: "UsageStatus") -> "UsageStatus":
+        return UsageStatus(
+            add_prevent_overflow(self._bytes_encrypt_count, other._bytes_encrypt_count),
+            add_prevent_overflow(self._message_encrypt_count, other._message_encrypt_count),
+        )
+
+
+class EncryptCacheEntry(ABC):
+
+    @abstractmethod
+    def get_usage_status(self) -> UsageStatus:
+        pass
+
+    @abstractmethod
+    def get_data_key_materials(self) -> DataKeyMaterials:
+        pass
+
+    def invalidate(self) -> None:
+        pass
+
+
+class DecryptCacheEntry(ABC):
+
+    @abstractmethod
+    def get_data_key_materials(self) -> DataKeyMaterials:
+        pass
+
+    def invalidate(self) -> None:
+        pass
+
+
+class DataKeyCache(ABC):
+
+    @abstractmethod
+    def get_entry_for_encrypt(self, cache_id: str, usage_status: UsageStatus) -> Optional[EncryptCacheEntry]:
+        pass
+
+    @abstractmethod
+    def put_entry_for_encrypt(self, cache_id: str, data_key_materials: DataKeyMaterials, initial_usage: UsageStatus) -> None:
+        pass
+
+    def put_entry_for_encrypt_with_ttl(self, cache_id: str, survive_time: int, data_key_materials: DataKeyMaterials, initial_usage: UsageStatus) -> None:
+        self.put_entry_for_encrypt(cache_id, data_key_materials, initial_usage)
+
+    @abstractmethod
+    def get_entry_for_decrypt(self, cache_id: str) -> Optional[DecryptCacheEntry]:
+        pass
+
+    @abstractmethod
+    def put_entry_for_decrypt(self, cache_id: str, data_key_materials: DataKeyMaterials) -> None:
+        pass
+
+    def put_entry_for_decrypt_with_ttl(self, cache_id: str, survive_time: int, data_key_materials: DataKeyMaterials) -> None:
+        self.put_entry_for_decrypt(cache_id, data_key_materials)
+
+    @abstractmethod
+    def set_survival_time(self, survival_time: int) -> None:
+        pass

huaweicloud_encryptionsdk/cache/local_data_key_cache.py

@@ -0,0 +1,161 @@
+import time
+import threading
+from collections import OrderedDict
+from typing import Optional, List
+
+from huaweicloud_encryptionsdk.cache.data_key_cache import DataKeyCache, UsageStatus, EncryptCacheEntry, DecryptCacheEntry
+from huaweicloud_encryptionsdk.model.data_key_materials import DataKeyMaterials
+
+
+class _BaseEntry:
+
+    def __init__(self, key: str, expire_timestamp: int, material: Optional[DataKeyMaterials] = None, usage_status: Optional[UsageStatus] = None):
+        self.key = key
+        self.expire_timestamp = expire_timestamp
+        self.create_timestamp = int(time.time() * 1000)
+        self.material = material
+        self.usage_status = usage_status
+        self._lock = threading.Lock()
+
+    def is_expired(self) -> bool:
+        return int(time.time() * 1000) > self.expire_timestamp
+
+    def add_usage_info(self, usage_status: UsageStatus) -> UsageStatus:
+        with self._lock:
+            self.usage_status = self.usage_status.add(usage_status)
+            return self.usage_status
+
+    def __lt__(self, other):
+        if self.expire_timestamp != other.expire_timestamp:
+            return self.expire_timestamp < other.expire_timestamp
+        return self.create_timestamp < other.create_timestamp
+
+    def __eq__(self, other):
+        if not isinstance(other, _BaseEntry):
+            return False
+        return (self.expire_timestamp == other.expire_timestamp
+                and self.create_timestamp == other.create_timestamp
+                and self.key == other.key)
+
+    def __hash__(self):
+        return hash((self.key, self.expire_timestamp, self.create_timestamp))
+
+
+class _EncryptionEntryExposed(EncryptCacheEntry):
+
+    def __init__(self, entry: _BaseEntry, usage_status: UsageStatus):
+        self._entry = entry
+        self._usage_status = usage_status
+
+    def get_usage_status(self) -> UsageStatus:
+        return self._usage_status
+
+    def get_data_key_materials(self) -> DataKeyMaterials:
+        return self._entry.material
+
+    def invalidate(self) -> None:
+        pass
+
+
+class _DecryptionEntry(_BaseEntry, DecryptCacheEntry):
+
+    def get_data_key_materials(self) -> DataKeyMaterials:
+        return self.material
+
+    def invalidate(self) -> None:
+        pass
+
+
+class LocalDataKeyCache(DataKeyCache):
+
+    DEFAULT_CAPACITY = 1 << 4
+    DEFAULT_TTL = 1000000
+
+    def __init__(self):
+        self._capacity = self.DEFAULT_CAPACITY
+        self._key_expire_time = self.DEFAULT_TTL
+        self._cache: OrderedDict[str, _BaseEntry] = OrderedDict()
+        self._tree_set: List[_BaseEntry] = []
+        self._lock = threading.Lock()
+
+    def set_capacity(self, capacity: int) -> None:
+        self._capacity = capacity
+
+    def get_entry_for_encrypt(self, cache_id: str, usage_status: UsageStatus) -> Optional[EncryptCacheEntry]:
+        entry = self._get_entry(cache_id)
+        if entry is None:
+            return None
+        if entry.is_expired():
+            self._remove_entry(entry)
+            return None
+        status = entry.add_usage_info(usage_status)
+        return _EncryptionEntryExposed(entry, status)
+
+    def put_entry_for_encrypt(self, cache_id: str, data_key_materials: DataKeyMaterials, initial_usage: UsageStatus) -> None:
+        base_entry = _BaseEntry(cache_id, self._key_expire_time + int(time.time() * 1000), data_key_materials, initial_usage)
+        self._put_entry(base_entry)
+
+    def put_entry_for_encrypt_with_ttl(self, cache_id: str, survive_time: int, data_key_materials: DataKeyMaterials, initial_usage: UsageStatus) -> None:
+        base_entry = _BaseEntry(cache_id, survive_time + int(time.time() * 1000), data_key_materials, initial_usage)
+        self._put_entry(base_entry)
+
+    def get_entry_for_decrypt(self, cache_id: str) -> Optional[DecryptCacheEntry]:
+        entry = self._get_entry(cache_id)
+        if entry is None:
+            return None
+        if entry.is_expired():
+            self._remove_entry(entry)
+            return None
+        return _DecryptionEntry(entry.key, entry.expire_timestamp, entry.material, entry.usage_status)
+
+    def put_entry_for_decrypt(self, cache_id: str, data_key_materials: DataKeyMaterials) -> None:
+        entry = _DecryptionEntry(cache_id, self._key_expire_time + int(time.time() * 1000), data_key_materials, None)
+        self._put_entry(entry)
+
+    def put_entry_for_decrypt_with_ttl(self, cache_id: str, survive_time: int, data_key_materials: DataKeyMaterials) -> None:
+        entry = _DecryptionEntry(cache_id, survive_time + int(time.time() * 1000), data_key_materials, None)
+        self._put_entry(entry)
+
+    def set_survival_time(self, survival_time: int) -> None:
+        self._key_expire_time = survival_time
+
+    def _get_entry(self, key: str) -> Optional[_BaseEntry]:
+        with self._lock:
+            entry = self._cache.get(key)
+            if entry is not None:
+                self._cache.move_to_end(key)
+            return entry
+
+    def _remove_entry(self, entry: _BaseEntry) -> None:
+        with self._lock:
+            self._cache.pop(entry.key, None)
+            try:
+                self._tree_set.remove(entry)
+            except ValueError:
+                pass
+
+    def _put_entry(self, entry: _BaseEntry) -> None:
+        with self._lock:
+            old_entry = self._cache.pop(entry.key, None)
+            if old_entry is not None:
+                try:
+                    self._tree_set.remove(old_entry)
+                except ValueError:
+                    pass
+            self._cache[entry.key] = entry
+            self._tree_set.append(entry)
+            self._tree_set.sort()
+            now = int(time.time() * 1000)
+            while self._tree_set and self._tree_set[0].expire_timestamp < now:
+                expired = self._tree_set.pop(0)
+                self._cache.pop(expired.key, None)
+            self._check_capacity()
+
+    def _check_capacity(self) -> None:
+        while len(self._cache) > self._capacity:
+            oldest_key = next(iter(self._cache))
+            old_entry = self._cache.pop(oldest_key)
+            try:
+                self._tree_set.remove(old_entry)
+            except ValueError:
+                pass

huaweicloud_encryptionsdk/common/constants.py

@@ -0,0 +1,15 @@
+class Constants:
+    KEY_SPEC_DELIMITER = "_"
+    NUM_0 = 0
+    NUM_1 = 1
+    BLOCK_SIZE = 1024 * 256
+    VERSION = 1
+    VERSION_2 = 2
+
+
+KEY_SPEC_DELIMITER = Constants.KEY_SPEC_DELIMITER
+NUM_0 = Constants.NUM_0
+NUM_1 = Constants.NUM_1
+BLOCK_SIZE = Constants.BLOCK_SIZE
+VERSION = Constants.VERSION
+VERSION_2 = Constants.VERSION_2

huaweicloud_encryptionsdk/common/counting_output_stream.py

@@ -0,0 +1,18 @@
+class CountingOutputStream:
+
+    def __init__(self, output_stream):
+        self._output_stream = output_stream
+        self._bytes_written = 0
+
+    def write(self, data):
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+        self._output_stream.write(data)
+        self._bytes_written += len(data)
+
+    def get_bytes_written(self):
+        return self._bytes_written
+
+    @property
+    def output_stream(self):
+        return self._output_stream

huaweicloud_encryptionsdk/common/crypto_factory.py

@@ -0,0 +1,48 @@
+import hashlib
+
+
+class CryptoFactory:
+
+    @staticmethod
+    def get_cipher(algorithm):
+        from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+        parts = algorithm.split("/")
+        alg_name = parts[0]
+        mode_name = parts[1] if len(parts) > 1 else ""
+        return alg_name, mode_name
+
+    @staticmethod
+    def get_message_digest(algorithm):
+        if algorithm is None:
+            raise ValueError('algorithm must not be None')
+        if CryptoFactory.is_sm_algorithm(algorithm):
+            try:
+                from gmssl import sm3
+                return SM3Digest()
+            except ImportError:
+                raise ValueError('SM3 algorithm requires gmssl library. Install with: pip install gmssl')
+        return hashlib.new(algorithm)
+
+    @staticmethod
+    def is_sm_algorithm(algorithm):
+        return algorithm is not None and "SM" in algorithm.upper()
+
+
+class SM3Digest:
+
+    def __init__(self):
+        from gmssl import sm3 as _sm3
+        self._sm3 = _sm3
+        self._data = bytearray()
+
+    def update(self, data):
+        self._data.extend(data)
+
+    def digest(self):
+        from gmssl import sm3 as _sm3, func
+        result = _sm3.sm3_hash(list(self._data))
+        return bytes.fromhex(result)
+
+    def hexdigest(self):
+        from gmssl import sm3 as _sm3
+        return _sm3.sm3_hash(list(self._data))

huaweicloud_encryptionsdk/common/ec_private_key_loader.py

@@ -0,0 +1,43 @@
+import base64
+
+from cryptography.hazmat.primitives.serialization import (
+    load_pem_private_key,
+    load_der_private_key,
+)
+from cryptography.hazmat.backends import default_backend
+
+
+class ECPrivateKeyLoader:
+
+    @staticmethod
+    def load(key_content):
+        if key_content.strip().startswith('-----BEGIN'):
+            try:
+                return ECPrivateKeyLoader.load_pem(key_content)
+            except Exception as e:
+                raise Exception("access point key exception") from e
+        try:
+            der = base64.b64decode(key_content)
+            return ECPrivateKeyLoader.load_der(der)
+        except Exception as e:
+            raise Exception("access point key exception") from e
+
+    @staticmethod
+    def load_pem(pem):
+        try:
+            private_key = load_pem_private_key(
+                pem.encode("utf-8"), password=None, backend=default_backend()
+            )
+            return private_key
+        except Exception as e:
+            raise Exception("access point key exception") from e
+
+    @staticmethod
+    def load_der(der):
+        try:
+            private_key = load_der_private_key(
+                der, password=None, backend=default_backend()
+            )
+            return private_key
+        except Exception as e:
+            raise Exception("access point key exception") from e

huaweicloud_encryptionsdk/common/ecs_metadata_service.py

@@ -0,0 +1,42 @@
+import json
+import requests
+
+
+class EcsMetadataService:
+
+    METADATA_BASE = "http://169.254.169.254/meta-data/latest/instance-identity"
+    DOCUMENT_URL = METADATA_BASE + "/document"
+    SIGNATURE_URL = METADATA_BASE + "/signature"
+
+    _cached_instance_id = None
+
+    @classmethod
+    def get_ecs_instance_id(cls):
+        if cls._cached_instance_id is not None:
+            return cls._cached_instance_id
+        try:
+            doc_str = cls.fetch_identity_document()
+            if doc_str is None:
+                return None
+            doc = json.loads(doc_str)
+            cls._cached_instance_id = doc.get("id")
+            return cls._cached_instance_id
+        except (json.JSONDecodeError, requests.exceptions.RequestException):
+            return None
+
+    @classmethod
+    def fetch_identity_document(cls):
+        return cls._fetch_metadata(cls.DOCUMENT_URL)
+
+    @classmethod
+    def fetch_identity_signature(cls):
+        return cls._fetch_metadata(cls.SIGNATURE_URL)
+
+    @classmethod
+    def _fetch_metadata(cls, url):
+        try:
+            resp = requests.get(url, timeout=5)
+            resp.raise_for_status()
+            return resp.text
+        except Exception:
+            return None

huaweicloud_encryptionsdk/common/file_path_for_example_constants.py

@@ -0,0 +1,4 @@
+PLAIN_TEXT_FILE_PATH = "plain_text.txt"
+CIPHER_TEXT_FILE_PATH = "cipher_text.txt"
+DECRYPT_TEXT_FILE_PATH = "decrypt_text.txt"
+KEY_FILE_PATH = "key.txt"