plexop-infrastructure 1.1.0__tar.gz

plexop_infrastructure-1.1.0/MANIFEST.in

@@ -0,0 +1,22 @@
+include README.md
+include pyproject.toml
+include LICENSE
+include MANIFEST.in
+# Include the package code
+recursive-include src *
+
+# Explicitly exclude lambda files
+prune lambda_env
+prune layer
+prune layer.zip
+prune lambda_layer.zip
+
+# Exclude these files from main directory
+exclude azure-pipelines.yml
+exclude .gitignore
+exclude .gitattributes
+exclude *.pyproj
+exclude *.version
+exclude conf.cfg
+exclude requirements.txt
+exclude *.bat

plexop_infrastructure-1.1.0/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: plexop-infrastructure
+Version: 1.1.0
+Summary: Plexop infrastructure shared code
+Author: Vlad Stremousov
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: boto3
+Requires-Dist: python-dateutil
+Requires-Dist: urllib3
+Requires-Dist: pygelf
+
+Package with Plexop Python infrastructure
+Include 
+	* Logger decorator
+	* Config decorator
+	* Requests decorator
+	* Some boto3 stuff decorators
+
+Install from http://static.plexop.com/packages/python/plexop-infrastructure-latest.tar.gz
+
+in order to upgrade dependencies:
+cd C:\Projects\python-lambda-infrastructure-layer
+pip install --upgrade -r requirements.txt --target=C:\Projects\python-lambda-infrastructure-layer\env\Lib\site-packages
+
+# Installation
+* The layer can be used as pip-installable package and as a lambda layer.  
+* Packages are located in `s3://static-plexop/packages/python/plexop-infrastructure`. Open a given version and copy it's URL.
+    * It can be installed via e.g. `pip install http://static.plexop.com/packages/python/plexop-infrastructure-0.2.8.tar.gz`
+    * Latest version is always here: http://static.plexop.com/packages/python/plexop-infrastructure-latest.tar.gz
+
+# Deployment
+* Each commit to this repo will create a new version (both on s3 & layer) with version name <branch>-<commit>-<date>
+* To deploy a given commit to PROD, create a new branch based off this commit and name the branch with some semantic version and add a `-release` suffix (e.g. `1.3-release`)

plexop_infrastructure-1.1.0/README.md

@@ -0,0 +1,22 @@
+Package with Plexop Python infrastructure
+Include 
+	* Logger decorator
+	* Config decorator
+	* Requests decorator
+	* Some boto3 stuff decorators
+
+Install from http://static.plexop.com/packages/python/plexop-infrastructure-latest.tar.gz
+
+in order to upgrade dependencies:
+cd C:\Projects\python-lambda-infrastructure-layer
+pip install --upgrade -r requirements.txt --target=C:\Projects\python-lambda-infrastructure-layer\env\Lib\site-packages
+
+# Installation
+* The layer can be used as pip-installable package and as a lambda layer.  
+* Packages are located in `s3://static-plexop/packages/python/plexop-infrastructure`. Open a given version and copy it's URL.
+    * It can be installed via e.g. `pip install http://static.plexop.com/packages/python/plexop-infrastructure-0.2.8.tar.gz`
+    * Latest version is always here: http://static.plexop.com/packages/python/plexop-infrastructure-latest.tar.gz
+
+# Deployment
+* Each commit to this repo will create a new version (both on s3 & layer) with version name <branch>-<commit>-<date>
+* To deploy a given commit to PROD, create a new branch based off this commit and name the branch with some semantic version and add a `-release` suffix (e.g. `1.3-release`)

plexop_infrastructure-1.1.0/azure-pipelines-build-tag.yml

@@ -0,0 +1,38 @@
+trigger:
+  branches:
+    include:
+      - master       # Run only on master commits
+  tags:
+    exclude:
+      - "*"          # Never run on tags
+
+pool:
+  name: default-linux
+
+stages:
+- stage: SemanticVersioning
+  displayName: "Semantic Versioning (Commitizen)"
+  jobs:
+    - job: BumpVersion
+      displayName: "Bump version & create git tag"
+      container:
+        image: python:3.13
+        options: --user root
+        
+      steps:
+        - checkout: self
+          fetchDepth: 0
+          persistCredentials: true
+
+        - script: |
+            pip install --upgrade pip
+            pip install commitizen
+            export PATH="$HOME/.local/bin:$PATH"
+            git config user.email "$(Build.RequestedForEmail)"
+            git config user.name  "$(Build.RequestedFor)"            
+            git fetch origin
+            git checkout master
+            git reset --hard origin/master
+            echo "Running Commitizen bump..."
+            cz bump --yes || echo "No new release detected."
+          displayName: "Semantic Versioning (Commitizen)"

plexop_infrastructure-1.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["setuptools>=68", "wheel", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "plexop-infrastructure"        # PyPI distribution name (hyphens OK)
+dynamic = ["version"]
+description = "Plexop infrastructure shared code"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Vlad Stremousov" }]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.13",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+]
+dependencies = [
+  "requests",
+  "boto3",
+  "python-dateutil",
+  "urllib3",
+  "pygelf",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["plexop*"]
+
+[tool.setuptools_scm]
+write_to = "src/plexop/_version.py"
+
+# Commitizen: parse conventional commits and bump version
+[tool.commitizen]
+name = "cz_conventional_commits"
+version = "1.1.0"
+tag_format = "v$version"
+version_files = ["pyproject.toml"]
+post_bump_hooks = [
+    "git push",
+    "git push --tags"
+]
+update_changelog_on_bump = false
+bump_message = "bump: version $current_version → $new_version [skip ci]"

plexop_infrastructure-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

plexop_infrastructure-1.1.0/src/debug.py

@@ -0,0 +1,18 @@
+from plexop.infrastructure import logger, config
+from plexop.infrastructure.http_requester import HttpRequester
+
+logger.debug('test', obj={'mobile_number': '+972-55-55-55-55', 'email': 'asdfsDf@gmaiL.com', 'test': 'test',
+                          'original_querystring': '?asdfd=213&mobile=&phone=234&email=asdfsdf@gmail.com'})
+
+#r = HttpRequester()
+#result = r.execute('http://5aklie.free.bg', read_as_json=False, use_cache=True, raw_response=1)
+#result = r.execute('http://5aklie.free.bg', read_as_json=False, use_cache=True, raw_response=1)
+
+# from plexop.infrastructure.aws.SNS import SNS
+# sns = SNS(profile_name='dev')
+# sns.publish('arn:aws:sns:us-east-1:872540588091:sophie-test', 'test')
+#
+# from plexop.infrastructure.aws.SQS import SQS
+# sqs = SQS(region=None, profile_name='dev')
+# sqs.push_message('https://sqs.us-east-1.amazonaws.com/872540588091/hello-world', 'test')
+# sqs.push_messages('https://sqs.us-east-1.amazonaws.com/872540588091/hello-world', ['test1', 'test2'])

plexop_infrastructure-1.1.0/src/plexop/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '1.1.0'
+__version_tuple__ = version_tuple = (1, 1, 0)
+
+__commit_id__ = commit_id = None

plexop_infrastructure-1.1.0/src/plexop/infrastructure/__init__.py

@@ -0,0 +1,14 @@
+from plexop.infrastructure.config_provider import get_config
+from plexop.infrastructure.log_provider import Logger
+
+if 'config' not in globals():
+    config = get_config()
+
+if 'logger' not in globals():
+    Logger(config, 'quart')
+    Logger(config, 'flask')
+    Logger(config, 'boto3')
+    logger = Logger(config)
+
+def get_logger(logger_name):
+    return Logger(config, logger_name)

plexop_infrastructure-1.1.0/src/plexop/infrastructure/aws/DynamoDB.py

@@ -0,0 +1,67 @@
+import boto3
+from botocore.exceptions import ClientError
+from plexop.infrastructure.models.write_condition_not_met_exception import WriteConditionNotMetError
+from plexop.infrastructure.sleep_iterator import sleep_iterator
+from plexop.infrastructure import config, logger
+import time
+from functools import wraps
+
+
+def decorate_with_try(fn):
+    @wraps(fn)
+    def surround(*args, **kwargs):
+        sleeper = sleep_iterator()
+        start_time = time.time()
+        while next(sleeper):
+            try:
+                logger.debug("try to execute dynamo operation", kwargs=kwargs, fn=str(fn))
+                result = fn(*args, **kwargs)
+                logger.debug("dynamo operation executed successfully", kwargs=kwargs, fn=str(fn), result=result)
+                return result
+            except ClientError as ex:
+                elapsed_time = time.time() - start_time
+                error_code = ex.response['Error']['Code']
+                if error_code == u'ConditionalCheckFailedException':
+                    raise WriteConditionNotMetError()
+                logger.exception(ex, function_name=fn.__name__, elapsed_time=elapsed_time, args=str(args),
+                                 kwargs=str(kwargs), error_code=error_code)
+                if error_code == u'ProvisionedThroughputExceededException':
+                    continue
+                raise
+
+    return surround
+
+
+class DynamoDB:
+    def __init__(self, region=None, profile_name=None):
+        region = region or config.get('dynamodb', 'region', fallback=None)
+        profile_name = profile_name or config.get('dynamodb', 'profile_name', fallback=None)
+        self.client = boto3.session.Session(profile_name=profile_name, region_name=region).client('dynamodb')
+
+    @decorate_with_try
+    def get_item(self, *args, **kwargs):
+        response = self.client.get_item(**kwargs)
+        if 'Item' not in response:
+            return None
+        result = dict((key, list(value.values())[0]) for key, value in response['Item'].items())
+        return result
+
+    @decorate_with_try
+    def query(self, *args, **kwargs):
+        response = self.client.query(**kwargs)
+        return response
+
+    @decorate_with_try
+    def put_item(self, *args, **kwargs):
+        result = self.client.put_item(**kwargs)
+        return result.get('Attributes')
+
+    @decorate_with_try
+    def update_item(self, *args, **kwargs):
+        result = self.client.update_item(**kwargs)
+        return result.get('Attributes')
+
+    @decorate_with_try
+    def delete_item(self, *args, **kwargs):
+        query_result = self.client.delete_item(**kwargs)
+        result = dict((key, list(value.values())[0]) for key, value in query_result['Attributes'].items())

plexop_infrastructure-1.1.0/src/plexop/infrastructure/aws/Firehose.py

@@ -0,0 +1,17 @@
+import boto3
+import json
+import base64
+from plexop.infrastructure import config, logger
+
+class Firehose:
+    def __init__(self, region=None, profile_name=None):
+        region = region or config.get('firehose', 'region', fallback=None)
+        profile_name = profile_name or config.get('firehose', 'profile_name', fallback=None)
+        self.client = boto3.session.Session(profile_name=profile_name, region_name=region).client('firehose')
+
+    def put_record(self, data, stream_name=None):
+        stream_name = stream_name or config.get('firehose', 'stream_name')
+        logger.debug("sending data to fireshose stream", stream_name=stream_name, data=data)
+        result = self.client.put_record(DeliveryStreamName=stream_name, Record={'Data': data})
+        logger.debug("data sent to firehose successfully", result=result)
+        return result

plexop_infrastructure-1.1.0/src/plexop/infrastructure/aws/KMS.py

@@ -0,0 +1,25 @@
+import boto3
+import json
+import base64
+from plexop.infrastructure import config, logger
+
+
+class KMS:
+    def __init__(self):
+        kms_region = config.get('kms', 'region')
+        self.key_id = config.get('kms', 'key_id')
+        if self.key_id:
+            self.client = boto3.client('kms', region_name=kms_region)
+
+    def encrypt(self, message):
+        if not self.key_id:
+            return message
+        result = self.client.encrypt(KeyId=self.key_id, Plaintext=message)
+        return base64.b64encode(result['CiphertextBlob']).decode("utf-8")
+
+    def decrypt(self, message):
+        if not self.key_id:
+            return message
+        decrypted_message = self.client.decrypt(CiphertextBlob=base64.b64decode(message))
+        result = decrypted_message['Plaintext']
+        return result

plexop_infrastructure-1.1.0/src/plexop/infrastructure/aws/S3.py

@@ -0,0 +1,41 @@
+import boto3
+import json
+from plexop.infrastructure import config, logger
+
+
+class S3:
+    def __init__(self, region=None, profile_name=None):
+        region = region or config.get('s3', 'region', fallback=None)
+        profile_name = profile_name or config.get('s3', 'profile_name', fallback=None)
+        self.client = boto3.session.Session(profile_name=profile_name, region_name=region).client('s3')
+
+    def read(self, bucket_name, key, read_as_json=True, suppress_404_exception=False, is_binary=False):
+        try:
+            logger.debug("try to read from S3", bucket_name=bucket_name, key=key)
+            file = self.client.get_object(Bucket=bucket_name, Key=key)
+            file_content = file['Body'].read()
+            if is_binary:
+                logger.debug("binary data loaded from S3")
+                return file_content
+            file_content = file_content.decode('utf-8')
+            if read_as_json:
+                json_result = json.loads(file_content)
+                logger.debug("json content loaded from S3", result=json_result)
+                return json_result
+            else:
+                logger.debug("content loaded from S3", result=file_content)
+                return file_content
+        except Exception as ex:
+            if hasattr(ex, 'response') and ex.response['Error']['Code'] == 'NoSuchKey' and suppress_404_exception:
+                logger.info("File not found", bucket_name=bucket_name, key=key)
+                return
+            logger.exception(ex)
+            raise
+
+    def save(self, bucket_name, key, body):
+        logger.debug("try to put object to S3", bucket_name=bucket_name, key=key, body=body)
+        if isinstance(body, dict):
+            body = json.dumps(body)
+        result = self.client.put_object(Bucket=bucket_name, Key=key, Body=body)
+        logger.debug("content saved", result=result)
+        return result

plexop_infrastructure-1.1.0/src/plexop/infrastructure/aws/SNS.py

@@ -0,0 +1,18 @@
+import boto3
+import json
+from plexop.infrastructure import config, logger
+
+
+class SNS:
+    def __init__(self, region=None, profile_name=None):
+        region = region or config.get('sns', 'region', fallback=None)
+        profile_name = profile_name or config.get('sqs', 'profile_name', fallback=None)
+        self.client = boto3.session.Session(profile_name=profile_name, region_name=region).client('sns')
+
+    def publish(self, TopicArn, Message, **kwargs):
+        logger.debug("sending message to sns", topic=TopicArn, message_to_publish=Message, **kwargs)
+        if not isinstance(Message, str):
+            Message = json.dumps(Message)
+        publication_result = self.client.publish(TopicArn=TopicArn, Message=Message, **kwargs)
+        logger.debug("message sent to sns", result=publication_result)
+        return publication_result

plexop_infrastructure-1.1.0/src/plexop/infrastructure/aws/SQS.py

@@ -0,0 +1,42 @@
+import boto3
+import json
+import uuid
+from plexop.infrastructure import config, logger
+
+
+class SQS:
+    def __init__(self, region=None, profile_name=None):
+        region = region or config.get('sqs', 'region', fallback=None)
+        profile_name = profile_name or config.get('sqs', 'profile_name', fallback=None)
+        self.client = boto3.session.Session(profile_name=profile_name, region_name=region).client('sqs')
+
+    def push_message(self, QueueUrl, MessageBody, **kwargs):
+        logger.debug("Try to send message to queue", queue_url=QueueUrl, message_body=MessageBody, **kwargs)
+        if isinstance(MessageBody, dict):
+            MessageBody = json.dumps(MessageBody)
+        receipt = self.client.send_message(QueueUrl=QueueUrl, MessageBody=MessageBody, **kwargs)
+        logger.debug("Message sent to SQS", receipt=receipt)
+        return receipt
+
+    def push_messages(self, QueueUrl, MessageBodies, **kwargs):
+        def build_message_for_bulk(message):
+            return {
+                'Id': str(uuid.uuid1()),
+                'MessageBody': message if isinstance(message, str) else json.dumps(message),
+            }
+        if not hasattr(MessageBodies, '__iter__'):
+            raise Exception("Bodies argument for bulk messages SQS push is not iterable")
+        if not len(MessageBodies):
+            logger.warn("Push bulk messages call failed. Bodies array length is 0")
+            return
+        logger.debug('Try to push messages to SQS', queue_url=QueueUrl, count=len(MessageBodies))
+        messages = [build_message_for_bulk(m) for m in MessageBodies]
+        result = self.client.send_message_batch(QueueUrl=QueueUrl, Entries=messages, **kwargs)
+        logger.debug("Messages bulk sent successfully to SQS", queue_url=QueueUrl, count=len(MessageBodies), response=result)
+        return result
+
+    def delete_message(self, QueueUrl, ReceiptHandle, **kwargs):
+        logger.debug("Try to delete message", queue_url=QueueUrl, receipt_handle=ReceiptHandle)
+        result = self.client.delete_message(QueueUrl=QueueUrl, ReceiptHandle=ReceiptHandle, **kwargs)
+        logger.debug("Message deleted successfully", queue_url=QueueUrl, receipt_handle=ReceiptHandle, response=result)
+        return result

plexop_infrastructure-1.1.0/src/plexop/infrastructure/cache_manager.py

@@ -0,0 +1,33 @@
+from functools import wraps
+import json
+from plexop.infrastructure import config, logger
+from plexop.infrastructure.expiringdict import ExpiringDict
+
+cache_ttl_seconds = config.getint('settings', 'cache_ttl_seconds', fallback=600)
+cache_max_len = config.getint('settings', 'cache_max_len', fallback=1000)
+
+_memory = ExpiringDict(max_len=cache_max_len, max_age_seconds=cache_ttl_seconds)
+
+
+def cache(fn):
+    @wraps(fn)
+    def surround(self, url, *args, **kwargs):
+        
+        if kwargs.get('method', 'GET').upper() != 'GET' or not kwargs.get('use_cache', False):
+            return fn(self, url, *args, **kwargs)
+       
+        cache_url = url
+        if 'params' in kwargs:
+            cache_url += json.dumps(kwargs['params'])
+        if 'headers' in kwargs:
+            cache_url += json.dumps(kwargs['headers'])
+        result = _memory.get(cache_url)
+        if not result:
+            result = fn(self, url, *args, **kwargs)
+            str_result = str(result)
+            if 'error' not in str_result and 'timed' not in str_result:
+                _memory[cache_url] = result
+        else:
+            logger.debug("result found in cache", key=cache_url)
+        return result
+    return surround

plexop_infrastructure-1.1.0/src/plexop/infrastructure/config_provider.py

@@ -0,0 +1,69 @@
+import configparser
+import os
+from base64 import b64decode
+
+_config: configparser.ConfigParser = None
+
+
+def read_config_from(path):
+    config = configparser.ConfigParser(allow_no_value=True)
+    try:
+        config.read(path)
+        return config
+    except IOError:
+        pass
+    return config
+
+
+def read_config_from_env_variables():
+    for key, value in os.environ.items():
+        section_name, _, param_name = key.partition('__')
+        if param_name and section_name:
+            if not _config.has_section(section_name.lower()):
+                _config.add_section(section_name.lower())
+            _config.set(section_name.lower(), param_name.lower(), value)
+    decrypt_env_variables()
+
+
+def decrypt_env_variables():
+    encrypted_fields = _config.get('settings', 'encrypted_fields', fallback=None)
+    if encrypted_fields:
+        encrypted_fields = encrypted_fields.split(',')
+        import boto3
+        kms_client = boto3.client('kms')
+        for k in encrypted_fields:
+            section, _, param_name = k.lower().partition('__')
+            enc_value = _config.get(section, param_name)
+            decryption_result = kms_client.decrypt(CiphertextBlob=b64decode(enc_value), EncryptionContext={
+                'LambdaFunctionName': os.environ['AWS_LAMBDA_FUNCTION_NAME']})
+            dec_value = decryption_result['Plaintext'].decode('utf-8')
+            _config.set(section, param_name, dec_value)
+
+
+def load_configs_from_aws_secrets_manager():
+    secret_names = _config.get('settings', 'aws_secrets_names', fallback=None)
+    if secret_names:
+        import boto3
+        import json
+        aws_region = _config.get('settings', 'aws_secrets_region', fallback=None)
+        if not aws_region:
+            boto3.setup_default_session()
+            aws_region = boto3.DEFAULT_SESSION.region_name
+        client = boto3.client('secretsmanager', region_name=aws_region)
+        for secret_name in secret_names.split(','):
+            response = client.get_secret_value(SecretId=secret_name)
+            result = json.loads(response['SecretString'])
+            for complex_key, value in result.items():
+                section, key = complex_key.split('__')
+                if not _config.has_section(section):
+                    _config.add_section(section)
+                _config.set(section, key, value)
+
+
+def get_config():
+    global _config
+    if not _config:
+        _config = read_config_from(os.getenv('CONFIG_FILE_PATH', 'conf.cfg'))
+        read_config_from_env_variables()
+        load_configs_from_aws_secrets_manager()
+    return _config

plexop_infrastructure-1.1.0/src/plexop/infrastructure/dict_csv_reader.py

@@ -0,0 +1,11 @@
+import csv
+
+
+def read_file(csv_file_path, delimiter='\t'):
+    results = []
+    with open(csv_file_path) as csv_file:
+        reader = csv.DictReader(csv_file, delimiter=delimiter)
+        for row in reader:
+            results.append(row)
+
+    return results