airless-google-cloud-core 0.0.1.dev1__tar.gz

airless_google_cloud_core-0.0.1.dev1/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: airless-google-cloud-core
+Version: 0.0.1.dev1
+Summary: Airless package with the main components to use airless in google cloud
+Project-URL: Homepage, https://github.com/astercapital/airless
+Project-URL: Bug Tracker, https://github.com/astercapital/airless/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: google-cloud-pubsub==2.13.11
+Requires-Dist: airless-core>=0.1.0.dev1
+
+# airless-google-cloud-core
+
+[![PyPI version](https://badge.fury.io/py/airless-google-cloud-core.svg)](https://badge.fury.io/py/airless-google-cloud-core)
+
+airless-google-cloud-core [Airless](https://github.com/astercapital/airless) package with the main components to use airless in google cloud.

airless_google_cloud_core-0.0.1.dev1/README.md

@@ -0,0 +1,5 @@
+# airless-google-cloud-core
+
+[![PyPI version](https://badge.fury.io/py/airless-google-cloud-core.svg)](https://badge.fury.io/py/airless-google-cloud-core)
+
+airless-google-cloud-core [Airless](https://github.com/astercapital/airless) package with the main components to use airless in google cloud.

airless_google_cloud_core-0.0.1.dev1/airless/google/cloud/core/operator/base.py

@@ -0,0 +1,18 @@
+
+from airless.core.operator.base import BaseFileOperator, BaseEventOperator
+
+from airless.google.cloud.pubsub.hook import GooglePubsubHook
+
+
+class BaseGoogleFileOperator(BaseFileOperator):
+
+    def __init__(self):
+        super().__init__()
+        self.queue_hook = GooglePubsubHook()  # Have to redefine this attribute for each vendor
+
+
+class BaseGoogleEventOperator(BaseEventOperator):
+
+    def __init__(self):
+        super().__init__()
+        self.queue_hook = GooglePubsubHook()  # Have to redefine this attribute for each vendor

airless_google_cloud_core-0.0.1.dev1/airless/google/cloud/core/operator/delay.py

@@ -0,0 +1,20 @@
+
+from time import sleep
+
+from airless.core.operator.delay import DelayOperator
+from airless.google.cloud.core.base import BaseGoogleEventOperator
+
+
+
+class GoogleDelayOperator(BaseGoogleEventOperator, DelayOperator):
+
+    """
+    Operator that adds a delay to the pipeline. The maximum delay is 500 due
+    to Cloud Functions time constraint of 540 of execution time
+
+    It can receive 1 parameter:
+    seconds: number of seconds to wait
+    """
+
+    def __init__(self):
+        super().__init__()

airless_google_cloud_core-0.0.1.dev1/airless/google/cloud/core/operator/error.py

@@ -0,0 +1,95 @@
+
+import json
+import time
+
+from datetime import datetime
+
+from airless.core.config import get_config
+from airless.core.dto.base import BaseDto
+from airless.google.cloud.core.base import BaseGoogleEventOperator
+
+
+class GoogleErrorReprocessOperator(BaseGoogleEventOperator):
+
+    def __init__(self):
+        super().__init__()
+
+    def execute(self, data, topic):
+
+        input_type = data['input_type']
+        origin = data.get('origin', 'undefined')
+        message_id = data.get('event_id')
+        original_data = data['data']
+        metadata = original_data.get('metadata', {})
+
+        retry_interval = metadata.get('retry_interval', 5)
+        retries = metadata.get('retries', 0)
+        max_retries = metadata.get('max_retries', 2)
+        max_interval = metadata.get('max_interval', 480)  # Cloud function max execution time is 540s (9 min), so set it to wait max 480s (8 min)
+
+        if (input_type == 'event') and (retries < max_retries):
+            time.sleep(min(retry_interval ** retries, max_interval))
+            original_data.setdefault('metadata', {})['retries'] = retries + 1
+            self.queue_hook.publish(
+                project=get_config('GCP_PROJECT'),
+                topic=origin,
+                data=original_data)
+
+        else:
+            dto = BaseDto(
+                event_id=message_id,
+                resource=origin,
+                to_project=get_config('GCP_PROJECT'),
+                to_dataset=get_config('BIGQUERY_DATASET_ERROR'),
+                to_table=get_config('BIGQUERY_TABLE_ERROR'),
+                to_schema=None,
+                to_partition_column='_created_at',
+                to_extract_to_cols=False,
+                to_keys_format=None,
+                data=data)
+            self.queue_hook.publish(
+                project=get_config('GCP_PROJECT'),
+                topic=get_config('PUBSUB_TOPIC_PUBSUB_TO_BQ'),
+                data=dto.as_dict())
+
+            email_send_topic = get_config('PUBSUB_TOPIC_EMAIL_SEND', False)
+            if email_send_topic and (origin != email_send_topic):
+                self.notify_email(origin, message_id, data)
+
+            slack_send_topic = get_config('PUBSUB_TOPIC_SLACK_SEND', False)
+            if slack_send_topic and (origin != slack_send_topic):
+                self.notify_slack(origin, message_id, data)
+
+    def notify_email(self, origin, message_id, data):
+        email_message = {
+            'sender': get_config('EMAIL_SENDER_ERROR'),
+            'recipients': eval(get_config('EMAIL_RECIPIENTS_ERROR')),
+            'subject': f'{origin} | {message_id}',
+            'content': f'Input Type: {data["input_type"]} Origin: {origin}\nMessage ID: {message_id}\n\n {json.dumps(data["data"])}\n\n{data["error"]}'
+        }
+        self.queue_hook.publish(
+            project=get_config('GCP_PROJECT'),
+            topic=get_config('PUBSUB_TOPIC_EMAIL_SEND'),
+            data=email_message)
+
+    def notify_slack(self, origin, message_id, data):
+        slack_message = {
+            'channels': eval(get_config('SLACK_CHANNELS_ERROR')),
+            'message': f'{origin} | {message_id}\n\n{json.dumps(data["data"])}\n\n{data["error"]}'
+        }
+        self.queue_hook.publish(
+            project=get_config('GCP_PROJECT'),
+            topic=get_config('PUBSUB_TOPIC_SLACK_SEND'),
+            data=slack_message)
+
+    def prepare_row(self, row, message_id, origin):
+        return {
+            '_event_id': message_id or 1234,
+            '_resource': origin or 'local',
+            '_json': json.dumps(row),
+            '_created_at': str(datetime.now())
+        }
+
+    def prepare_rows(self, data, message_id, origin):
+        prepared_rows = data if isinstance(data, list) else [data]
+        return [self.prepare_row(row, message_id, origin) for row in prepared_rows]

airless_google_cloud_core-0.0.1.dev1/airless/google/cloud/core/operator/redirect.py

@@ -0,0 +1,22 @@
+
+from airless.core.operator.redirect import RedirectOperator
+from airless.google.cloud.core.base import BaseGoogleEventOperator
+
+
+class GoogleRedirectOperator(BaseGoogleEventOperator, RedirectOperator):
+
+    """
+    Operator that receives one event from a pubsub topic
+    and publish multiple messages to another topic.
+
+    It can receive 4 parameters:
+    project: the project where the destination pubsub is hosted
+    topic: the pubsub topic it must publish the newly generated messages
+    messages: a list of messages to publish the topic
+    params: a list of dicts containing a key and a list of values
+
+    The output messages will be the product of messages and every param values list
+    """
+
+    def __init__(self):
+        super().__init__()

airless_google_cloud_core-0.0.1.dev1/airless/google/cloud/pubsub/hook/pubsub.py

@@ -0,0 +1,30 @@
+
+import json
+
+from google.cloud import pubsub_v1
+
+from airless.core.hook.queue import QueueHook
+from airless.core.config import get_config
+
+
+class GooglePubsubHook(QueueHook):
+
+    def __init__(self):
+        super().__init__()
+        self.publisher = pubsub_v1.PublisherClient()
+
+    def publish(self, project, topic, data):
+
+        if get_config('ENV') == 'prod':
+            topic_path = self.publisher.topic_path(project, topic)
+
+            message_bytes = json \
+                .dumps(data) \
+                .encode('utf-8')
+
+            publish_future = self.publisher.publish(topic_path, data=message_bytes)
+            publish_future.result(timeout=10)
+            self.logger.info(f'published to {project}.{topic}')
+            return 'Message published.'
+        else:
+            self.logger.debug(f'[DEV] Message published to Project {project}, Topic {topic}: {data}')

airless_google_cloud_core-0.0.1.dev1/airless_google_cloud_core.egg-info/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: airless-google-cloud-core
+Version: 0.0.1.dev1
+Summary: Airless package with the main components to use airless in google cloud
+Project-URL: Homepage, https://github.com/astercapital/airless
+Project-URL: Bug Tracker, https://github.com/astercapital/airless/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: google-cloud-pubsub==2.13.11
+Requires-Dist: airless-core>=0.1.0.dev1
+
+# airless-google-cloud-core
+
+[![PyPI version](https://badge.fury.io/py/airless-google-cloud-core.svg)](https://badge.fury.io/py/airless-google-cloud-core)
+
+airless-google-cloud-core [Airless](https://github.com/astercapital/airless) package with the main components to use airless in google cloud.

airless_google_cloud_core-0.0.1.dev1/airless_google_cloud_core.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+pyproject.toml
+setup.cfg
+airless/google/cloud/__init__.py
+airless/google/cloud/core/__init__.py
+airless/google/cloud/core/operator/__init__.py
+airless/google/cloud/core/operator/base.py
+airless/google/cloud/core/operator/delay.py
+airless/google/cloud/core/operator/error.py
+airless/google/cloud/core/operator/redirect.py
+airless/google/cloud/pubsub/__init__.py
+airless/google/cloud/pubsub/hook/__init__.py
+airless/google/cloud/pubsub/hook/pubsub.py
+airless_google_cloud_core.egg-info/PKG-INFO
+airless_google_cloud_core.egg-info/SOURCES.txt
+airless_google_cloud_core.egg-info/dependency_links.txt
+airless_google_cloud_core.egg-info/requires.txt
+airless_google_cloud_core.egg-info/top_level.txt

airless_google_cloud_core-0.0.1.dev1/airless_google_cloud_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

airless_google_cloud_core-0.0.1.dev1/airless_google_cloud_core.egg-info/requires.txt

@@ -0,0 +1,2 @@
+google-cloud-pubsub==2.13.11
+airless-core>=0.1.0.dev1

airless_google_cloud_core-0.0.1.dev1/airless_google_cloud_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+airless

airless_google_cloud_core-0.0.1.dev1/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "airless-google-cloud-core"
+version = "0.0.1.dev1"
+description = "Airless package with the main components to use airless in google cloud"
+readme = "README.md"
+dynamic = ["dependencies", "requires-python"]
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project.urls]
+"Homepage" = "https://github.com/astercapital/airless"
+"Bug Tracker" = "https://github.com/astercapital/airless/issues"
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "--capture=sys --cache-clear --disable-warnings --junitxml=pytest.xml --cov-report=xml:cov.xml --cov-report=term-missing:skip-covered --cov=airless"
+testpaths = ["tests"]

airless_google_cloud_core-0.0.1.dev1/setup.cfg

@@ -0,0 +1,32 @@
+[metadata]
+name = airless-google-cloud-core
+version = 0.0.0
+
+[options]
+python_requires = >=3.9
+packages = find_namespace:
+install_requires = 
+	google-cloud-pubsub==2.13.11
+	airless-core>=0.1.0.dev1
+
+[options.packages.find]
+include = airless.google.cloud*
+
+[flake8]
+exclude = 
+	.git,
+	__pycache__,
+	build,
+	dist,
+	*.pyc,
+	*.egg-info,
+	*.egg,
+	temp
+ignore = E501
+max-line-length = 120
+max-complexity = 15
+
+[egg_info]
+tag_build = 
+tag_date = 0
+