airless-google-cloud-core 0.0.1.dev1__py3-none-any.whl

airless/google/cloud/core/operator/base.py

@@ -0,0 +1,18 @@
+
+from airless.core.operator.base import BaseFileOperator, BaseEventOperator
+
+from airless.google.cloud.pubsub.hook import GooglePubsubHook
+
+
+class BaseGoogleFileOperator(BaseFileOperator):
+
+    def __init__(self):
+        super().__init__()
+        self.queue_hook = GooglePubsubHook()  # Have to redefine this attribute for each vendor
+
+
+class BaseGoogleEventOperator(BaseEventOperator):
+
+    def __init__(self):
+        super().__init__()
+        self.queue_hook = GooglePubsubHook()  # Have to redefine this attribute for each vendor

airless/google/cloud/core/operator/delay.py

@@ -0,0 +1,20 @@
+
+from time import sleep
+
+from airless.core.operator.delay import DelayOperator
+from airless.google.cloud.core.base import BaseGoogleEventOperator
+
+
+
+class GoogleDelayOperator(BaseGoogleEventOperator, DelayOperator):
+
+    """
+    Operator that adds a delay to the pipeline. The maximum delay is 500 due
+    to Cloud Functions time constraint of 540 of execution time
+
+    It can receive 1 parameter:
+    seconds: number of seconds to wait
+    """
+
+    def __init__(self):
+        super().__init__()

airless/google/cloud/core/operator/error.py

@@ -0,0 +1,95 @@
+
+import json
+import time
+
+from datetime import datetime
+
+from airless.core.config import get_config
+from airless.core.dto.base import BaseDto
+from airless.google.cloud.core.base import BaseGoogleEventOperator
+
+
+class GoogleErrorReprocessOperator(BaseGoogleEventOperator):
+
+    def __init__(self):
+        super().__init__()
+
+    def execute(self, data, topic):
+
+        input_type = data['input_type']
+        origin = data.get('origin', 'undefined')
+        message_id = data.get('event_id')
+        original_data = data['data']
+        metadata = original_data.get('metadata', {})
+
+        retry_interval = metadata.get('retry_interval', 5)
+        retries = metadata.get('retries', 0)
+        max_retries = metadata.get('max_retries', 2)
+        max_interval = metadata.get('max_interval', 480)  # Cloud function max execution time is 540s (9 min), so set it to wait max 480s (8 min)
+
+        if (input_type == 'event') and (retries < max_retries):
+            time.sleep(min(retry_interval ** retries, max_interval))
+            original_data.setdefault('metadata', {})['retries'] = retries + 1
+            self.queue_hook.publish(
+                project=get_config('GCP_PROJECT'),
+                topic=origin,
+                data=original_data)
+
+        else:
+            dto = BaseDto(
+                event_id=message_id,
+                resource=origin,
+                to_project=get_config('GCP_PROJECT'),
+                to_dataset=get_config('BIGQUERY_DATASET_ERROR'),
+                to_table=get_config('BIGQUERY_TABLE_ERROR'),
+                to_schema=None,
+                to_partition_column='_created_at',
+                to_extract_to_cols=False,
+                to_keys_format=None,
+                data=data)
+            self.queue_hook.publish(
+                project=get_config('GCP_PROJECT'),
+                topic=get_config('PUBSUB_TOPIC_PUBSUB_TO_BQ'),
+                data=dto.as_dict())
+
+            email_send_topic = get_config('PUBSUB_TOPIC_EMAIL_SEND', False)
+            if email_send_topic and (origin != email_send_topic):
+                self.notify_email(origin, message_id, data)
+
+            slack_send_topic = get_config('PUBSUB_TOPIC_SLACK_SEND', False)
+            if slack_send_topic and (origin != slack_send_topic):
+                self.notify_slack(origin, message_id, data)
+
+    def notify_email(self, origin, message_id, data):
+        email_message = {
+            'sender': get_config('EMAIL_SENDER_ERROR'),
+            'recipients': eval(get_config('EMAIL_RECIPIENTS_ERROR')),
+            'subject': f'{origin} | {message_id}',
+            'content': f'Input Type: {data["input_type"]} Origin: {origin}\nMessage ID: {message_id}\n\n {json.dumps(data["data"])}\n\n{data["error"]}'
+        }
+        self.queue_hook.publish(
+            project=get_config('GCP_PROJECT'),
+            topic=get_config('PUBSUB_TOPIC_EMAIL_SEND'),
+            data=email_message)
+
+    def notify_slack(self, origin, message_id, data):
+        slack_message = {
+            'channels': eval(get_config('SLACK_CHANNELS_ERROR')),
+            'message': f'{origin} | {message_id}\n\n{json.dumps(data["data"])}\n\n{data["error"]}'
+        }
+        self.queue_hook.publish(
+            project=get_config('GCP_PROJECT'),
+            topic=get_config('PUBSUB_TOPIC_SLACK_SEND'),
+            data=slack_message)
+
+    def prepare_row(self, row, message_id, origin):
+        return {
+            '_event_id': message_id or 1234,
+            '_resource': origin or 'local',
+            '_json': json.dumps(row),
+            '_created_at': str(datetime.now())
+        }
+
+    def prepare_rows(self, data, message_id, origin):
+        prepared_rows = data if isinstance(data, list) else [data]
+        return [self.prepare_row(row, message_id, origin) for row in prepared_rows]

airless/google/cloud/core/operator/redirect.py

@@ -0,0 +1,22 @@
+
+from airless.core.operator.redirect import RedirectOperator
+from airless.google.cloud.core.base import BaseGoogleEventOperator
+
+
+class GoogleRedirectOperator(BaseGoogleEventOperator, RedirectOperator):
+
+    """
+    Operator that receives one event from a pubsub topic
+    and publish multiple messages to another topic.
+
+    It can receive 4 parameters:
+    project: the project where the destination pubsub is hosted
+    topic: the pubsub topic it must publish the newly generated messages
+    messages: a list of messages to publish the topic
+    params: a list of dicts containing a key and a list of values
+
+    The output messages will be the product of messages and every param values list
+    """
+
+    def __init__(self):
+        super().__init__()

airless/google/cloud/pubsub/hook/pubsub.py

@@ -0,0 +1,30 @@
+
+import json
+
+from google.cloud import pubsub_v1
+
+from airless.core.hook.queue import QueueHook
+from airless.core.config import get_config
+
+
+class GooglePubsubHook(QueueHook):
+
+    def __init__(self):
+        super().__init__()
+        self.publisher = pubsub_v1.PublisherClient()
+
+    def publish(self, project, topic, data):
+
+        if get_config('ENV') == 'prod':
+            topic_path = self.publisher.topic_path(project, topic)
+
+            message_bytes = json \
+                .dumps(data) \
+                .encode('utf-8')
+
+            publish_future = self.publisher.publish(topic_path, data=message_bytes)
+            publish_future.result(timeout=10)
+            self.logger.info(f'published to {project}.{topic}')
+            return 'Message published.'
+        else:
+            self.logger.debug(f'[DEV] Message published to Project {project}, Topic {topic}: {data}')

airless_google_cloud_core-0.0.1.dev1.dist-info/METADATA

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: airless-google-cloud-core
+Version: 0.0.1.dev1
+Summary: Airless package with the main components to use airless in google cloud
+Project-URL: Homepage, https://github.com/astercapital/airless
+Project-URL: Bug Tracker, https://github.com/astercapital/airless/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: google-cloud-pubsub==2.13.11
+Requires-Dist: airless-core>=0.1.0.dev1
+
+# airless-google-cloud-core
+
+[![PyPI version](https://badge.fury.io/py/airless-google-cloud-core.svg)](https://badge.fury.io/py/airless-google-cloud-core)
+
+airless-google-cloud-core [Airless](https://github.com/astercapital/airless) package with the main components to use airless in google cloud.

airless_google_cloud_core-0.0.1.dev1.dist-info/RECORD

@@ -0,0 +1,14 @@
+airless/google/cloud/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airless/google/cloud/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airless/google/cloud/core/operator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airless/google/cloud/core/operator/base.py,sha256=FxHj0MgDV8jEBRSZVtUS3J7SeegS-7eaUZmvDDETRJc,537
+airless/google/cloud/core/operator/delay.py,sha256=S_aG20JjgQ6NDjq85pyO5vv8XDn0NvP4PncvjzKpy6s,498
+airless/google/cloud/core/operator/error.py,sha256=nf5I1lsRj9_3EHHwPauxZNzYNuyepjgYJZ7Co6wP4bU,3801
+airless/google/cloud/core/operator/redirect.py,sha256=ftGMaQJg1E4qx8pU_5bHDkY_hkbyJt4nCQB3iuyP_-E,757
+airless/google/cloud/pubsub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airless/google/cloud/pubsub/hook/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airless/google/cloud/pubsub/hook/pubsub.py,sha256=r5b_aObasfnsiC8WfE5JtQ4xa1_LSDz2Mhy6Rc6YzxY,886
+airless_google_cloud_core-0.0.1.dev1.dist-info/METADATA,sha256=rOGhmrhIjqveQQkwCdozGx8aRkR2CowR0QIwtuTKVXw,737
+airless_google_cloud_core-0.0.1.dev1.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+airless_google_cloud_core-0.0.1.dev1.dist-info/top_level.txt,sha256=vPLzRICE7VNGvU-zceoZ1N7H2TkerQXR475zjg_xPzU,8
+airless_google_cloud_core-0.0.1.dev1.dist-info/RECORD,,

airless_google_cloud_core-0.0.1.dev1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (74.1.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

airless_google_cloud_core-0.0.1.dev1.dist-info/top_level.txt

@@ -0,0 +1 @@
+airless