leadguru-jobs 0.702.0__tar.gz → 0.704.0__tar.gz

{leadguru_jobs-0.702.0 → leadguru_jobs-0.704.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: leadguru_jobs
-Version: 0.702.0
+Version: 0.704.0
 Summary: LGT jobs builds
 Author-email: developer@leadguru.co
 Classifier: Development Status :: 5 - Production/Stable

{leadguru_jobs-0.702.0 → leadguru_jobs-0.704.0}/leadguru_jobs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: leadguru_jobs
-Version: 0.702.0
+Version: 0.704.0
 Summary: LGT jobs builds
 Author-email: developer@leadguru.co
 Classifier: Development Status :: 5 - Production/Stable

{leadguru_jobs-0.702.0 → leadguru_jobs-0.704.0}/lgt_jobs/env.py

@@ -2,7 +2,7 @@ import os
 
 project_id = os.environ.get('PROJECT_ID')
 background_jobs_topic = os.environ.get('BACKGROUND_JOBS_TOPIC')
-background_jobs_subscriber = os.environ.get('BACKGROUND_JOBS_SUBSCRIBER')
+background_jobs_subscriber = os.environ.get('BACKGROUND_JOBS_SUBSCRIBER', 'subscriber')
 mongo_connection_string = os.environ.get('MONGO_CONNECTION_STRING')
 smtp_host = os.environ.get('SMTP_HOST')
 smtp_login = os.environ.get('SMTP_LOGIN')
@@ -14,3 +14,5 @@ service_account_password = os.environ.get('SERVICE_ACCOUNT_PASSWORD')
 v2_server_host = os.environ.get('V2_SERVER_HOST')
 v3_server_host = os.environ.get('V3_SERVER_HOST')
 v1_server_host = os.environ.get('V1_SERVER_HOST')
+max_messages = int(os.environ.get('MAX_MESSAGES', 10))
+pool_size = int(os.environ.get('POOL_SIZE', 10))

{leadguru_jobs-0.702.0 → leadguru_jobs-0.704.0}/lgt_jobs/jobs/chat_history.py

@@ -33,9 +33,6 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
 
     def exec(self, data: LoadChatHistoryJobData):
         user = UserMongoRepository().get(data.user_id)
-        if user.subscription_expired_at.replace(tzinfo=UTC) < datetime.now(UTC):
-            log.info(f"[LoadChatHistoryJob]: {user.email} has expired subscription")
-            return
         bots = DedicatedBotRepository().get_all(only_valid=True, user_id=user.id, source_type=SourceType.SLACK)
         if not bots:
             return
@@ -47,7 +44,6 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
             if not contacts:
                 continue
 
-            log.info(f"[LoadChatHistoryJob]: processing {len(contacts)} contacts for user: {user.email}")
             for contact in contacts:
                 message = self._update_history(contact=contact, bot=bot)
 
@@ -93,7 +89,7 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
 
         messages = history.get('messages', [])
         if not messages:
-            return None
+            return
 
         messages = [ChatMessage.from_slack_response(bot, m, contact.sender_id) for m in messages]
         new_messages = self._get_new_messages(contact, bot, messages)
@@ -106,4 +102,4 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
             self.user_leads_repo.update_many_by_sender_id(contact.sender_id, last_action_at=now)
             return new_messages[-1]
 
-        return None
+        return

leadguru_jobs-0.704.0/lgt_jobs/main.py

@@ -0,0 +1,53 @@
+import json
+import traceback
+
+from concurrent.futures import ThreadPoolExecutor
+
+from lgt_common.pubsub.pubsubfactory import PubSubFactory
+import logging as log
+from lgt_jobs.env import project_id, background_jobs_topic, background_jobs_subscriber
+from lgt_jobs.runner import BackgroundJobRunner
+from lgt_jobs import jobs_map, env
+import google.cloud.logging
+from google.cloud.pubsub_v1 import SubscriberClient
+from google.cloud.pubsub_v1.types import FlowControl
+from google.cloud.pubsub_v1.subscriber.scheduler import ThreadScheduler
+
+
+def callback(message):
+    try:
+        data = json.loads(message.data)
+        log.info(f"[JOB]: {data} [START]")
+        BackgroundJobRunner.run(jobs_map=jobs_map, data=data)
+        log.info(f"[JOB]: {data} [FINISHED]")
+    except Exception as exception:
+        log.error(f"[ERROR][JOB]: {message.data} [ERROR] {traceback.format_exception(exception)} ")
+        traceback.print_exception(exception)
+    finally:
+        # accept message any way
+        message.ack()
+
+
+if __name__ == '__main__':
+    client = google.cloud.logging.Client()
+    client.setup_logging()
+    factory = PubSubFactory(project_id)
+    factory.create_topic_if_doesnt_exist(background_jobs_topic)
+    factory.create_subscription_if_doesnt_exist(background_jobs_subscriber, background_jobs_topic, 600)
+    subscription_path = factory.get_subscription_path(background_jobs_subscriber, background_jobs_topic)
+    # Launching a subscription
+    future = SubscriberClient().subscribe(
+        subscription_path,
+        callback=callback,
+        flow_control=FlowControl(max_messages=env.max_messages),
+        scheduler=ThreadScheduler(executor=ThreadPoolExecutor(max_workers=env.pool_size))
+    )
+
+    log.info(f"Subscription to {subscription_path} started. We are waiting for a message..\n")
+
+    try:
+        future.result()  # Block the current thread until an exception occurs
+    except Exception as exception:
+        log.error(f"[ERROR][SUBSCRIBER]: {traceback.format_exception(exception)}")
+        traceback.print_exception(exception)
+        future.cancel()

leadguru_jobs-0.702.0/lgt_jobs/main.py

@@ -1,47 +0,0 @@
-import json
-import sys
-import threading
-import time
-from lgt_common.pubsub.pubsubfactory import PubSubFactory
-import logging as log
-from lgt_jobs.env import project_id, background_jobs_topic, background_jobs_subscriber
-from lgt_jobs.runner import BackgroundJobRunner
-from lgt_jobs import jobs_map
-import google.cloud.logging
-
-lock = threading.Lock()
-
-
-def run_background_job(data):
-    try:
-        log.info(f"[JOB]: {data} [START]")
-        BackgroundJobRunner.run(jobs_map=jobs_map, data=data)
-        log.info(f"[JOB]: {data} [FINISHED]")
-    except Exception:
-        raise
-
-
-def run_background_job_with_lock(message):
-    try:
-        data = json.loads(message.data)
-        with lock:
-            run_background_job(data)
-    except:
-        import traceback
-        log.error(f"[ERROR][JOB]: {message.data} [ERROR] {traceback.format_exception(*sys.exc_info())} ")
-        traceback.print_exception(*sys.exc_info())
-    finally:
-        # accept message any way
-        message.ack()
-
-
-if __name__ == '__main__':
-    client = google.cloud.logging.Client()
-    client.setup_logging()
-    factory = PubSubFactory(project_id)
-    factory.create_topic_if_doesnt_exist(background_jobs_topic)
-    factory.create_subscription_if_doesnt_exist(background_jobs_subscriber, background_jobs_topic, 600)
-    bot_subscription_path = factory.get_subscription_path(background_jobs_subscriber, background_jobs_topic)
-    factory.subscriber.subscribe(bot_subscription_path, callback=run_background_job_with_lock)
-    while True:
-        time.sleep(1)