leadguru-jobs 0.702.0__py3-none-any.whl → 0.703.0__py3-none-any.whl

{leadguru_jobs-0.702.0.dist-info → leadguru_jobs-0.703.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: leadguru_jobs
-Version: 0.702.0
+Version: 0.703.0
 Summary: LGT jobs builds
 Author-email: developer@leadguru.co
 Classifier: Development Status :: 5 - Production/Stable

{leadguru_jobs-0.702.0.dist-info → leadguru_jobs-0.703.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
 lgt_jobs/__init__.py,sha256=fQ97ZFeVtbOgODPdB26x7ku45AQPQ8ixeziiCAjNYZM,2451
 lgt_jobs/basejobs.py,sha256=LPEclvY6fIG17YMGy9pRK0Q58n62gkI5W1BTK6R2p-c,1234
-lgt_jobs/env.py,sha256=cRO03GGvstUjBsv3uYO-iakrOvAk_ZWUP_fnmf21iZQ,789
-lgt_jobs/main.py,sha256=cK_nkBtJHnUNDbba1WZotqPtI_6OWxiYQkAgco9OAmE,1539
+lgt_jobs/env.py,sha256=HHlpQgZrRQMMuPZYhP94A8b29T_10sKRo98mh0_hh3Y,907
+lgt_jobs/main.py,sha256=zd0TdwFBUXUmFfUw552rPt6Rd_uG-LWEE6wTs8TCxIk,2081
 lgt_jobs/runner.py,sha256=pS-Qwyfuk22aQ1Mk3p-Q0ysdZfKy1Bh9YoWBlDaLeM8,2234
 lgt_jobs/simple_job.py,sha256=EBxhre3749GEAzJyaV1GXYy3P0pzwHchxcZ2kSN1vWU,527
 lgt_jobs/smtp.py,sha256=k0qh-Qk9AeSU3WrCOVmuWkfpi9xzwaPMzbw9cED-pZw,1150
@@ -15,7 +15,7 @@ lgt_jobs/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgt_jobs/jobs/analytics.py,sha256=IIsWt4A1qUw3w-S-8W16uKY1FRVWfXXA41_mu4uCNiM,979
 lgt_jobs/jobs/bot_stats_update.py,sha256=3gq1Ilj_RHddYeSny_238ITs7YIfCf7d4BmfYU8PKlo,8177
 lgt_jobs/jobs/bots_killer.py,sha256=s2f7kal4vAMJd9YaMQ1X3FkCrLWCKNb-bWeI9RZ1Oco,506
-lgt_jobs/jobs/chat_history.py,sha256=rj337uHT2XmyDu3oMPbnCBxSsRn4roxDNS0hsbvoI7I,5040
+lgt_jobs/jobs/chat_history.py,sha256=H2ezntWSdtiE4KREL-RIHXKdDm2xs9PgEm-K8XKH9MA,4739
 lgt_jobs/jobs/connect_sources.py,sha256=vOKu2Q9cmQsj4VMVzOoEJkD1wzE9XurbIChx35rES8I,4957
 lgt_jobs/jobs/inbox_leads.py,sha256=S6sGXePmWDsT8FvtFOBi3BJALtaDlF6aaQZsHntzHm0,4958
 lgt_jobs/jobs/load_slack_people.py,sha256=SaDpp3JCNVKcyZ2WSJ9TyP5lUUZJnGlPUyWdRNzfMkU,4183
@@ -108,7 +108,7 @@ lgt_jobs/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 lgt_jobs/services/k8_manager.py,sha256=bXpka9psIQ5UJ6QG_e4xolmE_3gtmNrzzZeL03bJ62I,995
 lgt_jobs/services/web_client.py,sha256=oMyWJxwGeIe3f40fPT7xcisjDg3BhA3Ipf8dr1jVT-Y,1549
 lgt_jobs/templates/new_message.html,sha256=dZl8UmdAOOMq4nidvAgMFroSrTV7Pw0RWt2yLp_2idg,6989
-leadguru_jobs-0.702.0.dist-info/METADATA,sha256=KXsfGYtQ67w0SvI_5YpmqKCZFrG8z4hxX50_S72JDUs,1317
-leadguru_jobs-0.702.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-leadguru_jobs-0.702.0.dist-info/top_level.txt,sha256=rIuw1DqwbnZyeoarBSC-bYeGOhv9mZBs7_afl9q4_JI,9
-leadguru_jobs-0.702.0.dist-info/RECORD,,
+leadguru_jobs-0.703.0.dist-info/METADATA,sha256=31uspm0mO0bGWegRCtwa5F5_EthxKCK-rwJT-AmIXBQ,1317
+leadguru_jobs-0.703.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+leadguru_jobs-0.703.0.dist-info/top_level.txt,sha256=rIuw1DqwbnZyeoarBSC-bYeGOhv9mZBs7_afl9q4_JI,9
+leadguru_jobs-0.703.0.dist-info/RECORD,,

lgt_jobs/env.py

@@ -2,7 +2,7 @@ import os
 
 project_id = os.environ.get('PROJECT_ID')
 background_jobs_topic = os.environ.get('BACKGROUND_JOBS_TOPIC')
-background_jobs_subscriber = os.environ.get('BACKGROUND_JOBS_SUBSCRIBER')
+background_jobs_subscriber = os.environ.get('BACKGROUND_JOBS_SUBSCRIBER', 'subscriber')
 mongo_connection_string = os.environ.get('MONGO_CONNECTION_STRING')
 smtp_host = os.environ.get('SMTP_HOST')
 smtp_login = os.environ.get('SMTP_LOGIN')
@@ -14,3 +14,5 @@ service_account_password = os.environ.get('SERVICE_ACCOUNT_PASSWORD')
 v2_server_host = os.environ.get('V2_SERVER_HOST')
 v3_server_host = os.environ.get('V3_SERVER_HOST')
 v1_server_host = os.environ.get('V1_SERVER_HOST')
+max_messages = int(os.environ.get('MAX_MESSAGES', 10))
+pool_size = int(os.environ.get('POOL_SIZE', 10))

lgt_jobs/jobs/chat_history.py

@@ -33,9 +33,6 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
 
     def exec(self, data: LoadChatHistoryJobData):
         user = UserMongoRepository().get(data.user_id)
-        if user.subscription_expired_at.replace(tzinfo=UTC) < datetime.now(UTC):
-            log.info(f"[LoadChatHistoryJob]: {user.email} has expired subscription")
-            return
         bots = DedicatedBotRepository().get_all(only_valid=True, user_id=user.id, source_type=SourceType.SLACK)
         if not bots:
             return
@@ -47,7 +44,6 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
             if not contacts:
                 continue
 
-            log.info(f"[LoadChatHistoryJob]: processing {len(contacts)} contacts for user: {user.email}")
             for contact in contacts:
                 message = self._update_history(contact=contact, bot=bot)
 
@@ -93,7 +89,7 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
 
         messages = history.get('messages', [])
         if not messages:
-            return None
+            return
 
         messages = [ChatMessage.from_slack_response(bot, m, contact.sender_id) for m in messages]
         new_messages = self._get_new_messages(contact, bot, messages)
@@ -106,4 +102,4 @@ class LoadChatHistoryJob(BaseBackgroundJob, ABC):
             self.user_leads_repo.update_many_by_sender_id(contact.sender_id, last_action_at=now)
             return new_messages[-1]
 
-        return None
+        return

lgt_jobs/main.py

@@ -1,35 +1,28 @@
 import json
-import sys
-import threading
-import time
+import traceback
+
+from concurrent.futures import ThreadPoolExecutor
+
 from lgt_common.pubsub.pubsubfactory import PubSubFactory
 import logging as log
 from lgt_jobs.env import project_id, background_jobs_topic, background_jobs_subscriber
 from lgt_jobs.runner import BackgroundJobRunner
-from lgt_jobs import jobs_map
+from lgt_jobs import jobs_map, env
 import google.cloud.logging
-
-lock = threading.Lock()
+from google.cloud.pubsub_v1 import SubscriberClient
+from google.cloud.pubsub_v1.types import FlowControl
+from google.cloud.pubsub_v1.subscriber.scheduler import ThreadScheduler
 
 
-def run_background_job(data):
+def callback(message):
     try:
+        data = json.loads(message.data)
         log.info(f"[JOB]: {data} [START]")
         BackgroundJobRunner.run(jobs_map=jobs_map, data=data)
         log.info(f"[JOB]: {data} [FINISHED]")
-    except Exception:
-        raise
-
-
-def run_background_job_with_lock(message):
-    try:
-        data = json.loads(message.data)
-        with lock:
-            run_background_job(data)
-    except:
-        import traceback
-        log.error(f"[ERROR][JOB]: {message.data} [ERROR] {traceback.format_exception(*sys.exc_info())} ")
-        traceback.print_exception(*sys.exc_info())
+    except Exception as exception:
+        log.error(f"[ERROR][JOB]: {message.data} [ERROR] {traceback.format_exception(exception)} ")
+        traceback.print_exception(exception)
     finally:
         # accept message any way
         message.ack()
@@ -41,7 +34,20 @@ if __name__ == '__main__':
     factory = PubSubFactory(project_id)
     factory.create_topic_if_doesnt_exist(background_jobs_topic)
     factory.create_subscription_if_doesnt_exist(background_jobs_subscriber, background_jobs_topic, 600)
-    bot_subscription_path = factory.get_subscription_path(background_jobs_subscriber, background_jobs_topic)
-    factory.subscriber.subscribe(bot_subscription_path, callback=run_background_job_with_lock)
-    while True:
-        time.sleep(1)
+    subscription_path = factory.get_subscription_path(background_jobs_subscriber, background_jobs_topic)
+    # Launching a subscription
+    future = SubscriberClient().subscribe(
+        subscription_path,
+        callback=callback,
+        flow_control=FlowControl(max_messages=env.max_messages),
+        scheduler=ThreadScheduler(executor=ThreadPoolExecutor(max_workers=env.pool_size))
+    )
+
+    log.info(f"Subscription to {subscription_path} started. We are waiting for a message..\n")
+
+    try:
+        future.result()  # Block the current thread until an exception occurs
+    except Exception as exception:
+        log.error(f"[ERROR][SUBSCRIBER]: {traceback.format_exception(exception)}")
+        traceback.print_exception(exception)
+        future.cancel()