arpakitlib 1.7.93__py3-none-any.whl → 1.7.95__py3-none-any.whl

arpakitlib/ar_base_worker_util.py

@@ -6,11 +6,14 @@ import multiprocessing
 import threading
 from abc import ABC
 from datetime import timedelta
+from random import randint
 from typing import Any
+from uuid import uuid4
 
+from arpakitlib.ar_datetime_util import now_utc_dt
 from arpakitlib.ar_enumeration_util import Enumeration
 from arpakitlib.ar_func_util import is_async_function, is_sync_function
-from arpakitlib.ar_sleep_util import sync_safe_sleep, async_safe_sleep
+from arpakitlib.ar_sleep_util import sync_safe_sleep
 
 _ARPAKIT_LIB_MODULE_VERSION = "3.0"
 
@@ -21,15 +24,23 @@ class BaseWorker(ABC):
             *,
             timeout_after_run=timedelta(seconds=0.3),
             timeout_after_err_in_run=timedelta(seconds=1),
-            startup_funcs: list[Any] | None = None
+            startup_funcs: list[Any] | None = None,
+            worker_name: str | None = None
     ):
         if startup_funcs is None:
             startup_funcs = []
         self.startup_funcs = startup_funcs
-        self.worker_name = self.__class__.__name__
-        self._logger = logging.getLogger(self.worker_name)
-        self.timeout_after_run = timeout_after_run.total_seconds()
-        self.timeout_after_err_in_run = timeout_after_err_in_run.total_seconds()
+        if worker_name is None:
+            worker_name = self.__class__.__name__
+        self.worker_name = worker_name
+        self.worker_creation_dt = now_utc_dt()
+        self.worker_id = f"{str(uuid4()).replace(' - ', '')}_{randint(1000, 9999)}"
+        self.worker_fullname = (
+            f"{self.worker_name}_{self.worker_creation_dt.isoformat()}_{self.worker_id}"
+        )
+        self._logger = logging.getLogger(self.worker_fullname)
+        self.timeout_after_run = timeout_after_run
+        self.timeout_after_err_in_run = timeout_after_err_in_run
 
     def sync_run_startup_funcs(self):
         for startup_func in self.startup_funcs:
@@ -41,10 +52,10 @@ class BaseWorker(ABC):
                 raise TypeError("no sync and not async")
 
     def sync_on_startup(self):
-        pass
+        self.sync_run_startup_funcs()
 
     def sync_run(self):
-        self._logger.info("hello world")
+        self._logger.info(f"hello world, im {self.worker_fullname}")
 
     def sync_run_on_error(self, exception: BaseException, **kwargs):
         pass
@@ -66,8 +77,8 @@ class BaseWorker(ABC):
                 except BaseException as exception_:
                     self._logger.error("error in sync_run_on_error", exc_info=exception_)
                     raise exception_
-            if self.timeout_after_run is not None:
-                sync_safe_sleep(self.timeout_after_run)
+                sync_safe_sleep(self.timeout_after_err_in_run)
+            sync_safe_sleep(self.timeout_after_run)
 
     async def async_run_startup_funcs(self):
         for startup_func in self.startup_funcs:
@@ -79,10 +90,10 @@ class BaseWorker(ABC):
                 raise TypeError("no sync and not async")
 
     async def async_on_startup(self):
-        pass
+        await self.async_run_startup_funcs()
 
     async def async_run(self):
-        self._logger.info("hello world")
+        self._logger.info(f"hello world, im {self.worker_fullname}")
 
     async def async_run_on_error(self, exception: BaseException, **kwargs):
         pass
@@ -104,8 +115,8 @@ class BaseWorker(ABC):
                 except BaseException as exception_:
                     self._logger.error("error in async_run_on_error", exc_info=exception_)
                     raise exception_
-            if self.timeout_after_err_in_run is not None:
-                await async_safe_sleep(self.timeout_after_err_in_run)
+                sync_safe_sleep(self.timeout_after_err_in_run)
+            sync_safe_sleep(self.timeout_after_run)
 
 
 class SafeRunInBackgroundModes(Enumeration):
@@ -136,6 +147,15 @@ def safe_run_worker_in_background(*, worker: BaseWorker, mode: str) -> (
     return res
 
 
+def safe_run_workers_in_background(
+        *, workers: list[BaseWorker], mode: str
+) -> list[asyncio.Task] | list[threading.Thread] | list[multiprocessing.Process]:
+    res = []
+    for worker in workers:
+        res.append(safe_run_worker_in_background(worker=worker, mode=mode))
+    return res
+
+
 def __example():
     pass
 

arpakitlib/ar_operation_execution_util.py

@@ -31,7 +31,8 @@ def get_operation_for_execution(
         *,
         session: Session | None = None,
         sqlalchemy_db: SQLAlchemyDB | None = None,
-        filter_operation_types: list[str] | str | None = None
+        filter_operation_types: list[str] | str | None = None,
+        lock: bool = False
 ) -> OperationDBM | None:
     if isinstance(filter_operation_types, str):
         filter_operation_types = [filter_operation_types]
@@ -44,6 +45,10 @@ def get_operation_for_execution(
         )
         if filter_operation_types:
             query = query.filter(OperationDBM.type.in_(filter_operation_types))
+
+        if lock:
+            query = query.with_for_update()
+
         query = query.order_by(asc(OperationDBM.creation_dt))
         operation_dbm: OperationDBM | None = query.first()
         return operation_dbm
@@ -148,7 +153,9 @@ class BaseOperationExecutor:
             raise Exception("raise_fake_exception")
         return operation_dbm
 
-    def sync_safe_execute_operation(self, operation_dbm: OperationDBM) -> OperationDBM:
+    def sync_safe_execute_operation(
+            self, operation_dbm: OperationDBM, worker: OperationExecutorWorker
+    ) -> OperationDBM:
         self._logger.info(
             f"start "
             f"operation_dbm.id={operation_dbm.id}, "
@@ -158,10 +165,16 @@ class BaseOperationExecutor:
 
         with self.sql_alchemy_db.new_session() as session:
             operation_dbm: OperationDBM = get_operation_by_id(
-                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True
+                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True, lock=True
             )
             operation_dbm.execution_start_dt = now_utc_dt()
             operation_dbm.status = OperationDBM.Statuses.executing
+            operation_dbm.output_data = combine_dicts(
+                operation_dbm.output_data,
+                {
+                    worker.worker_fullname: True
+                }
+            )
             session.commit()
             session.refresh(operation_dbm)
 
@@ -181,7 +194,7 @@ class BaseOperationExecutor:
         with self.sql_alchemy_db.new_session() as session:
 
             operation_dbm: OperationDBM = get_operation_by_id(
-                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True
+                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True, lock=True
             )
             operation_dbm.execution_finish_dt = now_utc_dt()
             if exception:
@@ -231,7 +244,9 @@ class BaseOperationExecutor:
             raise Exception("raise_fake_exception")
         return operation_dbm
 
-    async def async_safe_execute_operation(self, operation_dbm: OperationDBM) -> OperationDBM:
+    async def async_safe_execute_operation(
+            self, operation_dbm: OperationDBM, worker: OperationExecutorWorker
+    ) -> OperationDBM:
         self._logger.info(
             f"start "
             f"operation_dbm.id={operation_dbm.id}, "
@@ -241,10 +256,16 @@ class BaseOperationExecutor:
 
         with self.sql_alchemy_db.new_session() as session:
             operation_dbm: OperationDBM = get_operation_by_id(
-                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True
+                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True, lock=True
             )
             operation_dbm.execution_start_dt = now_utc_dt()
             operation_dbm.status = OperationDBM.Statuses.executing
+            operation_dbm.output_data = combine_dicts(
+                operation_dbm.output_data,
+                {
+                    worker.worker_fullname: True
+                }
+            )
             session.commit()
             session.refresh(operation_dbm)
 
@@ -264,7 +285,7 @@ class BaseOperationExecutor:
         with self.sql_alchemy_db.new_session() as session:
 
             operation_dbm: OperationDBM = get_operation_by_id(
-                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True
+                session=session, filter_operation_id=operation_dbm.id, raise_if_not_found=True, lock=True
             )
             operation_dbm.execution_finish_dt = now_utc_dt()
             if exception:
@@ -315,19 +336,15 @@ class OperationExecutorWorker(BaseWorker):
             sqlalchemy_db: SQLAlchemyDB,
             operation_executor: BaseOperationExecutor | None = None,
             filter_operation_types: str | list[str] | None = None,
-            timeout_after_run=timedelta(seconds=0.3),
-            timeout_after_err_in_run=timedelta(seconds=1),
             startup_funcs: list[Any] | None = None
     ):
-        super().__init__(
-            timeout_after_run=timeout_after_run,
-            timeout_after_err_in_run=timeout_after_err_in_run,
-            startup_funcs=startup_funcs
-        )
+        super().__init__(startup_funcs=startup_funcs)
         self.sqlalchemy_db = sqlalchemy_db
         if operation_executor is None:
             operation_executor = BaseOperationExecutor(sqlalchemy_db=sqlalchemy_db)
         self.operation_executor = operation_executor
+        if isinstance(filter_operation_types, str):
+            filter_operation_types = [filter_operation_types]
         self.filter_operation_types = filter_operation_types
 
     def sync_on_startup(self):
@@ -335,12 +352,13 @@ class OperationExecutorWorker(BaseWorker):
         self.sync_run_startup_funcs()
 
     def sync_execute_operation(self, operation_dbm: OperationDBM) -> OperationDBM:
-        return self.operation_executor.sync_safe_execute_operation(operation_dbm=operation_dbm)
+        return self.operation_executor.sync_safe_execute_operation(operation_dbm=operation_dbm, worker=self)
 
     def sync_run(self):
         operation_dbm: OperationDBM | None = get_operation_for_execution(
             sqlalchemy_db=self.sqlalchemy_db,
-            filter_operation_types=self.filter_operation_types
+            filter_operation_types=self.filter_operation_types,
+            lock=True
         )
         if not operation_dbm:
             return
@@ -354,12 +372,13 @@ class OperationExecutorWorker(BaseWorker):
         await self.async_run_startup_funcs()
 
     async def async_execute_operation(self, operation_dbm: OperationDBM) -> OperationDBM:
-        return await self.operation_executor.async_safe_execute_operation(operation_dbm=operation_dbm)
+        return await self.operation_executor.async_safe_execute_operation(operation_dbm=operation_dbm, worker=self)
 
     async def async_run(self):
         operation_dbm: OperationDBM | None = get_operation_for_execution(
             sqlalchemy_db=self.sqlalchemy_db,
-            filter_operation_types=self.filter_operation_types
+            filter_operation_types=self.filter_operation_types,
+            lock=True
         )
         if not operation_dbm:
             return
@@ -383,19 +402,15 @@ class ScheduledOperationCreatorWorker(BaseWorker):
             self,
             *,
             sqlalchemy_db: SQLAlchemyDB,
-            scheduled_operations: list[ScheduledOperation] | None = None,
-            timeout_after_run=timedelta(seconds=0.3),
-            timeout_after_err_in_run=timedelta(seconds=1),
+            scheduled_operations: ScheduledOperation | list[ScheduledOperation] | None = None,
             startup_funcs: list[Any] | None = None
     ):
-        super().__init__(
-            timeout_after_run=timeout_after_run,
-            timeout_after_err_in_run=timeout_after_err_in_run,
-            startup_funcs=startup_funcs
-        )
+        super().__init__(startup_funcs=startup_funcs)
         self.sqlalchemy_db = sqlalchemy_db
         if scheduled_operations is None:
             scheduled_operations = []
+        if isinstance(scheduled_operations, ScheduledOperation):
+            scheduled_operations = [scheduled_operations]
         self.scheduled_operations = scheduled_operations
 
     def sync_on_startup(self):

{arpakitlib-1.7.93.dist-info → arpakitlib-1.7.95.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arpakitlib
-Version: 1.7.93
+Version: 1.7.95
 Summary: arpakitlib
 Home-page: https://github.com/ARPAKIT-Company/arpakitlib
 License: Apache-2.0

{arpakitlib-1.7.93.dist-info → arpakitlib-1.7.95.dist-info}/RECORD

@@ -120,7 +120,7 @@ arpakitlib/ar_arpakit_project_template_util.py,sha256=AswzQvvb-zfUyrcP4EP0K756YL
 arpakitlib/ar_arpakit_schedule_uust_api_client_util.py,sha256=jGbP6egs2yhgfheyqhM0J-SeM2qp2YrW7dV-u9djv4Q,19223
 arpakitlib/ar_arpakitlib_cli_util.py,sha256=8lhEDxnwMSRX2PGV2xQtQru1AYKSA92SVolol5u7iBk,3154
 arpakitlib/ar_base64_util.py,sha256=aZkg2cZTuAaP2IWeG_LXJ6RO7qhyskVwec-Lks0iM-k,676
-arpakitlib/ar_base_worker_util.py,sha256=3pK03ps_AwtdDJofA8mRIGqlEDR1mfWd_QsSS3IR6sM,4828
+arpakitlib/ar_base_worker_util.py,sha256=hV_bMfkO897s00VGMAQ-z8GFzq-vEcRW5TKcmeBhbYM,5667
 arpakitlib/ar_cache_file_util.py,sha256=Fo2pH-Zqm966KWFBHG_pbiySGZvhIFCYqy7k1weRfJ0,3476
 arpakitlib/ar_datetime_util.py,sha256=Xe1NiT9oPQzNSG7RVRkhukhbg4i-hhS5ImmV7sPUc8o,971
 arpakitlib/ar_dict_util.py,sha256=cF5LQJ6tLqyGoEXfDljMDZrikeZoWPw7CgINHIFGvXM,419
@@ -164,7 +164,7 @@ arpakitlib/ar_logging_util.py,sha256=mx3H6CzX9dsh29ruFmYnva8lL6mwvdBXmeHH9E2tvu8
 arpakitlib/ar_mongodb_util.py,sha256=2ECkTnGAZ92qxioL-fmN6R4yZOSr3bXdXLWTzT1C3vk,4038
 arpakitlib/ar_need_type_util.py,sha256=GETiREPMEYhch-yU6T--Bdawlbb04Jp1Qy7cOsUlIeA,2228
 arpakitlib/ar_openai_api_client_util.py,sha256=_XmlApvHFMSyjvZydPa_kASIt9LsFrZmSC7YEzIG8Bg,1806
-arpakitlib/ar_operation_execution_util.py,sha256=D0pUb0hJ7VRsBrq1tiW5otPJMSqZdPDQLUnlPJxkAL0,18242
+arpakitlib/ar_operation_execution_util.py,sha256=pSOpYDDT7HE0MgoW3eolJ2bI4eIYI8TUy0jlulmcRHQ,18709
 arpakitlib/ar_parse_command.py,sha256=-s61xcATIsfw1eV_iD3xi-grsitbGzSDoAFc5V0OFy4,3447
 arpakitlib/ar_postgresql_util.py,sha256=1AuLjEaa1Lg4pzn-ukCVnDi35Eg1k91APRTqZhIJAdo,945
 arpakitlib/ar_run_cmd_util.py,sha256=D_rPavKMmWkQtwvZFz-Io5Ak8eSODHkcFeLPzNVC68g,1072
@@ -178,9 +178,9 @@ arpakitlib/ar_str_util.py,sha256=tFoGSDYoGpfdVHWor5Li9pEOFmDFlHkX-Z8iOy1LK7Y,353
 arpakitlib/ar_type_util.py,sha256=s0NsTM7mV3HuwyRwyYLdNn7Ep2HbyI4FIr-dd8x0lfI,3734
 arpakitlib/ar_yookassa_api_client_util.py,sha256=sh4fcUkAkdOetFn9JYoTvjcSXP-M1wU04KEY-ECLfLg,5137
 arpakitlib/ar_zabbix_api_client_util.py,sha256=Q-VR4MvoZ9aHwZeYZr9G3LwN-ANx1T5KFmF6pvPM-9M,6402
-arpakitlib-1.7.93.dist-info/LICENSE,sha256=GPEDQMam2r7FSTYqM1mm7aKnxLaWcBotH7UvQtea-ec,11355
-arpakitlib-1.7.93.dist-info/METADATA,sha256=pZPltqRitxj6pZ7I8pHAauB5P2sUINz3NUjA8ZGSnBY,2824
-arpakitlib-1.7.93.dist-info/NOTICE,sha256=95aUzaPJjVpDsGAsNzVnq7tHTxAl0s5UFznCTkVCau4,763
-arpakitlib-1.7.93.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-arpakitlib-1.7.93.dist-info/entry_points.txt,sha256=36xqR3PJFT2kuwjkM_EqoIy0qFUDPKSm_mJaI7emewE,87
-arpakitlib-1.7.93.dist-info/RECORD,,
+arpakitlib-1.7.95.dist-info/LICENSE,sha256=GPEDQMam2r7FSTYqM1mm7aKnxLaWcBotH7UvQtea-ec,11355
+arpakitlib-1.7.95.dist-info/METADATA,sha256=7AdfJVZL5IKD-fAW1diQvDtisHJlpoQ1EaAxaWbHnEU,2824
+arpakitlib-1.7.95.dist-info/NOTICE,sha256=95aUzaPJjVpDsGAsNzVnq7tHTxAl0s5UFznCTkVCau4,763
+arpakitlib-1.7.95.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+arpakitlib-1.7.95.dist-info/entry_points.txt,sha256=36xqR3PJFT2kuwjkM_EqoIy0qFUDPKSm_mJaI7emewE,87
+arpakitlib-1.7.95.dist-info/RECORD,,