schd 0.0.12__py3-none-any.whl → 0.0.13__py3-none-any.whl

schd/config.py

@@ -70,6 +70,8 @@ class JobConfig(ConfigValue):
     cron: str
     cmd: Optional[str] = None
     params: dict = field(default_factory=dict)
+    timezone: Optional[str] = None
+    queue: str = ''
 
 
 @dataclass

schd/scheduler.py

@@ -195,7 +195,7 @@ class LocalScheduler:
     async def init(self):
         pass
 
-    async def add_job(self, job: Job, cron_expression: str, job_name: str) -> None:
+    async def add_job(self, job: Job, job_name: str, job_config:JobConfig) -> None:
         """
         Add a job to the scheduler.
 
@@ -205,6 +205,7 @@ class LocalScheduler:
         """
         self._jobs[job_name] = job
         try:
+            cron_expression = job_config.cron
             cron_trigger = CronTrigger.from_crontab(cron_expression)
             self.scheduler.add_job(self.execute_job, cron_trigger, kwargs={'job_name':job_name})
             logger.info(f"Job '{job_name or job.__class__.__name__}' added with cron expression: {cron_expression}")
@@ -304,7 +305,7 @@ async def run_daemon(config_file=None):
         job_class_name = job_config.cls
         job_cron = job_config.cron
         job = build_job(job_name, job_class_name, job_config)
-        await scheduler.add_job(job, job_cron, job_name=job_name)
+        await scheduler.add_job(job, job_name, job_config)
         logger.info('job added, %s', job_name)
 
     logger.info('scheduler starting.')

schd/schedulers/remote.py

@@ -3,10 +3,11 @@ from contextlib import redirect_stdout
 import io
 import json
 import os
-from typing import Dict
+from typing import Dict, Tuple
 from urllib.parse import urljoin
 import aiohttp
 import aiohttp.client_exceptions
+from schd.config import JobConfig
 from schd.job import JobContext, Job
 
 import logging
@@ -25,11 +26,14 @@ class RemoteApiClient:
                 response.raise_for_status()
                 result = await response.json()
 
-    async def register_job(self, worker_name, job_name, cron):
+    async def register_job(self, worker_name, job_name, cron, timezone=None):
         url = urljoin(self._base_url, f'/api/workers/{worker_name}/jobs/{job_name}')
         post_data = {
             'cron': cron,
         }
+        if timezone:
+            post_data['timezone'] = timezone
+
         async with aiohttp.ClientSession() as session:
             async with session.put(url, json=post_data) as response:
                 response.raise_for_status()
@@ -80,16 +84,23 @@ class RemoteScheduler:
     def __init__(self, worker_name:str, remote_host:str):
         self.client = RemoteApiClient(remote_host)
         self._worker_name = worker_name
-        self._jobs:"Dict[str,Job]" = {}
+        self._jobs:"Dict[str,Tuple[Job,str]]" = {}
         self._loop_task = None
         self._loop = asyncio.get_event_loop()
+        self.queue_semaphores = {}
 
     async def init(self):
         await self.client.register_worker(self._worker_name)
 
-    async def add_job(self, job:Job, cron, job_name):
-        await self.client.register_job(self._worker_name, job_name=job_name, cron=cron)
-        self._jobs[job_name] = job
+    async def add_job(self, job:Job, job_name:str, job_config:JobConfig):
+        cron = job_config.cron
+        queue_name = job_config.queue or ''
+        await self.client.register_job(self._worker_name, job_name=job_name, cron=cron, timezone=job_config.timezone)
+        self._jobs[job_name] = (job, queue_name)
+        if queue_name not in self.queue_semaphores:
+            # each queue has a max concurrency of 1
+            max_conc = 1
+            self.queue_semaphores[queue_name] = asyncio.Semaphore(max_conc)
 
     async def start_main_loop(self):
         while True:
@@ -97,11 +108,19 @@ class RemoteScheduler:
             try:
                 async for event in self.client.subscribe_worker_eventstream(self._worker_name):
                     print(event)
-                    await self.execute_task(event['data']['job_name'], event['data']['id'])
+                    job_name = event['data']['job_name']
+                    instance_id = event['data']['id']
+                    _, queue_name = self._jobs[job_name]
+                    # Queue concurrency control
+                    semaphore = self.queue_semaphores[queue_name]
+                    self._loop.create_task(self._run_with_semaphore(semaphore, job_name, instance_id))
+                    # await self.execute_task(event['data']['job_name'], event['data']['id'])
             except aiohttp.client_exceptions.ClientPayloadError:
                 logger.info('connection lost')
+                await asyncio.sleep(1)
             except aiohttp.client_exceptions.SocketTimeoutError:
                 logger.info('SocketTimeoutError')
+                await asyncio.sleep(1)
             except aiohttp.client_exceptions.ClientConnectorError:
                 # cannot connect, try later
                 logger.debug('connect failed, ClientConnectorError, try later.')
@@ -115,7 +134,7 @@ class RemoteScheduler:
         self._loop_task = self._loop.create_task(self.start_main_loop())
 
     async def execute_task(self, job_name, instance_id:int):
-        job = self._jobs[job_name]
+        job, _ = self._jobs[job_name]
         logfile_dir = f'joblog/{instance_id}'
         if not os.path.exists(logfile_dir):
             os.makedirs(logfile_dir)
@@ -148,3 +167,7 @@ class RemoteScheduler:
         output_stream.close()
         await self.client.commit_job_log(self._worker_name, job_name, instance_id, logfile_path)
         await self.client.update_job_instance(self._worker_name, job_name, instance_id, status='COMPLETED', ret_code=ret_code)
+
+    async def _run_with_semaphore(self, semaphore, job_name, instance_id):
+        async with semaphore:
+            await self.execute_task(job_name, instance_id)

{schd-0.0.12.dist-info → schd-0.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: schd
-Version: 0.0.12
+Version: 0.0.13
 Home-page: https://github.com/kevenli/schd
 License: ApacheV2
 Requires-Dist: apscheduler<4.0

{schd-0.0.12.dist-info → schd-0.0.13.dist-info}/RECORD

@@ -1,7 +1,7 @@
 schd/__init__.py,sha256=ZOKi_EM_Ao1bVuMO_3kKsyHwO0zn1I0hNk7eirjowrA,24
-schd/config.py,sha256=21KpKPOwa7sAu4m2YS7kd644M2Lxnx2HuU_Czqs-R48,3873
+schd/config.py,sha256=YBSRiXZApC_ylPYUM4cwTU7FjVqMHwG3vPy-F6SA95k,3930
 schd/job.py,sha256=AoW-2W1hRY_O4nz_pKukgmEXYkjlP2XmYimyoGCs-Bw,614
-schd/scheduler.py,sha256=PlcHiwUebI2GHrgL23CvDXrhRrlgpkY9O0n39FZJVzk,12891
+schd/scheduler.py,sha256=EYIU7aX8qs-RYS_eXJc-JZU3e1T-mTMzkEv51b-AXgM,12931
 schd/util.py,sha256=NH4EqIns1Y01yz1Rf3cw-tlKSLc-XNuS9hHbAdq5D_I,592
 schd/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 schd/cmds/base.py,sha256=ZnNcJozQFLbpYyNp8dhHzm3BzFQa0hm6KyCC6URfueY,122
@@ -10,9 +10,9 @@ schd/cmds/jobs.py,sha256=843M6rcMjqmm8yevHF4QsJxc0fw8LQfSX7OEA_OH0Dg,590
 schd/cmds/run.py,sha256=T7WeTMf6-crmC8fD6YoMsp0nQbKemNTf4_Q9a-b789w,853
 schd/cmds/schd.py,sha256=vOlfQCQT81KhLMvlP3tlymEWpxmqlBOa5vnJoeFXVlw,996
 schd/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-schd/schedulers/remote.py,sha256=m9UiHE8TziBzzOGHwdCOnOraOiy-78bFAQJ-m21fOtg,6567
-schd-0.0.12.dist-info/METADATA,sha256=tXxLQ9jdE_Kyn1Yee0zNmUjJCvZSk1o6RWOXXS0T7Cc,195
-schd-0.0.12.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-schd-0.0.12.dist-info/entry_points.txt,sha256=VvUhIaucvHlggoz-4lWtQgXbDwoWH9x-iT0QOUHEMyI,45
-schd-0.0.12.dist-info/top_level.txt,sha256=Vojim8xSOsYyQHrZBNhx7n0F7WqbEJ2SeBFAvEnrJ_U,5
-schd-0.0.12.dist-info/RECORD,,
+schd/schedulers/remote.py,sha256=QMALl5VKAC1YzVZUF0JBYYF2XCuF4rwd5OBpn3mKJQY,7736
+schd-0.0.13.dist-info/METADATA,sha256=-WBWRbJgcsJ-lNKcbKLJDYKLrd1f5tv6Mri5CqTxNEI,195
+schd-0.0.13.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+schd-0.0.13.dist-info/entry_points.txt,sha256=VvUhIaucvHlggoz-4lWtQgXbDwoWH9x-iT0QOUHEMyI,45
+schd-0.0.13.dist-info/top_level.txt,sha256=Vojim8xSOsYyQHrZBNhx7n0F7WqbEJ2SeBFAvEnrJ_U,5
+schd-0.0.13.dist-info/RECORD,,