schd 0.0.12__tar.gz → 0.0.13__tar.gz

{schd-0.0.12 → schd-0.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: schd
-Version: 0.0.12
+Version: 0.0.13
 Home-page: https://github.com/kevenli/schd
 License: ApacheV2
 Requires-Dist: apscheduler<4.0

{schd-0.0.12 → schd-0.0.13}/schd/config.py

@@ -70,6 +70,8 @@ class JobConfig(ConfigValue):
     cron: str
     cmd: Optional[str] = None
     params: dict = field(default_factory=dict)
+    timezone: Optional[str] = None
+    queue: str = ''
 
 
 @dataclass

{schd-0.0.12 → schd-0.0.13}/schd/scheduler.py

@@ -195,7 +195,7 @@ class LocalScheduler:
     async def init(self):
         pass
 
-    async def add_job(self, job: Job, cron_expression: str, job_name: str) -> None:
+    async def add_job(self, job: Job, job_name: str, job_config:JobConfig) -> None:
         """
         Add a job to the scheduler.
 
@@ -205,6 +205,7 @@ class LocalScheduler:
         """
         self._jobs[job_name] = job
         try:
+            cron_expression = job_config.cron
             cron_trigger = CronTrigger.from_crontab(cron_expression)
             self.scheduler.add_job(self.execute_job, cron_trigger, kwargs={'job_name':job_name})
             logger.info(f"Job '{job_name or job.__class__.__name__}' added with cron expression: {cron_expression}")
@@ -304,7 +305,7 @@ async def run_daemon(config_file=None):
         job_class_name = job_config.cls
         job_cron = job_config.cron
         job = build_job(job_name, job_class_name, job_config)
-        await scheduler.add_job(job, job_cron, job_name=job_name)
+        await scheduler.add_job(job, job_name, job_config)
         logger.info('job added, %s', job_name)
 
     logger.info('scheduler starting.')

{schd-0.0.12 → schd-0.0.13}/schd/schedulers/remote.py

@@ -3,10 +3,11 @@ from contextlib import redirect_stdout
 import io
 import json
 import os
-from typing import Dict
+from typing import Dict, Tuple
 from urllib.parse import urljoin
 import aiohttp
 import aiohttp.client_exceptions
+from schd.config import JobConfig
 from schd.job import JobContext, Job
 
 import logging
@@ -25,11 +26,14 @@ class RemoteApiClient:
                 response.raise_for_status()
                 result = await response.json()
 
-    async def register_job(self, worker_name, job_name, cron):
+    async def register_job(self, worker_name, job_name, cron, timezone=None):
         url = urljoin(self._base_url, f'/api/workers/{worker_name}/jobs/{job_name}')
         post_data = {
             'cron': cron,
         }
+        if timezone:
+            post_data['timezone'] = timezone
+
         async with aiohttp.ClientSession() as session:
             async with session.put(url, json=post_data) as response:
                 response.raise_for_status()
@@ -80,16 +84,23 @@ class RemoteScheduler:
     def __init__(self, worker_name:str, remote_host:str):
         self.client = RemoteApiClient(remote_host)
         self._worker_name = worker_name
-        self._jobs:"Dict[str,Job]" = {}
+        self._jobs:"Dict[str,Tuple[Job,str]]" = {}
         self._loop_task = None
         self._loop = asyncio.get_event_loop()
+        self.queue_semaphores = {}
 
     async def init(self):
         await self.client.register_worker(self._worker_name)
 
-    async def add_job(self, job:Job, cron, job_name):
-        await self.client.register_job(self._worker_name, job_name=job_name, cron=cron)
-        self._jobs[job_name] = job
+    async def add_job(self, job:Job, job_name:str, job_config:JobConfig):
+        cron = job_config.cron
+        queue_name = job_config.queue or ''
+        await self.client.register_job(self._worker_name, job_name=job_name, cron=cron, timezone=job_config.timezone)
+        self._jobs[job_name] = (job, queue_name)
+        if queue_name not in self.queue_semaphores:
+            # each queue has a max concurrency of 1
+            max_conc = 1
+            self.queue_semaphores[queue_name] = asyncio.Semaphore(max_conc)
 
     async def start_main_loop(self):
         while True:
@@ -97,11 +108,19 @@ class RemoteScheduler:
             try:
                 async for event in self.client.subscribe_worker_eventstream(self._worker_name):
                     print(event)
-                    await self.execute_task(event['data']['job_name'], event['data']['id'])
+                    job_name = event['data']['job_name']
+                    instance_id = event['data']['id']
+                    _, queue_name = self._jobs[job_name]
+                    # Queue concurrency control
+                    semaphore = self.queue_semaphores[queue_name]
+                    self._loop.create_task(self._run_with_semaphore(semaphore, job_name, instance_id))
+                    # await self.execute_task(event['data']['job_name'], event['data']['id'])
             except aiohttp.client_exceptions.ClientPayloadError:
                 logger.info('connection lost')
+                await asyncio.sleep(1)
             except aiohttp.client_exceptions.SocketTimeoutError:
                 logger.info('SocketTimeoutError')
+                await asyncio.sleep(1)
             except aiohttp.client_exceptions.ClientConnectorError:
                 # cannot connect, try later
                 logger.debug('connect failed, ClientConnectorError, try later.')
@@ -115,7 +134,7 @@ class RemoteScheduler:
         self._loop_task = self._loop.create_task(self.start_main_loop())
 
     async def execute_task(self, job_name, instance_id:int):
-        job = self._jobs[job_name]
+        job, _ = self._jobs[job_name]
         logfile_dir = f'joblog/{instance_id}'
         if not os.path.exists(logfile_dir):
             os.makedirs(logfile_dir)
@@ -148,3 +167,7 @@ class RemoteScheduler:
         output_stream.close()
         await self.client.commit_job_log(self._worker_name, job_name, instance_id, logfile_path)
         await self.client.update_job_instance(self._worker_name, job_name, instance_id, status='COMPLETED', ret_code=ret_code)
+
+    async def _run_with_semaphore(self, semaphore, job_name, instance_id):
+        async with semaphore:
+            await self.execute_task(job_name, instance_id)

{schd-0.0.12 → schd-0.0.13}/schd.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: schd
-Version: 0.0.12
+Version: 0.0.13
 Home-page: https://github.com/kevenli/schd
 License: ApacheV2
 Requires-Dist: apscheduler<4.0

{schd-0.0.12 → schd-0.0.13}/setup.py

@@ -7,7 +7,7 @@ def read_requirements():
 
 setup(
     name="schd",
-    version="0.0.12",
+    version="0.0.13",
     url="https://github.com/kevenli/schd",
     packages=find_packages(exclude=('tests', 'tests.*')),
     install_requires=['apscheduler<4.0', 'pyaml', 'aiohttp'],