kiba-core 0.5.3.dev69__tar.gz → 0.5.3.dev71__tar.gz

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/CHANGELOG.md

@@ -8,6 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 
 ### Added
+- [PATCH] Fixed log records from `logging.info`, `warning`, `error`, `critical`, `debug`, `exception`, `stat` and `api` recording their location as `core/logging.py` instead of the calling code; this was only visible in `init_logging`'s text format, since the JSON and basic formats don't output the location
+- [MINOR] Updated `init_logging`, `init_json_logging` and `init_basic_logging` to set the loggers of the third-party libraries core uses (`DEFAULT_EXTERNAL_LOGGER_NAMES`: httpx2, httpcore2, sqlalchemy, aiosqlite, asyncpg, botocore, aiobotocore, azure, apscheduler, web3) to `WARNING`; pass `extraLoggerNames` to quieten more loggers on top of these, or `loggerNames` to replace them (`loggerNames=[]` keeps the previous behaviour). `init_external_loggers` takes the same arguments and now defaults to the same list
+- [MINOR] Added `core.cron` (install with the `cron` extra): `CronJob` schedules (interval with jitter, or crontab) that declare a `jobType`, `CronJobProcessor`s that declare the `jobTypes` they process, and `CronExecutor`, which takes a list of processors, routes each job to the processor for its type (failing at startup if a job type has no processor or more than one), runs jobs on an APScheduler scheduler, logs each run as a `CRON` api log with its status and duration, and keeps running when a job fails; also added `ApiRequestCronJob` with `ApiRequestCronJobProcessor` (needs the `requester` extra) for jobs that call a URL
 - [MAJOR] Updated `Requester` to raise `RequesterTimeoutException` (a `GatewayTimeoutException`, 504) instead of `httpx` timeout exceptions; it carries the method, the URL without credentials or query string, the timeout, how long the request ran and the `httpx` timeout type, and keeps the original exception as its cause. Code catching `httpx.TimeoutException` (or `ReadTimeout`, `ConnectTimeout`, …) around `Requester` calls must catch `RequesterTimeoutException` instead
 - [PATCH] Updated `MessageQueueProcessor` failure notifications to show the exception type before its message, so failures from exceptions without text (e.g. `httpx` timeouts) are still identifiable
 - [MINOR] Added `authorize_bearer_jwt_request` for validating bearer JWTs in `RouteAuthResolver` implementations; it and `authorize_bearer_jwt` now only strip the leading `Bearer ` prefix, let cancellation propagate, and log why a JWT was rejected

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev69
+Version: 0.5.3.dev71
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12
@@ -34,6 +34,8 @@ Provides-Extra: requester
 Requires-Dist: httpx2>=2.13.0; extra == "requester"
 Provides-Extra: web3
 Requires-Dist: web3<8.0.0,>=7.16.0; extra == "web3"
+Provides-Extra: cron
+Requires-Dist: apscheduler<4,>=3.11.0; extra == "cron"
 Provides-Extra: types
 Requires-Dist: types-aiobotocore[essential]>=3.9.1; extra == "types"
 Requires-Dist: types-aiofiles>=25.1.0.20260518; extra == "types"

kiba_core-0.5.3.dev71/core/cron/api_request_cron_job_processor.py

@@ -0,0 +1,34 @@
+from typing import ClassVar
+
+from core.cron.cron_job_processor import CronJobProcessor
+from core.cron.model import CronJob
+from core.exceptions import KibaException
+from core.requester import Requester
+from core.util.typing_util import Json
+
+
+class ApiRequestCronJob(CronJob):
+    jobType: ClassVar[str] = 'api_request'
+    method: str
+    url: str
+    dataDict: Json | None = None
+    headers: dict[str, str] | None = None
+    timeout: int | None = None
+
+    @property
+    def query(self) -> dict[str, str]:
+        if not isinstance(self.dataDict, dict):
+            return {}
+        return {key: str(value) for key, value in self.dataDict.items() if isinstance(value, str | int | float | bool)}
+
+
+class ApiRequestCronJobProcessor(CronJobProcessor):
+    jobTypes: ClassVar[frozenset[str]] = frozenset({ApiRequestCronJob.jobType})
+
+    def __init__(self, requester: Requester) -> None:
+        self.requester = requester
+
+    async def process_job(self, job: CronJob) -> None:
+        if not isinstance(job, ApiRequestCronJob):
+            raise KibaException(message=f'{type(self).__name__} cannot process {type(job).__name__}')
+        await self.requester.make_request(method=job.method, url=job.url, dataDict=job.dataDict, headers=job.headers, timeout=job.timeout)

kiba_core-0.5.3.dev71/core/cron/cron_executor.py

@@ -0,0 +1,68 @@
+import asyncio
+import datetime
+import time
+import urllib.parse as urlparse
+from collections.abc import Sequence
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler  # type: ignore[import-untyped]
+from apscheduler.triggers.cron import CronTrigger  # type: ignore[import-untyped]
+from apscheduler.triggers.interval import IntervalTrigger  # type: ignore[import-untyped]
+
+from core import logging
+from core.cron.cron_job_processor import CronJobProcessor
+from core.cron.model import CronJob
+from core.exceptions import KibaException
+
+# NOTE(krishan711): interval jobs are anchored to a fixed date so a restart keeps the same schedule instead of restarting every interval from boot
+INTERVAL_START_DATE = datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC)
+
+
+class CronExecutor:
+    def __init__(self, jobs: Sequence[CronJob], jobProcessors: Sequence[CronJobProcessor]) -> None:
+        self.jobs = jobs
+        self.processorByJobType: dict[str, CronJobProcessor] = {}
+        for jobProcessor in jobProcessors:
+            for jobType in jobProcessor.jobTypes:
+                existingProcessor = self.processorByJobType.get(jobType)
+                if existingProcessor is not None:
+                    raise KibaException(message=f'Cron job type {jobType} is handled by both {type(existingProcessor).__name__} and {type(jobProcessor).__name__}')
+                self.processorByJobType[jobType] = jobProcessor
+        unprocessableJobNames = [job.name for job in jobs if job.jobType not in self.processorByJobType]
+        if unprocessableJobNames:
+            raise KibaException(message=f'No cron job processor for jobs: {", ".join(unprocessableJobNames)}')
+        self.scheduler = AsyncIOScheduler()
+
+    async def execute_job(self, job: CronJob) -> None:
+        path = job.name.replace(' ', '_').replace(':', '_').lower()
+        query = urlparse.urlencode(job.query, doseq=True)
+        logging.api(action='CRON', path=path, pathPattern=path, query=query)
+        startTime = time.time()
+        statusCode = 200
+        try:
+            await self.processorByJobType[job.jobType].process_job(job=job)
+        except Exception as exception:  # noqa: BLE001
+            statusCode = exception.statusCode if isinstance(exception, KibaException) else 500
+            logging.error('Caught exception whilst processing cron job:')
+            logging.exception(exception)
+        duration = time.time() - startTime
+        logging.api(action='CRON', path=path, pathPattern=path, query=query, response=statusCode, duration=duration)
+
+    def _build_trigger(self, job: CronJob) -> CronTrigger | IntervalTrigger:
+        if job.crontab:
+            return CronTrigger.from_crontab(expr=job.crontab)
+        return IntervalTrigger(weeks=job.weeks, days=job.days, hours=job.hours, minutes=job.minutes, seconds=job.seconds, jitter=job.jitter, start_date=INTERVAL_START_DATE)
+
+    async def start(self) -> None:
+        logging.info(f'Initializing CronExecutor with {len(self.jobs)} jobs')
+        for job in self.jobs:
+            self.scheduler.add_job(func=self.execute_job, kwargs={'job': job}, trigger=self._build_trigger(job=job), id=job.name, name=job.name, replace_existing=True)
+        self.scheduler.start()
+        logging.info('CronExecutor started successfully')
+
+    async def run(self) -> None:
+        await self.start()
+        try:
+            await asyncio.Event().wait()
+        finally:
+            logging.info('Shutting down scheduler...')
+            self.scheduler.shutdown()

kiba_core-0.5.3.dev71/core/cron/cron_job_processor.py

@@ -0,0 +1,13 @@
+import abc
+from abc import ABC
+from typing import ClassVar
+
+from core.cron.model import CronJob
+
+
+class CronJobProcessor(ABC):
+    jobTypes: ClassVar[frozenset[str]]
+
+    @abc.abstractmethod
+    async def process_job(self, job: CronJob) -> None:
+        raise NotImplementedError

kiba_core-0.5.3.dev71/core/cron/model.py

@@ -0,0 +1,19 @@
+from typing import ClassVar
+
+from pydantic import BaseModel
+
+
+class CronJob(BaseModel):
+    jobType: ClassVar[str]
+    name: str
+    weeks: int = 0
+    days: int = 0
+    hours: int = 0
+    minutes: int = 0
+    seconds: int = 0
+    jitter: int = 0
+    crontab: str | None = None
+
+    @property
+    def query(self) -> dict[str, str]:
+        return {}

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/core/logging.py

@@ -120,36 +120,43 @@ def init_logger(logger: Logger, loggingLevel: int, handler: StreamHandler) -> No
     logger.setLevel(level=loggingLevel)
 
 
-def init_external_loggers(loggerNames: Collection[str], loggingLevel: int = logging.WARNING) -> None:
-    for loggerName in loggerNames:
+# NOTE(krishan711): third-party libraries pulled in by core's extras; their info-level logs drown out the app's own
+DEFAULT_EXTERNAL_LOGGER_NAMES = ('httpx2', 'httpcore2', 'sqlalchemy', 'aiosqlite', 'asyncpg', 'botocore', 'aiobotocore', 'azure', 'apscheduler', 'web3')
+
+
+def init_external_loggers(loggerNames: Collection[str] | None = None, extraLoggerNames: Collection[str] | None = None, loggingLevel: int = logging.WARNING) -> None:
+    for loggerName in {*(loggerNames if loggerNames is not None else DEFAULT_EXTERNAL_LOGGER_NAMES), *(extraLoggerNames or [])}:
         logging.getLogger(loggerName).setLevel(loggingLevel)
 
 
-def init_logging(name: str, version: str, environment: str, showDebug: bool = False, requestIdHolder: RequestIdHolder | None = None) -> None:
+def init_logging(name: str, version: str, environment: str, showDebug: bool = False, requestIdHolder: RequestIdHolder | None = None, loggerNames: Collection[str] | None = None, extraLoggerNames: Collection[str] | None = None) -> None:
     loggingLevel = logging.DEBUG if showDebug else logging.INFO
     for logFormat in ALL_LOGGER_FORMATS:
         logger = logging.getLogger(name=logFormat.loggerType)
         handler = StreamHandler(stream=sys.stdout)
         handler.setFormatter(fmt=KibaLoggingFormatter(logFormat=logFormat, name=name, version=version, environment=environment, requestIdHolder=requestIdHolder))
         init_logger(logger=logger, loggingLevel=loggingLevel, handler=handler)
+    init_external_loggers(loggerNames=loggerNames, extraLoggerNames=extraLoggerNames)
 
 
-def init_json_logging(name: str, version: str, environment: str, showDebug: bool = False, requestIdHolder: RequestIdHolder | None = None) -> None:
+def init_json_logging(name: str, version: str, environment: str, showDebug: bool = False, requestIdHolder: RequestIdHolder | None = None, loggerNames: Collection[str] | None = None, extraLoggerNames: Collection[str] | None = None) -> None:
     loggingLevel = logging.DEBUG if showDebug else logging.INFO
     for logFormat in ALL_LOGGER_FORMATS:
         logger = logging.getLogger(name=logFormat.loggerType)
         handler = StreamHandler(stream=sys.stdout)
         handler.setFormatter(fmt=KibaJsonLoggingFormatter(logFormat=logFormat, name=name, version=version, environment=environment, requestIdHolder=requestIdHolder))
         init_logger(logger=logger, loggingLevel=loggingLevel, handler=handler)
+    init_external_loggers(loggerNames=loggerNames, extraLoggerNames=extraLoggerNames)
 
 
-def init_basic_logging(showDebug: bool = False) -> None:
+def init_basic_logging(showDebug: bool = False, loggerNames: Collection[str] | None = None, extraLoggerNames: Collection[str] | None = None) -> None:
     loggingLevel = logging.DEBUG if showDebug else logging.INFO
     for logFormat in ALL_LOGGER_FORMATS:
         logger = logging.getLogger(name=logFormat.loggerType)
         handler = StreamHandler(stream=sys.stdout)
         handler.setFormatter(fmt=Formatter(fmt=logFormat.loggerFormat))
         init_logger(logger=logger, loggingLevel=loggingLevel, handler=handler)
+    init_external_loggers(loggerNames=loggerNames, extraLoggerNames=extraLoggerNames)
 
 
 def _serialize_numeric_value(value: Union[float, None]) -> str:
@@ -168,7 +175,8 @@ def stat(name: str, key: str, value: float = 1) -> None:
         nameValue = _serialize_string_value(value=str(name))
         keyValue = _serialize_string_value(value=str(key))
         statValue = _serialize_numeric_value(value=value)
-        STAT_LOGGER.log(level=logging.INFO, msg='', extra={'statName': nameValue, 'statKey': keyValue, 'statValue': statValue})
+        # NOTE(krishan711): stacklevel=2 skips this function so records point at the caller
+        STAT_LOGGER.log(level=logging.INFO, msg='', extra={'statName': nameValue, 'statKey': keyValue, 'statValue': statValue}, stacklevel=2)
 
 
 # TODO(krishan711): make pathPattern mandatory in next major release
@@ -184,6 +192,7 @@ def api(action: str, path: str, query: str, pathPattern: str | None = None, resp
             level=logging.INFO,
             msg='',
             extra={'apiAction': actionString, 'apiPath': pathString, 'apiPathPattern': pathPatternString, 'apiQuery': queryString, 'apiResponse': responseString or '', 'apiDuration': durationString or ''},
+            stacklevel=2,
         )
 
 
@@ -197,9 +206,10 @@ INFO = logging.INFO
 DEBUG = logging.DEBUG
 
 
-def _log(level: int, msg: str, *args: Any, **kwargs: Any) -> None:  # type: ignore[explicit-any]
+def _log(level: int, msg: str, *args: Any, stacklevel: int = 1, **kwargs: Any) -> None:  # type: ignore[explicit-any]
     if ROOT_LOGGER.isEnabledFor(level=level):
-        ROOT_LOGGER._log(level=level, msg=msg, args=args, **kwargs)  # noqa: SLF001
+        # NOTE(krishan711): +2 skips this function and the public wrapper (info, error, ...) so records point at the caller
+        ROOT_LOGGER._log(level=level, msg=msg, args=args, stacklevel=stacklevel + 2, **kwargs)  # noqa: SLF001
 
 
 def critical(msg: str, *args: Any, **kwargs: Any) -> None:  # type: ignore[explicit-any]

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/kiba_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev69
+Version: 0.5.3.dev71
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12
@@ -34,6 +34,8 @@ Provides-Extra: requester
 Requires-Dist: httpx2>=2.13.0; extra == "requester"
 Provides-Extra: web3
 Requires-Dist: web3<8.0.0,>=7.16.0; extra == "web3"
+Provides-Extra: cron
+Requires-Dist: apscheduler<4,>=3.11.0; extra == "cron"
 Provides-Extra: types
 Requires-Dist: types-aiobotocore[essential]>=3.9.1; extra == "types"
 Requires-Dist: types-aiofiles>=25.1.0.20260518; extra == "types"

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/kiba_core.egg-info/SOURCES.txt

@@ -40,6 +40,11 @@ core/caching/__init__.py
 core/caching/cache.py
 core/caching/dict_cache.py
 core/caching/file_cache.py
+core/cron/__init__.py
+core/cron/api_request_cron_job_processor.py
+core/cron/cron_executor.py
+core/cron/cron_job_processor.py
+core/cron/model.py
 core/http/__init__.py
 core/http/basic_authentication.py
 core/http/jwt.py
@@ -92,6 +97,7 @@ kiba_core.egg-info/top_level.txt
 tests/__init__.py
 tests/test_database.py
 tests/test_exceptions.py
+tests/test_logging.py
 tests/api/test_api_response.py
 tests/api/test_authorizer.py
 tests/api/test_exception_handling_middleware.py
@@ -104,6 +110,8 @@ tests/api/test_route.py
 tests/api/test_streaming_json_route.py
 tests/caching/test_dict_cache.py
 tests/caching/test_file_cache.py
+tests/cron/__init__.py
+tests/cron/test_cron_executor.py
 tests/locks/test_sql_lock.py
 tests/queues/conftest.py
 tests/queues/test_aqs_queue.py

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/kiba_core.egg-info/requires.txt

@@ -10,6 +10,9 @@ uvicorn[standard]>=0.53.0
 starlette>=1.6.0
 uvicorn[standard]>=0.53.0
 
+[cron]
+apscheduler<4,>=3.11.0
+
 [database-psql]
 sqlalchemy[asyncio]>=2.0.54
 asyncpg>=0.31.0

{kiba_core-0.5.3.dev69 → kiba_core-0.5.3.dev71}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kiba-core"
-version = "0.5.3.dev69"
+version = "0.5.3.dev71"
 description = "Kiba Labs' python utilities"
 readme = "README.md"
 requires-python = ">=3.12, <4"
@@ -49,6 +49,9 @@ requester = [
 web3 = [
     "web3>=7.16.0,<8.0.0",
 ]
+cron = [
+    "apscheduler>=3.11.0,<4",
+]
 types = [
     "types-aiobotocore[essential]>=3.9.1",
     "types-aiofiles>=25.1.0.20260518",

kiba_core-0.5.3.dev71/tests/cron/test_cron_executor.py

@@ -0,0 +1,75 @@
+from typing import ClassVar
+
+import pytest
+
+from core.cron.cron_executor import CronExecutor
+from core.cron.cron_job_processor import CronJobProcessor
+from core.cron.model import CronJob
+from core.exceptions import KibaException
+
+
+class EmailCronJob(CronJob):
+    jobType: ClassVar[str] = 'email'
+
+
+class ReportCronJob(CronJob):
+    jobType: ClassVar[str] = 'report'
+
+
+class RecordingProcessor(CronJobProcessor):
+    def __init__(self, failingJobNames: frozenset[str] = frozenset()) -> None:
+        self.failingJobNames = failingJobNames
+        self.processedJobNames: list[str] = []
+
+    async def process_job(self, job: CronJob) -> None:
+        self.processedJobNames.append(job.name)
+        if job.name in self.failingJobNames:
+            raise ValueError(f'{job.name} failed')
+
+
+class EmailProcessor(RecordingProcessor):
+    jobTypes: ClassVar[frozenset[str]] = frozenset({EmailCronJob.jobType})
+
+
+class ReportProcessor(RecordingProcessor):
+    jobTypes: ClassVar[frozenset[str]] = frozenset({ReportCronJob.jobType})
+
+
+class EmailAndReportProcessor(RecordingProcessor):
+    jobTypes: ClassVar[frozenset[str]] = frozenset({EmailCronJob.jobType, ReportCronJob.jobType})
+
+
+@pytest.mark.asyncio
+async def test_jobs_are_routed_to_the_processor_for_their_type() -> None:
+    emailProcessor = EmailProcessor()
+    reportProcessor = ReportProcessor()
+    emailJob = EmailCronJob(name='send-emails', minutes=1)
+    reportJob = ReportCronJob(name='build-report', minutes=1)
+    executor = CronExecutor(jobs=[emailJob, reportJob], jobProcessors=[emailProcessor, reportProcessor])
+
+    await executor.execute_job(job=reportJob)
+    await executor.execute_job(job=emailJob)
+
+    assert (emailProcessor.processedJobNames, reportProcessor.processedJobNames) == (['send-emails'], ['build-report'])
+
+
+@pytest.mark.asyncio
+async def test_failing_job_does_not_stop_later_jobs() -> None:
+    processor = EmailProcessor(failingJobNames=frozenset({'first'}))
+    first, second = EmailCronJob(name='first', minutes=1), EmailCronJob(name='second', minutes=1)
+    executor = CronExecutor(jobs=[first, second], jobProcessors=[processor])
+
+    await executor.execute_job(job=first)
+    await executor.execute_job(job=second)
+
+    assert processor.processedJobNames == ['first', 'second']
+
+
+def test_job_without_a_processor_fails_at_startup() -> None:
+    with pytest.raises(KibaException, match='No cron job processor for jobs: build-report'):
+        CronExecutor(jobs=[ReportCronJob(name='build-report', minutes=1)], jobProcessors=[EmailProcessor()])
+
+
+def test_job_type_claimed_by_two_processors_fails_at_startup() -> None:
+    with pytest.raises(KibaException, match='Cron job type email is handled by both'):
+        CronExecutor(jobs=[], jobProcessors=[EmailProcessor(), EmailAndReportProcessor()])