tarka 0.19.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. tarka/__init__.py +10 -0
  2. tarka/asqla/__init__.py +3 -0
  3. tarka/asqla/alembic.py +47 -0
  4. tarka/asqla/database.py +161 -0
  5. tarka/asqla/postgres.py +46 -0
  6. tarka/asqla/tx.py +106 -0
  7. tarka/asqla/types.py +20 -0
  8. tarka/logging/__init__.py +0 -0
  9. tarka/logging/bootstrap.py +54 -0
  10. tarka/logging/formatter.py +76 -0
  11. tarka/logging/handler.py +114 -0
  12. tarka/logging/hook.py +37 -0
  13. tarka/logging/level.py +51 -0
  14. tarka/logging/manager.py +141 -0
  15. tarka/logging/patch.py +106 -0
  16. tarka/logging/utility.py +25 -0
  17. tarka/utility/__init__.py +3 -0
  18. tarka/utility/aio_sqlite.py +259 -0
  19. tarka/utility/algorithm/__init__.py +0 -0
  20. tarka/utility/algorithm/iter.py +253 -0
  21. tarka/utility/algorithm/seq.py +49 -0
  22. tarka/utility/algorithm/traverse.py +70 -0
  23. tarka/utility/envvar.py +33 -0
  24. tarka/utility/file/__init__.py +2 -0
  25. tarka/utility/file/name.py +44 -0
  26. tarka/utility/file/reserve.py +167 -0
  27. tarka/utility/file/resources.py +37 -0
  28. tarka/utility/file/safe.py +62 -0
  29. tarka/utility/importing.py +92 -0
  30. tarka/utility/sentinel.py +28 -0
  31. tarka/utility/serialize/__init__.py +0 -0
  32. tarka/utility/serialize/deterministic.py +220 -0
  33. tarka/utility/thread.py +84 -0
  34. tarka/utility/time/__init__.py +0 -0
  35. tarka/utility/time/format.py +47 -0
  36. tarka/utility/time/utc.py +125 -0
  37. tarka/utility/utf8.py +40 -0
  38. tarka-0.19.1.dist-info/METADATA +40 -0
  39. tarka-0.19.1.dist-info/RECORD +43 -0
  40. tarka-0.19.1.dist-info/WHEEL +5 -0
  41. tarka-0.19.1.dist-info/licenses/LICENSE +177 -0
  42. tarka-0.19.1.dist-info/licenses/NOTICE +2 -0
  43. tarka-0.19.1.dist-info/top_level.txt +1 -0
tarka/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ """
2
+ Tarka
3
+
4
+ :copyright: 2022-2025 Nándor Mátravölgyi
5
+ :license: Apache2, see LICENSE for more details.
6
+ """
7
+
8
+ __copyright__ = "Copyright 2022-2025 Nándor Mátravölgyi"
9
+ __credits__ = ["Nándor Mátravölgyi"]
10
+ __version__ = "0.19.1"
@@ -0,0 +1,3 @@
1
+ """
2
+ This implements a slightly opinionated reusable async Alembic+SQLAlchemy setup, for well managed database persistence.
3
+ """
tarka/asqla/alembic.py ADDED
@@ -0,0 +1,47 @@
1
+ import os
2
+ from io import StringIO
3
+
4
+ from alembic import command
5
+ from alembic.config import Config
6
+ from alembic.script import ScriptDirectory
7
+ from sqlalchemy import Connection
8
+
9
+
10
+ def get_alembic_config(alembic_dir: str, sqlalchemy_url: str = None) -> Config:
11
+ alembic_cfg = Config(os.path.join(alembic_dir, "alembic.ini"))
12
+ alembic_cfg.set_main_option("script_location", alembic_dir.replace("%", "%%"))
13
+ if sqlalchemy_url:
14
+ alembic_cfg.set_main_option("sqlalchemy.url", str(sqlalchemy_url).replace("%", "%%"))
15
+ alembic_cfg.attributes["skip-logging-setup"] = True # do not mess up the server logging
16
+ return alembic_cfg
17
+
18
+
19
+ class NoHeadRevision(Exception):
20
+ pass
21
+
22
+
23
+ class AlembicHelper:
24
+ def __init__(self, config: Config):
25
+ self.config = config
26
+
27
+ def run(self, conn: Connection, alembic_command: str, *args: str) -> str:
28
+ """
29
+ Call the alembic CLI.
30
+ """
31
+ self.config.attributes["connection"] = conn
32
+ self.config.stdout = StringIO()
33
+ getattr(command, alembic_command)(self.config, *args)
34
+ return self.config.stdout.getvalue()
35
+
36
+ def run_strip_output(self, conn: Connection, alembic_command: str, *args: str) -> str:
37
+ return self.run(conn, alembic_command, *args).strip()
38
+
39
+ def get_head_revision(self) -> str:
40
+ """
41
+ Get current single head revision.
42
+ """
43
+ script = ScriptDirectory.from_config(self.config)
44
+ revs = list(script.get_revisions("head"))
45
+ if len(revs) != 1:
46
+ raise NoHeadRevision()
47
+ return revs[0].revision
@@ -0,0 +1,161 @@
1
+ import asyncio
2
+ from contextlib import asynccontextmanager
3
+ from typing import Type, Dict, Any, Optional
4
+
5
+ from alembic.config import Config
6
+ from sqlalchemy import event
7
+ from sqlalchemy.exc import DatabaseError
8
+ from sqlalchemy.ext.asyncio import create_async_engine, AsyncConnection, AsyncSession, AsyncEngine
9
+ from sqlalchemy.orm import sessionmaker
10
+
11
+ from tarka.asqla.alembic import get_alembic_config, AlembicHelper
12
+ from tarka.asqla.tx import (
13
+ TransactionExecutor,
14
+ RetryableTransactionFactory,
15
+ SessionTransactionExecutor,
16
+ RetryableSessionTransactionFactory,
17
+ )
18
+
19
+
20
+ class Database(object):
21
+ engine: AsyncEngine = None
22
+ serializable_engine: AsyncEngine = None
23
+ session_maker: Type[AsyncSession] = None
24
+
25
+ def __init__(
26
+ self,
27
+ alembic_dir: str,
28
+ connect_url: str,
29
+ engine_kwargs: Dict[str, Any] = None, # like echo, connect_args, etc.
30
+ session_maker_kwargs: Dict[str, Any] = None, # like expire_on_commit, autoflush, etc.
31
+ aiosqlite_serializable_begin: str = "BEGIN", # serializable tx support workaround for aiosqlite
32
+ tx_retry_delay: Optional[float] = None, # tx retry delay helps to relieve db pressure at conflicts
33
+ ):
34
+ """
35
+ If aiosqlite is used aiosqlite_serializable_begin="BEGIN IMMEDIATE" could be better, because then no
36
+ conflicts can happen, each tx caller will wait in line to acquire the lock and no resources will be
37
+ wasted retrying. However, this does not work well with session-tx, see TODO: tx.py:105.
38
+ """
39
+ self.alembic_dir = alembic_dir
40
+ self._connect_url = connect_url
41
+ self._engine_kwargs = engine_kwargs
42
+ self._session_maker_kwargs = session_maker_kwargs
43
+ self._aiosqlite_serializable_begin = aiosqlite_serializable_begin
44
+ self._tx_retry_delay = tx_retry_delay
45
+ self.alembic_head_at_startup = ""
46
+
47
+ def get_alembic_config(self) -> Config:
48
+ return get_alembic_config(self.alembic_dir, str(self.engine.sync_engine.url))
49
+
50
+ async def _init_engine(self):
51
+ """
52
+ If further customization is necessary for the engines, this shall be overridden.
53
+ To switch SQLite to WAL mode for example:
54
+
55
+ if self.engine.dialect.name == "sqlite":
56
+ @event.listens_for(self.engine.sync_engine, "connect")
57
+ def _setup_sqlite_connection(dbapi_con, con_record):
58
+ dbapi_con.execute("PRAGMA journal_mode = WAL;")
59
+ dbapi_con.execute("PRAGMA synchronous = NORMAL;")
60
+ """
61
+ self.engine = create_async_engine(self._connect_url, **(self._engine_kwargs or {}))
62
+ self.serializable_engine = self.engine.execution_options(isolation_level="SERIALIZABLE")
63
+ if self.engine.driver == "aiosqlite" and self._aiosqlite_serializable_begin:
64
+ # aiosqlite (<=0.20) does not honor the isolation_level with appropriate BEGIN commands, in fact it
65
+ # never issues begin commands, transactions always start implicitly for reads.
66
+ # SQLite itself does not support concurrent mutating transactions in any mode. In WAL journal_mode, there
67
+ # can be parallel and concurrent readers, but writing is always with an exclusive lock. In this sense "any"
68
+ # transaction will actually be serializable provided that an explicit BEGIN is emitted. The actual
69
+ # difference is the point and time that parallel connections encounter locking errors.
70
+ # The workaround (if not disabled) is to explicitly execute a BEGIN command at transaction start.
71
+ # The command can be set for the database (DEFERRED, IMMEDIATE or EXCLUSIVE) when relevant.
72
+ @event.listens_for(self.serializable_engine.sync_engine, "begin")
73
+ def do_begin(conn):
74
+ conn.exec_driver_sql(self._aiosqlite_serializable_begin)
75
+
76
+ self.session_maker = sessionmaker(self.engine, class_=AsyncSession, **(self._session_maker_kwargs or {}))
77
+
78
+ async def startup(self):
79
+ """
80
+ Initialize engine first, then bootstrap or migrate schema of the database to be up-to-date.
81
+ """
82
+ await self._init_engine()
83
+ alembic_helper = AlembicHelper(self.get_alembic_config())
84
+ retry = 0
85
+ while True: # handle conflicting migration attempt by parallel workers
86
+ try:
87
+ async with self.engine.begin() as connection:
88
+ connection: AsyncConnection
89
+ self.alembic_head_at_startup = await connection.run_sync(alembic_helper.run_strip_output, "current")
90
+
91
+ await self._upgrade(alembic_helper, connection)
92
+ await self._start_hook(connection)
93
+ except DatabaseError:
94
+ if retry >= 5:
95
+ raise
96
+ retry += 1
97
+ await asyncio.sleep(0.25)
98
+ else:
99
+ break
100
+
101
+ async def _upgrade(self, alembic_helper: AlembicHelper, connection: AsyncConnection):
102
+ if not self.alembic_head_at_startup.endswith(" (head)"):
103
+ await self._pre_upgrade_hook(connection)
104
+ await connection.run_sync(alembic_helper.run, "upgrade", "head")
105
+ await self._post_upgrade_hook(connection)
106
+
107
+ async def _pre_upgrade_hook(self, connection: AsyncConnection):
108
+ """
109
+ Place to put custom DDL commands before alembic upgrade.
110
+ Keep in mind that it is usually better to implement all changes as alembic revisions.
111
+ NOTE: This is called before bootstrap (first upgrade) as well.
112
+ """
113
+
114
+ async def _post_upgrade_hook(self, connection: AsyncConnection):
115
+ """
116
+ Place to put custom DDL commands after alembic upgrade.
117
+ Keep in mind that it is usually better to implement all changes as alembic revisions.
118
+ """
119
+
120
+ async def _start_hook(self, connection: AsyncConnection):
121
+ """
122
+ Place to put custom DDL commands that are executed each time a server starts.
123
+ The connection is currently at the HEAD alembic revision, in the upgrade transaction.
124
+ """
125
+
126
+ async def shutdown(self):
127
+ await self.engine.dispose()
128
+
129
+ @asynccontextmanager
130
+ async def run(self):
131
+ """
132
+ Convenience wrapper for startup & shutdown.
133
+ """
134
+ await self.startup()
135
+ try:
136
+ yield self
137
+ finally:
138
+ await self.shutdown()
139
+
140
+ def serializable_tx(
141
+ self, tx_factory: RetryableTransactionFactory, retry_delay: Optional[float] = None
142
+ ) -> TransactionExecutor:
143
+ """
144
+ Get a transaction executor with SERIALIZABLE isolation_level and automatic retry.
145
+ """
146
+ return TransactionExecutor(
147
+ self.serializable_engine, tx_factory, retry_delay=retry_delay or self._tx_retry_delay
148
+ )
149
+
150
+ def session_serializable_tx(
151
+ self, tx_factory: RetryableSessionTransactionFactory, retry_delay: Optional[float] = None
152
+ ) -> SessionTransactionExecutor:
153
+ """
154
+ Get an ORM transaction executor with SERIALIZABLE isolation_level and automatic retry.
155
+ """
156
+ return SessionTransactionExecutor(
157
+ self.session_maker,
158
+ tx_factory,
159
+ engine=self.serializable_engine,
160
+ retry_delay=retry_delay or self._tx_retry_delay,
161
+ )
@@ -0,0 +1,46 @@
1
+ import warnings
2
+ from contextlib import asynccontextmanager
3
+ from typing import AsyncContextManager, Union
4
+
5
+ from sqlalchemy import select, func
6
+ from sqlalchemy.ext.asyncio import AsyncSession, AsyncConnection
7
+
8
+
9
+ async def _pg_advisory_unlock(cos: Union[AsyncConnection, AsyncSession], *keys: int):
10
+ unlocked = (await cos.execute(select(func.pg_advisory_unlock(*keys)))).scalar_one()
11
+ if not unlocked:
12
+ warnings.warn(
13
+ "An inner scope has released the pg_try_advisory_lock() before the lock manager context did so explicitly. "
14
+ "If such is expected, then call the lock manager with 'unlock=False' or restructure the transaction and"
15
+ "locking to be strictly nested."
16
+ )
17
+
18
+
19
+ @asynccontextmanager
20
+ async def pg_try_advisory_lock(
21
+ cos: Union[AsyncConnection, AsyncSession], *keys: int, unlock: bool = True
22
+ ) -> AsyncContextManager[bool]:
23
+ """
24
+ Acquire advisory session lock, yields False if the lock was not acquired.
25
+ """
26
+ locked = (await cos.execute(select(func.pg_try_advisory_lock(*keys)))).scalar_one()
27
+ try:
28
+ yield locked
29
+ finally:
30
+ if locked and unlock:
31
+ await _pg_advisory_unlock(cos, *keys)
32
+
33
+
34
+ @asynccontextmanager
35
+ async def pg_advisory_lock(
36
+ cos: Union[AsyncConnection, AsyncSession], *keys: int, unlock: bool = True
37
+ ) -> AsyncContextManager:
38
+ """
39
+ Acquire advisory session lock, block until acquired.
40
+ """
41
+ (await cos.execute(select(func.pg_advisory_lock(*keys)))).scalar_one()
42
+ try:
43
+ yield
44
+ finally:
45
+ if unlock:
46
+ await _pg_advisory_unlock(cos, *keys)
tarka/asqla/tx.py ADDED
@@ -0,0 +1,106 @@
1
+ import asyncio
2
+ import inspect
3
+ import random
4
+ import sqlite3
5
+ from typing import Any, Callable, Awaitable, Union, Type, Optional
6
+
7
+ from sqlalchemy.exc import DBAPIError, OperationalError
8
+ from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, AsyncConnection
9
+
10
+
11
+ class AbstractRetryableTransaction:
12
+ async def run(self) -> Any:
13
+ raise NotImplementedError()
14
+
15
+
16
+ _MaybeRetryableTransaction = Union[AbstractRetryableTransaction, Awaitable[AbstractRetryableTransaction]]
17
+
18
+
19
+ class AbstractTransactionExecutor:
20
+ def __init__(self, retry_delay: Optional[float] = None):
21
+ self.retry_delay = retry_delay if retry_delay and retry_delay > 0 else 0
22
+
23
+ async def run(self) -> Any:
24
+ while True:
25
+ try:
26
+ return await self._run_tx()
27
+ except OperationalError as e:
28
+ # SQLite: This catches locking errors in both the rollback and WAL modes.
29
+ if (
30
+ isinstance(e.orig, sqlite3.OperationalError)
31
+ and isinstance(e.orig.args, tuple)
32
+ and len(e.orig.args) == 1
33
+ and e.orig.args[0]
34
+ in (
35
+ "database is locked", # normal transaction is in progress
36
+ # Rarely happens when transactions are competing with "BEGIN (DEFERRED)" transactions.
37
+ # Read https://sqlite.org/forum/forumpost/2507664507 for example and more info.
38
+ "cannot start a transaction within a transaction",
39
+ )
40
+ ):
41
+ pass
42
+ else:
43
+ raise e
44
+ except DBAPIError as e:
45
+ # Postgresql: https://www.postgresql.org/docs/current/mvcc-serialization-failure-handling.html
46
+ if e.orig and hasattr(e.orig, "sqlstate") and e.orig.sqlstate == "40001":
47
+ pass
48
+ else:
49
+ raise e
50
+ # TODO: implement more backend specific error handling (MariaDB, etc.)
51
+ if self.retry_delay > 0:
52
+ await asyncio.sleep(self.retry_delay * random.random())
53
+
54
+ async def _run_tx(self) -> Any:
55
+ raise NotImplementedError()
56
+
57
+ async def _check_run_tx(self, tmp: _MaybeRetryableTransaction) -> Any:
58
+ if inspect.isawaitable(tmp):
59
+ tmp = await tmp
60
+ if not isinstance(tmp, AbstractRetryableTransaction):
61
+ raise Exception(f"not AbstractRetryableTransaction: {tmp}")
62
+ return await tmp.run()
63
+
64
+
65
+ RetryableTransactionFactory = Callable[[AsyncConnection], _MaybeRetryableTransaction]
66
+
67
+
68
+ class TransactionExecutor(AbstractTransactionExecutor):
69
+
70
+ def __init__(
71
+ self, engine: AsyncEngine, tx_factory: RetryableTransactionFactory, retry_delay: Optional[float] = None
72
+ ):
73
+ AbstractTransactionExecutor.__init__(self, retry_delay)
74
+ self.engine = engine
75
+ self.tx_factory = tx_factory
76
+
77
+ async def _run_tx(self) -> Any:
78
+ async with self.engine.connect() as conn:
79
+ async with conn.begin():
80
+ return await self._check_run_tx(self.tx_factory(conn))
81
+
82
+
83
+ RetryableSessionTransactionFactory = Callable[[AsyncSession], _MaybeRetryableTransaction]
84
+
85
+
86
+ class SessionTransactionExecutor(AbstractTransactionExecutor):
87
+ def __init__(
88
+ self,
89
+ session_maker: Type[AsyncSession],
90
+ tx_factory: RetryableSessionTransactionFactory,
91
+ engine: Optional[AsyncEngine] = None,
92
+ retry_delay: Optional[float] = None,
93
+ ):
94
+ AbstractTransactionExecutor.__init__(self, retry_delay)
95
+ self.session_maker = session_maker
96
+ self.tx_factory = tx_factory
97
+ self._session_maker_kwargs = {}
98
+ if engine:
99
+ self._session_maker_kwargs["bind"] = engine
100
+
101
+ async def _run_tx(self) -> Any:
102
+ async with self.session_maker(**self._session_maker_kwargs) as session:
103
+ async with session.begin():
104
+ # TODO: The @event.listens_for(self.serializable_engine.sync_engine, "begin") hook for aiosqlite is
105
+ # not called here immediately, so the "BEGIN IMMEDIATE" tx mode does not work well.
106
+ return await self._check_run_tx(self.tx_factory(session))
tarka/asqla/types.py ADDED
@@ -0,0 +1,20 @@
1
+ import datetime
2
+
3
+ from sqlalchemy import TypeDecorator, DateTime
4
+
5
+
6
+ class UTCDateTime(TypeDecorator):
7
+ impl = DateTime
8
+ cache_ok = True
9
+
10
+ def process_bind_param(self, value, dialect):
11
+ if value is not None:
12
+ if not value.tzinfo:
13
+ raise TypeError("tzinfo is required")
14
+ value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None)
15
+ return value
16
+
17
+ def process_result_value(self, value, dialect):
18
+ if value is not None:
19
+ value = value.replace(tzinfo=datetime.timezone.utc)
20
+ return value
File without changes
@@ -0,0 +1,54 @@
1
+ import logging
2
+ from typing import Dict
3
+
4
+ from tarka.logging.handler import NamedHandlersAndHandlerConfigs, StreamHandlerConfig
5
+ from tarka.logging.hook import setup_logging_exception_hooks
6
+ from tarka.logging.level import LoggingLevelOrNamedLoggingLevelConfig, NamedLoggingLevelConfig
7
+ from tarka.logging.manager import LoggerHandlerManager
8
+ from tarka.logging.patch import TarkaLoggingPatcher
9
+
10
+
11
+ def init_tarka_logging() -> None:
12
+ """
13
+ Use this in a top level __init__ module, to ensure the Logger.trace is patched early for use.
14
+ """
15
+ TarkaLoggingPatcher.patch_custom_level(5, "TRACE")
16
+
17
+
18
+ def setup_basic_logging(
19
+ root_handlers: NamedHandlersAndHandlerConfigs = None,
20
+ logger_levels: Dict[str, LoggingLevelOrNamedLoggingLevelConfig] = None,
21
+ setup_exc_hooks: bool = True,
22
+ root_logger_level: LoggingLevelOrNamedLoggingLevelConfig = logging.WARNING,
23
+ ) -> None:
24
+ """
25
+ Convenience function to setup loggers and handlers easily.
26
+ - Setup handlers for the root logger. (because error logging shall be setup globally)
27
+ - Setup levels for the specified loggers.
28
+
29
+ A typical usage example:
30
+ parser = argparse.ArgumentParser()
31
+ parser.add_argument("--verbose", "-v", action="count", default=0)
32
+ parser.add_argument("--silent", "-s", action="count", default=0)
33
+ parser.add_argument("--log_file", help="path to write log to")
34
+ parser.add_argument("--log_backups", type=int, default=16, help="how many log files to keep when rotating")
35
+ args = parser.parse_args()
36
+ setup_basic_logging(
37
+ root_handlers=[RotatingFileHandlerConfig(args.log_file, backup_count=args.log_backups)],
38
+ logger_levels={__name__: NamedLoggingLevelConfig(args.verbose, args.silent, default=logging.INFO)},
39
+ )
40
+
41
+ This should be used as early as possible in the application's lifespan.
42
+ """
43
+ if not root_handlers: # Just select stderr for default.
44
+ root_handlers = [StreamHandlerConfig()]
45
+
46
+ root_lhm = LoggerHandlerManager.instance()
47
+ root_lhm.setup_handlers(root_handlers)
48
+ NamedLoggingLevelConfig.apply(root_lhm.logger, root_logger_level)
49
+
50
+ for logger_name, level in (logger_levels or {}).items():
51
+ NamedLoggingLevelConfig.apply(logging.getLogger(logger_name), level)
52
+
53
+ if setup_exc_hooks:
54
+ setup_logging_exception_hooks()
@@ -0,0 +1,76 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import logging.handlers
5
+ from datetime import datetime
6
+ from typing import Union
7
+
8
+ LOGGER_FORMAT_VERBOSE = "%(asctime)s|%(levelname).1s|%(name)s:%(funcName)s:%(lineno)d| %(message)s"
9
+ LOGGER_FORMAT_VERBOSE_THREAD = "%(asctime)s|%(thread)d|%(levelname).1s|%(name)s:%(funcName)s:%(lineno)d| %(message)s"
10
+ LOGGER_FORMAT_TIME_CONCISE = "%y%m%d %H%M%S"
11
+ LOGGER_FORMAT_MSEC_WITH_DOT = "%s.%03d" # use decimal dot
12
+
13
+
14
+ def create_logging_formatter(
15
+ fmt: str = None,
16
+ time_format: str = None,
17
+ msec_format: str = None,
18
+ concise_asctime: bool = False,
19
+ include_thread: bool = False,
20
+ ) -> logging.Formatter:
21
+ """
22
+ Better default formats and support for overriding the formatting of asctime, which has no API. (datefmt lacks msec)
23
+ """
24
+ if fmt:
25
+ pass
26
+ elif include_thread:
27
+ fmt = LOGGER_FORMAT_VERBOSE_THREAD
28
+ else:
29
+ fmt = LOGGER_FORMAT_VERBOSE
30
+ formatter = logging.Formatter(fmt)
31
+ if formatter.usesTime(): # noqa: this is not publicly documented... should be avaialable since Python 3.2
32
+ if time_format:
33
+ pass
34
+ elif concise_asctime:
35
+ time_format = LOGGER_FORMAT_TIME_CONCISE
36
+ else:
37
+ time_format = logging.Formatter.default_time_format
38
+ if msec_format:
39
+ pass
40
+ else:
41
+ msec_format = LOGGER_FORMAT_MSEC_WITH_DOT
42
+ msec_format % (datetime.now().strftime(time_format), 123456) # validate
43
+ formatter.default_time_format = time_format
44
+ formatter.default_msec_format = msec_format
45
+ return formatter
46
+
47
+
48
+ class HandlerFormatterConfig:
49
+ def __init__(
50
+ self,
51
+ fmt: str = None,
52
+ time_format: str = None,
53
+ msec_format: str = None,
54
+ concise_asctime: bool = False,
55
+ include_thread: bool = False,
56
+ ):
57
+ self.fmt = fmt
58
+ self.time_format = time_format
59
+ self.msec_format = msec_format
60
+ self.concise_asctime = concise_asctime
61
+ self.include_thread = include_thread
62
+
63
+ @property
64
+ def formatter(self) -> logging.Formatter:
65
+ return create_logging_formatter(
66
+ self.fmt, self.time_format, self.msec_format, self.concise_asctime, self.include_thread
67
+ )
68
+
69
+ @classmethod
70
+ def apply(cls, handler: logging.Handler, formatter: HandlerFormatterOrHandlerFormatterConfig) -> None:
71
+ if isinstance(formatter, HandlerFormatterConfig):
72
+ formatter = formatter.formatter
73
+ handler.setFormatter(formatter)
74
+
75
+
76
+ HandlerFormatterOrHandlerFormatterConfig = Union[logging.Formatter, HandlerFormatterConfig]
@@ -0,0 +1,114 @@
1
+ import logging
2
+ import logging.handlers
3
+ import sys
4
+ from io import TextIOWrapper
5
+ from pathlib import Path
6
+ from typing import Union, Sequence, Tuple, TypeVar, Generic, Optional
7
+
8
+ from tarka.logging.formatter import HandlerFormatterOrHandlerFormatterConfig, HandlerFormatterConfig
9
+
10
+ _HandlerT = TypeVar("_HandlerT", bound=logging.Handler)
11
+
12
+
13
+ class AbstractHandlerConfig(Generic[_HandlerT]):
14
+ name: str
15
+
16
+ def __init__(self, formatter: Optional[HandlerFormatterOrHandlerFormatterConfig]):
17
+ self.formatter = formatter or HandlerFormatterConfig()
18
+
19
+ def create(self) -> _HandlerT:
20
+ handler = self._create()
21
+ if self.formatter:
22
+ HandlerFormatterConfig.apply(handler, self.formatter)
23
+ return handler
24
+
25
+ def _create(self) -> _HandlerT:
26
+ raise NotImplementedError()
27
+
28
+
29
+ class RotatingFileHandlerConfig(AbstractHandlerConfig[logging.handlers.RotatingFileHandler]):
30
+ DEFAULT_NAME = "ROTFILE"
31
+
32
+ def __init__(
33
+ self,
34
+ log_path: Union[str, Path],
35
+ log_file_name: str = None,
36
+ max_bytes: int = 4096000,
37
+ backup_count: int = 8,
38
+ ensure_directory: bool = True,
39
+ formatter: Optional[HandlerFormatterOrHandlerFormatterConfig] = None,
40
+ name: str = DEFAULT_NAME,
41
+ ):
42
+ AbstractHandlerConfig.__init__(self, formatter)
43
+ self.name = name
44
+ log_path = Path(log_path)
45
+ if log_file_name is None:
46
+ self.log_directory = log_path.parent
47
+ self.log_file_name = log_path.name
48
+ else:
49
+ self.log_directory = log_path
50
+ self.log_file_name = log_file_name
51
+ self.max_bytes = max_bytes
52
+ self.backup_count = backup_count
53
+ self.ensure_directory = ensure_directory
54
+
55
+ def _create(self) -> logging.handlers.RotatingFileHandler:
56
+ if self.ensure_directory:
57
+ self.log_directory.mkdir(parents=True, exist_ok=True)
58
+ return logging.handlers.RotatingFileHandler(
59
+ str(self.log_directory / self.log_file_name),
60
+ maxBytes=self.max_bytes,
61
+ backupCount=self.backup_count,
62
+ )
63
+
64
+
65
+ class FileHandlerConfig(AbstractHandlerConfig[logging.FileHandler]):
66
+ DEFAULT_NAME = "ONEFILE"
67
+
68
+ def __init__(
69
+ self,
70
+ log_path: Union[str, Path],
71
+ log_file_name: str = None,
72
+ ensure_directory: bool = True,
73
+ formatter: Optional[HandlerFormatterOrHandlerFormatterConfig] = None,
74
+ name: str = DEFAULT_NAME,
75
+ ):
76
+ AbstractHandlerConfig.__init__(self, formatter)
77
+ self.name = name
78
+ log_path = Path(log_path)
79
+ if log_file_name is None:
80
+ self.log_directory = log_path.parent
81
+ self.log_file_name = log_path.name
82
+ else:
83
+ self.log_directory = log_path
84
+ self.log_file_name = log_file_name
85
+ self.ensure_directory = ensure_directory
86
+
87
+ def _create(self) -> logging.FileHandler:
88
+ if self.ensure_directory:
89
+ self.log_directory.mkdir(parents=True, exist_ok=True)
90
+ return logging.FileHandler(str(self.log_directory / self.log_file_name))
91
+
92
+
93
+ class StreamHandlerConfig(AbstractHandlerConfig[logging.StreamHandler]):
94
+ DEFAULT_NAME = "STDERR"
95
+
96
+ def __init__(
97
+ self,
98
+ stream: TextIOWrapper = None,
99
+ formatter: Optional[HandlerFormatterOrHandlerFormatterConfig] = None,
100
+ name: str = DEFAULT_NAME,
101
+ ):
102
+ AbstractHandlerConfig.__init__(self, formatter)
103
+ self.name = name
104
+ # NOTE: We acquire the reference to the stderr stream in the method body, so we'll see the current stream
105
+ # if any post-import monkey-patching is being used.
106
+ if stream is None:
107
+ stream = sys.stderr
108
+ self.stream = stream
109
+
110
+ def _create(self) -> logging.StreamHandler:
111
+ return logging.StreamHandler(self.stream)
112
+
113
+
114
+ NamedHandlersAndHandlerConfigs = Sequence[Union[Tuple[str, logging.Handler], AbstractHandlerConfig]]
tarka/logging/hook.py ADDED
@@ -0,0 +1,37 @@
1
+ import logging
2
+ import sys
3
+ import threading
4
+
5
+
6
+ def logging_excepthook(type_, value, traceback):
7
+ logging.getLogger().critical("Unhandled exception", exc_info=(type_, value, traceback))
8
+
9
+
10
+ def logging_unraisablehook(unraisable):
11
+ logging.getLogger().critical(
12
+ "%s: %r",
13
+ "Exception ignored in" if unraisable.err_msg is None else unraisable.err_msg,
14
+ unraisable.object,
15
+ exc_info=(unraisable.exc_type, unraisable.exc_value, unraisable.exc_traceback),
16
+ )
17
+
18
+
19
+ def logging_threading_excepthook(args):
20
+ if args.exc_type is not SystemExit:
21
+ logging.getLogger().critical(
22
+ "Unhandled exception in %r", args.thread, exc_info=(args.exc_type, args.exc_value, args.exc_traceback)
23
+ )
24
+
25
+
26
+ def setup_logging_exception_hooks(
27
+ excepthook: bool = True, unraisablehook: bool = True, threading_excepthook: bool = True
28
+ ) -> None:
29
+ if excepthook and not hasattr(sys, "original_excepthook"):
30
+ sys.original_excepthook = sys.excepthook
31
+ sys.excepthook = logging_excepthook
32
+ if unraisablehook and not hasattr(sys, "original_unraisablehook"):
33
+ sys.original_unraisablehook = sys.unraisablehook
34
+ sys.unraisablehook = logging_unraisablehook
35
+ if threading_excepthook and not hasattr(threading, "original_excepthook"):
36
+ threading.original_excepthook = threading.excepthook
37
+ threading.excepthook = logging_threading_excepthook