tarka 0.19.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tarka/__init__.py +10 -0
- tarka/asqla/__init__.py +3 -0
- tarka/asqla/alembic.py +47 -0
- tarka/asqla/database.py +161 -0
- tarka/asqla/postgres.py +46 -0
- tarka/asqla/tx.py +106 -0
- tarka/asqla/types.py +20 -0
- tarka/logging/__init__.py +0 -0
- tarka/logging/bootstrap.py +54 -0
- tarka/logging/formatter.py +76 -0
- tarka/logging/handler.py +114 -0
- tarka/logging/hook.py +37 -0
- tarka/logging/level.py +51 -0
- tarka/logging/manager.py +141 -0
- tarka/logging/patch.py +106 -0
- tarka/logging/utility.py +25 -0
- tarka/utility/__init__.py +3 -0
- tarka/utility/aio_sqlite.py +259 -0
- tarka/utility/algorithm/__init__.py +0 -0
- tarka/utility/algorithm/iter.py +253 -0
- tarka/utility/algorithm/seq.py +49 -0
- tarka/utility/algorithm/traverse.py +70 -0
- tarka/utility/envvar.py +33 -0
- tarka/utility/file/__init__.py +2 -0
- tarka/utility/file/name.py +44 -0
- tarka/utility/file/reserve.py +167 -0
- tarka/utility/file/resources.py +37 -0
- tarka/utility/file/safe.py +62 -0
- tarka/utility/importing.py +92 -0
- tarka/utility/sentinel.py +28 -0
- tarka/utility/serialize/__init__.py +0 -0
- tarka/utility/serialize/deterministic.py +220 -0
- tarka/utility/thread.py +84 -0
- tarka/utility/time/__init__.py +0 -0
- tarka/utility/time/format.py +47 -0
- tarka/utility/time/utc.py +125 -0
- tarka/utility/utf8.py +40 -0
- tarka-0.19.1.dist-info/METADATA +40 -0
- tarka-0.19.1.dist-info/RECORD +43 -0
- tarka-0.19.1.dist-info/WHEEL +5 -0
- tarka-0.19.1.dist-info/licenses/LICENSE +177 -0
- tarka-0.19.1.dist-info/licenses/NOTICE +2 -0
- tarka-0.19.1.dist-info/top_level.txt +1 -0
tarka/__init__.py
ADDED
tarka/asqla/__init__.py
ADDED
tarka/asqla/alembic.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from io import StringIO
|
|
3
|
+
|
|
4
|
+
from alembic import command
|
|
5
|
+
from alembic.config import Config
|
|
6
|
+
from alembic.script import ScriptDirectory
|
|
7
|
+
from sqlalchemy import Connection
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_alembic_config(alembic_dir: str, sqlalchemy_url: str = None) -> Config:
|
|
11
|
+
alembic_cfg = Config(os.path.join(alembic_dir, "alembic.ini"))
|
|
12
|
+
alembic_cfg.set_main_option("script_location", alembic_dir.replace("%", "%%"))
|
|
13
|
+
if sqlalchemy_url:
|
|
14
|
+
alembic_cfg.set_main_option("sqlalchemy.url", str(sqlalchemy_url).replace("%", "%%"))
|
|
15
|
+
alembic_cfg.attributes["skip-logging-setup"] = True # do not mess up the server logging
|
|
16
|
+
return alembic_cfg
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class NoHeadRevision(Exception):
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class AlembicHelper:
|
|
24
|
+
def __init__(self, config: Config):
|
|
25
|
+
self.config = config
|
|
26
|
+
|
|
27
|
+
def run(self, conn: Connection, alembic_command: str, *args: str) -> str:
|
|
28
|
+
"""
|
|
29
|
+
Call the alembic CLI.
|
|
30
|
+
"""
|
|
31
|
+
self.config.attributes["connection"] = conn
|
|
32
|
+
self.config.stdout = StringIO()
|
|
33
|
+
getattr(command, alembic_command)(self.config, *args)
|
|
34
|
+
return self.config.stdout.getvalue()
|
|
35
|
+
|
|
36
|
+
def run_strip_output(self, conn: Connection, alembic_command: str, *args: str) -> str:
|
|
37
|
+
return self.run(conn, alembic_command, *args).strip()
|
|
38
|
+
|
|
39
|
+
def get_head_revision(self) -> str:
|
|
40
|
+
"""
|
|
41
|
+
Get current single head revision.
|
|
42
|
+
"""
|
|
43
|
+
script = ScriptDirectory.from_config(self.config)
|
|
44
|
+
revs = list(script.get_revisions("head"))
|
|
45
|
+
if len(revs) != 1:
|
|
46
|
+
raise NoHeadRevision()
|
|
47
|
+
return revs[0].revision
|
tarka/asqla/database.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from contextlib import asynccontextmanager
|
|
3
|
+
from typing import Type, Dict, Any, Optional
|
|
4
|
+
|
|
5
|
+
from alembic.config import Config
|
|
6
|
+
from sqlalchemy import event
|
|
7
|
+
from sqlalchemy.exc import DatabaseError
|
|
8
|
+
from sqlalchemy.ext.asyncio import create_async_engine, AsyncConnection, AsyncSession, AsyncEngine
|
|
9
|
+
from sqlalchemy.orm import sessionmaker
|
|
10
|
+
|
|
11
|
+
from tarka.asqla.alembic import get_alembic_config, AlembicHelper
|
|
12
|
+
from tarka.asqla.tx import (
|
|
13
|
+
TransactionExecutor,
|
|
14
|
+
RetryableTransactionFactory,
|
|
15
|
+
SessionTransactionExecutor,
|
|
16
|
+
RetryableSessionTransactionFactory,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Database(object):
|
|
21
|
+
engine: AsyncEngine = None
|
|
22
|
+
serializable_engine: AsyncEngine = None
|
|
23
|
+
session_maker: Type[AsyncSession] = None
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
alembic_dir: str,
|
|
28
|
+
connect_url: str,
|
|
29
|
+
engine_kwargs: Dict[str, Any] = None, # like echo, connect_args, etc.
|
|
30
|
+
session_maker_kwargs: Dict[str, Any] = None, # like expire_on_commit, autoflush, etc.
|
|
31
|
+
aiosqlite_serializable_begin: str = "BEGIN", # serializable tx support workaround for aiosqlite
|
|
32
|
+
tx_retry_delay: Optional[float] = None, # tx retry delay helps to relieve db pressure at conflicts
|
|
33
|
+
):
|
|
34
|
+
"""
|
|
35
|
+
If aiosqlite is used aiosqlite_serializable_begin="BEGIN IMMEDIATE" could be better, because then no
|
|
36
|
+
conflicts can happen, each tx caller will wait in line to acquire the lock and no resources will be
|
|
37
|
+
wasted retrying. However, this does not work well with session-tx, see TODO: tx.py:105.
|
|
38
|
+
"""
|
|
39
|
+
self.alembic_dir = alembic_dir
|
|
40
|
+
self._connect_url = connect_url
|
|
41
|
+
self._engine_kwargs = engine_kwargs
|
|
42
|
+
self._session_maker_kwargs = session_maker_kwargs
|
|
43
|
+
self._aiosqlite_serializable_begin = aiosqlite_serializable_begin
|
|
44
|
+
self._tx_retry_delay = tx_retry_delay
|
|
45
|
+
self.alembic_head_at_startup = ""
|
|
46
|
+
|
|
47
|
+
def get_alembic_config(self) -> Config:
|
|
48
|
+
return get_alembic_config(self.alembic_dir, str(self.engine.sync_engine.url))
|
|
49
|
+
|
|
50
|
+
async def _init_engine(self):
|
|
51
|
+
"""
|
|
52
|
+
If further customization is necessary for the engines, this shall be overridden.
|
|
53
|
+
To switch SQLite to WAL mode for example:
|
|
54
|
+
|
|
55
|
+
if self.engine.dialect.name == "sqlite":
|
|
56
|
+
@event.listens_for(self.engine.sync_engine, "connect")
|
|
57
|
+
def _setup_sqlite_connection(dbapi_con, con_record):
|
|
58
|
+
dbapi_con.execute("PRAGMA journal_mode = WAL;")
|
|
59
|
+
dbapi_con.execute("PRAGMA synchronous = NORMAL;")
|
|
60
|
+
"""
|
|
61
|
+
self.engine = create_async_engine(self._connect_url, **(self._engine_kwargs or {}))
|
|
62
|
+
self.serializable_engine = self.engine.execution_options(isolation_level="SERIALIZABLE")
|
|
63
|
+
if self.engine.driver == "aiosqlite" and self._aiosqlite_serializable_begin:
|
|
64
|
+
# aiosqlite (<=0.20) does not honor the isolation_level with appropriate BEGIN commands, in fact it
|
|
65
|
+
# never issues begin commands, transactions always start implicitly for reads.
|
|
66
|
+
# SQLite itself does not support concurrent mutating transactions in any mode. In WAL journal_mode, there
|
|
67
|
+
# can be parallel and concurrent readers, but writing is always with an exclusive lock. In this sense "any"
|
|
68
|
+
# transaction will actually be serializable provided that an explicit BEGIN is emitted. The actual
|
|
69
|
+
# difference is the point and time that parallel connections encounter locking errors.
|
|
70
|
+
# The workaround (if not disabled) is to explicitly execute a BEGIN command at transaction start.
|
|
71
|
+
# The command can be set for the database (DEFERRED, IMMEDIATE or EXCLUSIVE) when relevant.
|
|
72
|
+
@event.listens_for(self.serializable_engine.sync_engine, "begin")
|
|
73
|
+
def do_begin(conn):
|
|
74
|
+
conn.exec_driver_sql(self._aiosqlite_serializable_begin)
|
|
75
|
+
|
|
76
|
+
self.session_maker = sessionmaker(self.engine, class_=AsyncSession, **(self._session_maker_kwargs or {}))
|
|
77
|
+
|
|
78
|
+
async def startup(self):
|
|
79
|
+
"""
|
|
80
|
+
Initialize engine first, then bootstrap or migrate schema of the database to be up-to-date.
|
|
81
|
+
"""
|
|
82
|
+
await self._init_engine()
|
|
83
|
+
alembic_helper = AlembicHelper(self.get_alembic_config())
|
|
84
|
+
retry = 0
|
|
85
|
+
while True: # handle conflicting migration attempt by parallel workers
|
|
86
|
+
try:
|
|
87
|
+
async with self.engine.begin() as connection:
|
|
88
|
+
connection: AsyncConnection
|
|
89
|
+
self.alembic_head_at_startup = await connection.run_sync(alembic_helper.run_strip_output, "current")
|
|
90
|
+
|
|
91
|
+
await self._upgrade(alembic_helper, connection)
|
|
92
|
+
await self._start_hook(connection)
|
|
93
|
+
except DatabaseError:
|
|
94
|
+
if retry >= 5:
|
|
95
|
+
raise
|
|
96
|
+
retry += 1
|
|
97
|
+
await asyncio.sleep(0.25)
|
|
98
|
+
else:
|
|
99
|
+
break
|
|
100
|
+
|
|
101
|
+
async def _upgrade(self, alembic_helper: AlembicHelper, connection: AsyncConnection):
|
|
102
|
+
if not self.alembic_head_at_startup.endswith(" (head)"):
|
|
103
|
+
await self._pre_upgrade_hook(connection)
|
|
104
|
+
await connection.run_sync(alembic_helper.run, "upgrade", "head")
|
|
105
|
+
await self._post_upgrade_hook(connection)
|
|
106
|
+
|
|
107
|
+
async def _pre_upgrade_hook(self, connection: AsyncConnection):
|
|
108
|
+
"""
|
|
109
|
+
Place to put custom DDL commands before alembic upgrade.
|
|
110
|
+
Keep in mind that it is usually better to implement all changes as alembic revisions.
|
|
111
|
+
NOTE: This is called before bootstrap (first upgrade) as well.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
async def _post_upgrade_hook(self, connection: AsyncConnection):
|
|
115
|
+
"""
|
|
116
|
+
Place to put custom DDL commands after alembic upgrade.
|
|
117
|
+
Keep in mind that it is usually better to implement all changes as alembic revisions.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
async def _start_hook(self, connection: AsyncConnection):
|
|
121
|
+
"""
|
|
122
|
+
Place to put custom DDL commands that are executed each time a server starts.
|
|
123
|
+
The connection is currently at the HEAD alembic revision, in the upgrade transaction.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
async def shutdown(self):
|
|
127
|
+
await self.engine.dispose()
|
|
128
|
+
|
|
129
|
+
@asynccontextmanager
|
|
130
|
+
async def run(self):
|
|
131
|
+
"""
|
|
132
|
+
Convenience wrapper for startup & shutdown.
|
|
133
|
+
"""
|
|
134
|
+
await self.startup()
|
|
135
|
+
try:
|
|
136
|
+
yield self
|
|
137
|
+
finally:
|
|
138
|
+
await self.shutdown()
|
|
139
|
+
|
|
140
|
+
def serializable_tx(
|
|
141
|
+
self, tx_factory: RetryableTransactionFactory, retry_delay: Optional[float] = None
|
|
142
|
+
) -> TransactionExecutor:
|
|
143
|
+
"""
|
|
144
|
+
Get a transaction executor with SERIALIZABLE isolation_level and automatic retry.
|
|
145
|
+
"""
|
|
146
|
+
return TransactionExecutor(
|
|
147
|
+
self.serializable_engine, tx_factory, retry_delay=retry_delay or self._tx_retry_delay
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def session_serializable_tx(
|
|
151
|
+
self, tx_factory: RetryableSessionTransactionFactory, retry_delay: Optional[float] = None
|
|
152
|
+
) -> SessionTransactionExecutor:
|
|
153
|
+
"""
|
|
154
|
+
Get an ORM transaction executor with SERIALIZABLE isolation_level and automatic retry.
|
|
155
|
+
"""
|
|
156
|
+
return SessionTransactionExecutor(
|
|
157
|
+
self.session_maker,
|
|
158
|
+
tx_factory,
|
|
159
|
+
engine=self.serializable_engine,
|
|
160
|
+
retry_delay=retry_delay or self._tx_retry_delay,
|
|
161
|
+
)
|
tarka/asqla/postgres.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from contextlib import asynccontextmanager
|
|
3
|
+
from typing import AsyncContextManager, Union
|
|
4
|
+
|
|
5
|
+
from sqlalchemy import select, func
|
|
6
|
+
from sqlalchemy.ext.asyncio import AsyncSession, AsyncConnection
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def _pg_advisory_unlock(cos: Union[AsyncConnection, AsyncSession], *keys: int):
|
|
10
|
+
unlocked = (await cos.execute(select(func.pg_advisory_unlock(*keys)))).scalar_one()
|
|
11
|
+
if not unlocked:
|
|
12
|
+
warnings.warn(
|
|
13
|
+
"An inner scope has released the pg_try_advisory_lock() before the lock manager context did so explicitly. "
|
|
14
|
+
"If such is expected, then call the lock manager with 'unlock=False' or restructure the transaction and"
|
|
15
|
+
"locking to be strictly nested."
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@asynccontextmanager
|
|
20
|
+
async def pg_try_advisory_lock(
|
|
21
|
+
cos: Union[AsyncConnection, AsyncSession], *keys: int, unlock: bool = True
|
|
22
|
+
) -> AsyncContextManager[bool]:
|
|
23
|
+
"""
|
|
24
|
+
Acquire advisory session lock, yields False if the lock was not acquired.
|
|
25
|
+
"""
|
|
26
|
+
locked = (await cos.execute(select(func.pg_try_advisory_lock(*keys)))).scalar_one()
|
|
27
|
+
try:
|
|
28
|
+
yield locked
|
|
29
|
+
finally:
|
|
30
|
+
if locked and unlock:
|
|
31
|
+
await _pg_advisory_unlock(cos, *keys)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@asynccontextmanager
|
|
35
|
+
async def pg_advisory_lock(
|
|
36
|
+
cos: Union[AsyncConnection, AsyncSession], *keys: int, unlock: bool = True
|
|
37
|
+
) -> AsyncContextManager:
|
|
38
|
+
"""
|
|
39
|
+
Acquire advisory session lock, block until acquired.
|
|
40
|
+
"""
|
|
41
|
+
(await cos.execute(select(func.pg_advisory_lock(*keys)))).scalar_one()
|
|
42
|
+
try:
|
|
43
|
+
yield
|
|
44
|
+
finally:
|
|
45
|
+
if unlock:
|
|
46
|
+
await _pg_advisory_unlock(cos, *keys)
|
tarka/asqla/tx.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import inspect
|
|
3
|
+
import random
|
|
4
|
+
import sqlite3
|
|
5
|
+
from typing import Any, Callable, Awaitable, Union, Type, Optional
|
|
6
|
+
|
|
7
|
+
from sqlalchemy.exc import DBAPIError, OperationalError
|
|
8
|
+
from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, AsyncConnection
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AbstractRetryableTransaction:
|
|
12
|
+
async def run(self) -> Any:
|
|
13
|
+
raise NotImplementedError()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_MaybeRetryableTransaction = Union[AbstractRetryableTransaction, Awaitable[AbstractRetryableTransaction]]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AbstractTransactionExecutor:
|
|
20
|
+
def __init__(self, retry_delay: Optional[float] = None):
|
|
21
|
+
self.retry_delay = retry_delay if retry_delay and retry_delay > 0 else 0
|
|
22
|
+
|
|
23
|
+
async def run(self) -> Any:
|
|
24
|
+
while True:
|
|
25
|
+
try:
|
|
26
|
+
return await self._run_tx()
|
|
27
|
+
except OperationalError as e:
|
|
28
|
+
# SQLite: This catches locking errors in both the rollback and WAL modes.
|
|
29
|
+
if (
|
|
30
|
+
isinstance(e.orig, sqlite3.OperationalError)
|
|
31
|
+
and isinstance(e.orig.args, tuple)
|
|
32
|
+
and len(e.orig.args) == 1
|
|
33
|
+
and e.orig.args[0]
|
|
34
|
+
in (
|
|
35
|
+
"database is locked", # normal transaction is in progress
|
|
36
|
+
# Rarely happens when transactions are competing with "BEGIN (DEFERRED)" transactions.
|
|
37
|
+
# Read https://sqlite.org/forum/forumpost/2507664507 for example and more info.
|
|
38
|
+
"cannot start a transaction within a transaction",
|
|
39
|
+
)
|
|
40
|
+
):
|
|
41
|
+
pass
|
|
42
|
+
else:
|
|
43
|
+
raise e
|
|
44
|
+
except DBAPIError as e:
|
|
45
|
+
# Postgresql: https://www.postgresql.org/docs/current/mvcc-serialization-failure-handling.html
|
|
46
|
+
if e.orig and hasattr(e.orig, "sqlstate") and e.orig.sqlstate == "40001":
|
|
47
|
+
pass
|
|
48
|
+
else:
|
|
49
|
+
raise e
|
|
50
|
+
# TODO: implement more backend specific error handling (MariaDB, etc.)
|
|
51
|
+
if self.retry_delay > 0:
|
|
52
|
+
await asyncio.sleep(self.retry_delay * random.random())
|
|
53
|
+
|
|
54
|
+
async def _run_tx(self) -> Any:
|
|
55
|
+
raise NotImplementedError()
|
|
56
|
+
|
|
57
|
+
async def _check_run_tx(self, tmp: _MaybeRetryableTransaction) -> Any:
|
|
58
|
+
if inspect.isawaitable(tmp):
|
|
59
|
+
tmp = await tmp
|
|
60
|
+
if not isinstance(tmp, AbstractRetryableTransaction):
|
|
61
|
+
raise Exception(f"not AbstractRetryableTransaction: {tmp}")
|
|
62
|
+
return await tmp.run()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
RetryableTransactionFactory = Callable[[AsyncConnection], _MaybeRetryableTransaction]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class TransactionExecutor(AbstractTransactionExecutor):
|
|
69
|
+
|
|
70
|
+
def __init__(
|
|
71
|
+
self, engine: AsyncEngine, tx_factory: RetryableTransactionFactory, retry_delay: Optional[float] = None
|
|
72
|
+
):
|
|
73
|
+
AbstractTransactionExecutor.__init__(self, retry_delay)
|
|
74
|
+
self.engine = engine
|
|
75
|
+
self.tx_factory = tx_factory
|
|
76
|
+
|
|
77
|
+
async def _run_tx(self) -> Any:
|
|
78
|
+
async with self.engine.connect() as conn:
|
|
79
|
+
async with conn.begin():
|
|
80
|
+
return await self._check_run_tx(self.tx_factory(conn))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
RetryableSessionTransactionFactory = Callable[[AsyncSession], _MaybeRetryableTransaction]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class SessionTransactionExecutor(AbstractTransactionExecutor):
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
session_maker: Type[AsyncSession],
|
|
90
|
+
tx_factory: RetryableSessionTransactionFactory,
|
|
91
|
+
engine: Optional[AsyncEngine] = None,
|
|
92
|
+
retry_delay: Optional[float] = None,
|
|
93
|
+
):
|
|
94
|
+
AbstractTransactionExecutor.__init__(self, retry_delay)
|
|
95
|
+
self.session_maker = session_maker
|
|
96
|
+
self.tx_factory = tx_factory
|
|
97
|
+
self._session_maker_kwargs = {}
|
|
98
|
+
if engine:
|
|
99
|
+
self._session_maker_kwargs["bind"] = engine
|
|
100
|
+
|
|
101
|
+
async def _run_tx(self) -> Any:
|
|
102
|
+
async with self.session_maker(**self._session_maker_kwargs) as session:
|
|
103
|
+
async with session.begin():
|
|
104
|
+
# TODO: The @event.listens_for(self.serializable_engine.sync_engine, "begin") hook for aiosqlite is
|
|
105
|
+
# not called here immediately, so the "BEGIN IMMEDIATE" tx mode does not work well.
|
|
106
|
+
return await self._check_run_tx(self.tx_factory(session))
|
tarka/asqla/types.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
|
|
3
|
+
from sqlalchemy import TypeDecorator, DateTime
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class UTCDateTime(TypeDecorator):
|
|
7
|
+
impl = DateTime
|
|
8
|
+
cache_ok = True
|
|
9
|
+
|
|
10
|
+
def process_bind_param(self, value, dialect):
|
|
11
|
+
if value is not None:
|
|
12
|
+
if not value.tzinfo:
|
|
13
|
+
raise TypeError("tzinfo is required")
|
|
14
|
+
value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None)
|
|
15
|
+
return value
|
|
16
|
+
|
|
17
|
+
def process_result_value(self, value, dialect):
|
|
18
|
+
if value is not None:
|
|
19
|
+
value = value.replace(tzinfo=datetime.timezone.utc)
|
|
20
|
+
return value
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Dict
|
|
3
|
+
|
|
4
|
+
from tarka.logging.handler import NamedHandlersAndHandlerConfigs, StreamHandlerConfig
|
|
5
|
+
from tarka.logging.hook import setup_logging_exception_hooks
|
|
6
|
+
from tarka.logging.level import LoggingLevelOrNamedLoggingLevelConfig, NamedLoggingLevelConfig
|
|
7
|
+
from tarka.logging.manager import LoggerHandlerManager
|
|
8
|
+
from tarka.logging.patch import TarkaLoggingPatcher
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def init_tarka_logging() -> None:
|
|
12
|
+
"""
|
|
13
|
+
Use this in a top level __init__ module, to ensure the Logger.trace is patched early for use.
|
|
14
|
+
"""
|
|
15
|
+
TarkaLoggingPatcher.patch_custom_level(5, "TRACE")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def setup_basic_logging(
|
|
19
|
+
root_handlers: NamedHandlersAndHandlerConfigs = None,
|
|
20
|
+
logger_levels: Dict[str, LoggingLevelOrNamedLoggingLevelConfig] = None,
|
|
21
|
+
setup_exc_hooks: bool = True,
|
|
22
|
+
root_logger_level: LoggingLevelOrNamedLoggingLevelConfig = logging.WARNING,
|
|
23
|
+
) -> None:
|
|
24
|
+
"""
|
|
25
|
+
Convenience function to setup loggers and handlers easily.
|
|
26
|
+
- Setup handlers for the root logger. (because error logging shall be setup globally)
|
|
27
|
+
- Setup levels for the specified loggers.
|
|
28
|
+
|
|
29
|
+
A typical usage example:
|
|
30
|
+
parser = argparse.ArgumentParser()
|
|
31
|
+
parser.add_argument("--verbose", "-v", action="count", default=0)
|
|
32
|
+
parser.add_argument("--silent", "-s", action="count", default=0)
|
|
33
|
+
parser.add_argument("--log_file", help="path to write log to")
|
|
34
|
+
parser.add_argument("--log_backups", type=int, default=16, help="how many log files to keep when rotating")
|
|
35
|
+
args = parser.parse_args()
|
|
36
|
+
setup_basic_logging(
|
|
37
|
+
root_handlers=[RotatingFileHandlerConfig(args.log_file, backup_count=args.log_backups)],
|
|
38
|
+
logger_levels={__name__: NamedLoggingLevelConfig(args.verbose, args.silent, default=logging.INFO)},
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
This should be used as early as possible in the application's lifespan.
|
|
42
|
+
"""
|
|
43
|
+
if not root_handlers: # Just select stderr for default.
|
|
44
|
+
root_handlers = [StreamHandlerConfig()]
|
|
45
|
+
|
|
46
|
+
root_lhm = LoggerHandlerManager.instance()
|
|
47
|
+
root_lhm.setup_handlers(root_handlers)
|
|
48
|
+
NamedLoggingLevelConfig.apply(root_lhm.logger, root_logger_level)
|
|
49
|
+
|
|
50
|
+
for logger_name, level in (logger_levels or {}).items():
|
|
51
|
+
NamedLoggingLevelConfig.apply(logging.getLogger(logger_name), level)
|
|
52
|
+
|
|
53
|
+
if setup_exc_hooks:
|
|
54
|
+
setup_logging_exception_hooks()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import logging.handlers
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from typing import Union
|
|
7
|
+
|
|
8
|
+
LOGGER_FORMAT_VERBOSE = "%(asctime)s|%(levelname).1s|%(name)s:%(funcName)s:%(lineno)d| %(message)s"
|
|
9
|
+
LOGGER_FORMAT_VERBOSE_THREAD = "%(asctime)s|%(thread)d|%(levelname).1s|%(name)s:%(funcName)s:%(lineno)d| %(message)s"
|
|
10
|
+
LOGGER_FORMAT_TIME_CONCISE = "%y%m%d %H%M%S"
|
|
11
|
+
LOGGER_FORMAT_MSEC_WITH_DOT = "%s.%03d" # use decimal dot
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def create_logging_formatter(
|
|
15
|
+
fmt: str = None,
|
|
16
|
+
time_format: str = None,
|
|
17
|
+
msec_format: str = None,
|
|
18
|
+
concise_asctime: bool = False,
|
|
19
|
+
include_thread: bool = False,
|
|
20
|
+
) -> logging.Formatter:
|
|
21
|
+
"""
|
|
22
|
+
Better default formats and support for overriding the formatting of asctime, which has no API. (datefmt lacks msec)
|
|
23
|
+
"""
|
|
24
|
+
if fmt:
|
|
25
|
+
pass
|
|
26
|
+
elif include_thread:
|
|
27
|
+
fmt = LOGGER_FORMAT_VERBOSE_THREAD
|
|
28
|
+
else:
|
|
29
|
+
fmt = LOGGER_FORMAT_VERBOSE
|
|
30
|
+
formatter = logging.Formatter(fmt)
|
|
31
|
+
if formatter.usesTime(): # noqa: this is not publicly documented... should be avaialable since Python 3.2
|
|
32
|
+
if time_format:
|
|
33
|
+
pass
|
|
34
|
+
elif concise_asctime:
|
|
35
|
+
time_format = LOGGER_FORMAT_TIME_CONCISE
|
|
36
|
+
else:
|
|
37
|
+
time_format = logging.Formatter.default_time_format
|
|
38
|
+
if msec_format:
|
|
39
|
+
pass
|
|
40
|
+
else:
|
|
41
|
+
msec_format = LOGGER_FORMAT_MSEC_WITH_DOT
|
|
42
|
+
msec_format % (datetime.now().strftime(time_format), 123456) # validate
|
|
43
|
+
formatter.default_time_format = time_format
|
|
44
|
+
formatter.default_msec_format = msec_format
|
|
45
|
+
return formatter
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class HandlerFormatterConfig:
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
fmt: str = None,
|
|
52
|
+
time_format: str = None,
|
|
53
|
+
msec_format: str = None,
|
|
54
|
+
concise_asctime: bool = False,
|
|
55
|
+
include_thread: bool = False,
|
|
56
|
+
):
|
|
57
|
+
self.fmt = fmt
|
|
58
|
+
self.time_format = time_format
|
|
59
|
+
self.msec_format = msec_format
|
|
60
|
+
self.concise_asctime = concise_asctime
|
|
61
|
+
self.include_thread = include_thread
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def formatter(self) -> logging.Formatter:
|
|
65
|
+
return create_logging_formatter(
|
|
66
|
+
self.fmt, self.time_format, self.msec_format, self.concise_asctime, self.include_thread
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
@classmethod
|
|
70
|
+
def apply(cls, handler: logging.Handler, formatter: HandlerFormatterOrHandlerFormatterConfig) -> None:
|
|
71
|
+
if isinstance(formatter, HandlerFormatterConfig):
|
|
72
|
+
formatter = formatter.formatter
|
|
73
|
+
handler.setFormatter(formatter)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
HandlerFormatterOrHandlerFormatterConfig = Union[logging.Formatter, HandlerFormatterConfig]
|
tarka/logging/handler.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import logging.handlers
|
|
3
|
+
import sys
|
|
4
|
+
from io import TextIOWrapper
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Union, Sequence, Tuple, TypeVar, Generic, Optional
|
|
7
|
+
|
|
8
|
+
from tarka.logging.formatter import HandlerFormatterOrHandlerFormatterConfig, HandlerFormatterConfig
|
|
9
|
+
|
|
10
|
+
_HandlerT = TypeVar("_HandlerT", bound=logging.Handler)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AbstractHandlerConfig(Generic[_HandlerT]):
|
|
14
|
+
name: str
|
|
15
|
+
|
|
16
|
+
def __init__(self, formatter: Optional[HandlerFormatterOrHandlerFormatterConfig]):
|
|
17
|
+
self.formatter = formatter or HandlerFormatterConfig()
|
|
18
|
+
|
|
19
|
+
def create(self) -> _HandlerT:
|
|
20
|
+
handler = self._create()
|
|
21
|
+
if self.formatter:
|
|
22
|
+
HandlerFormatterConfig.apply(handler, self.formatter)
|
|
23
|
+
return handler
|
|
24
|
+
|
|
25
|
+
def _create(self) -> _HandlerT:
|
|
26
|
+
raise NotImplementedError()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class RotatingFileHandlerConfig(AbstractHandlerConfig[logging.handlers.RotatingFileHandler]):
|
|
30
|
+
DEFAULT_NAME = "ROTFILE"
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
log_path: Union[str, Path],
|
|
35
|
+
log_file_name: str = None,
|
|
36
|
+
max_bytes: int = 4096000,
|
|
37
|
+
backup_count: int = 8,
|
|
38
|
+
ensure_directory: bool = True,
|
|
39
|
+
formatter: Optional[HandlerFormatterOrHandlerFormatterConfig] = None,
|
|
40
|
+
name: str = DEFAULT_NAME,
|
|
41
|
+
):
|
|
42
|
+
AbstractHandlerConfig.__init__(self, formatter)
|
|
43
|
+
self.name = name
|
|
44
|
+
log_path = Path(log_path)
|
|
45
|
+
if log_file_name is None:
|
|
46
|
+
self.log_directory = log_path.parent
|
|
47
|
+
self.log_file_name = log_path.name
|
|
48
|
+
else:
|
|
49
|
+
self.log_directory = log_path
|
|
50
|
+
self.log_file_name = log_file_name
|
|
51
|
+
self.max_bytes = max_bytes
|
|
52
|
+
self.backup_count = backup_count
|
|
53
|
+
self.ensure_directory = ensure_directory
|
|
54
|
+
|
|
55
|
+
def _create(self) -> logging.handlers.RotatingFileHandler:
|
|
56
|
+
if self.ensure_directory:
|
|
57
|
+
self.log_directory.mkdir(parents=True, exist_ok=True)
|
|
58
|
+
return logging.handlers.RotatingFileHandler(
|
|
59
|
+
str(self.log_directory / self.log_file_name),
|
|
60
|
+
maxBytes=self.max_bytes,
|
|
61
|
+
backupCount=self.backup_count,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class FileHandlerConfig(AbstractHandlerConfig[logging.FileHandler]):
|
|
66
|
+
DEFAULT_NAME = "ONEFILE"
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
log_path: Union[str, Path],
|
|
71
|
+
log_file_name: str = None,
|
|
72
|
+
ensure_directory: bool = True,
|
|
73
|
+
formatter: Optional[HandlerFormatterOrHandlerFormatterConfig] = None,
|
|
74
|
+
name: str = DEFAULT_NAME,
|
|
75
|
+
):
|
|
76
|
+
AbstractHandlerConfig.__init__(self, formatter)
|
|
77
|
+
self.name = name
|
|
78
|
+
log_path = Path(log_path)
|
|
79
|
+
if log_file_name is None:
|
|
80
|
+
self.log_directory = log_path.parent
|
|
81
|
+
self.log_file_name = log_path.name
|
|
82
|
+
else:
|
|
83
|
+
self.log_directory = log_path
|
|
84
|
+
self.log_file_name = log_file_name
|
|
85
|
+
self.ensure_directory = ensure_directory
|
|
86
|
+
|
|
87
|
+
def _create(self) -> logging.FileHandler:
|
|
88
|
+
if self.ensure_directory:
|
|
89
|
+
self.log_directory.mkdir(parents=True, exist_ok=True)
|
|
90
|
+
return logging.FileHandler(str(self.log_directory / self.log_file_name))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class StreamHandlerConfig(AbstractHandlerConfig[logging.StreamHandler]):
|
|
94
|
+
DEFAULT_NAME = "STDERR"
|
|
95
|
+
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
stream: TextIOWrapper = None,
|
|
99
|
+
formatter: Optional[HandlerFormatterOrHandlerFormatterConfig] = None,
|
|
100
|
+
name: str = DEFAULT_NAME,
|
|
101
|
+
):
|
|
102
|
+
AbstractHandlerConfig.__init__(self, formatter)
|
|
103
|
+
self.name = name
|
|
104
|
+
# NOTE: We acquire the reference to the stderr stream in the method body, so we'll see the current stream
|
|
105
|
+
# if any post-import monkey-patching is being used.
|
|
106
|
+
if stream is None:
|
|
107
|
+
stream = sys.stderr
|
|
108
|
+
self.stream = stream
|
|
109
|
+
|
|
110
|
+
def _create(self) -> logging.StreamHandler:
|
|
111
|
+
return logging.StreamHandler(self.stream)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
NamedHandlersAndHandlerConfigs = Sequence[Union[Tuple[str, logging.Handler], AbstractHandlerConfig]]
|
tarka/logging/hook.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
import threading
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def logging_excepthook(type_, value, traceback):
|
|
7
|
+
logging.getLogger().critical("Unhandled exception", exc_info=(type_, value, traceback))
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def logging_unraisablehook(unraisable):
|
|
11
|
+
logging.getLogger().critical(
|
|
12
|
+
"%s: %r",
|
|
13
|
+
"Exception ignored in" if unraisable.err_msg is None else unraisable.err_msg,
|
|
14
|
+
unraisable.object,
|
|
15
|
+
exc_info=(unraisable.exc_type, unraisable.exc_value, unraisable.exc_traceback),
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def logging_threading_excepthook(args):
|
|
20
|
+
if args.exc_type is not SystemExit:
|
|
21
|
+
logging.getLogger().critical(
|
|
22
|
+
"Unhandled exception in %r", args.thread, exc_info=(args.exc_type, args.exc_value, args.exc_traceback)
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def setup_logging_exception_hooks(
|
|
27
|
+
excepthook: bool = True, unraisablehook: bool = True, threading_excepthook: bool = True
|
|
28
|
+
) -> None:
|
|
29
|
+
if excepthook and not hasattr(sys, "original_excepthook"):
|
|
30
|
+
sys.original_excepthook = sys.excepthook
|
|
31
|
+
sys.excepthook = logging_excepthook
|
|
32
|
+
if unraisablehook and not hasattr(sys, "original_unraisablehook"):
|
|
33
|
+
sys.original_unraisablehook = sys.unraisablehook
|
|
34
|
+
sys.unraisablehook = logging_unraisablehook
|
|
35
|
+
if threading_excepthook and not hasattr(threading, "original_excepthook"):
|
|
36
|
+
threading.original_excepthook = threading.excepthook
|
|
37
|
+
threading.excepthook = logging_threading_excepthook
|