GeneralManager 0.55.0__tar.gz → 0.56.0__tar.gz

{generalmanager-0.55.0/src/GeneralManager.egg-info → generalmanager-0.56.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.55.0
+Version: 0.56.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.55.0 → generalmanager-0.56.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.55.0"
+version = "0.56.0"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]

{generalmanager-0.55.0 → generalmanager-0.56.0/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.55.0
+Version: 0.56.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.55.0 → generalmanager-0.56.0}/src/GeneralManager.egg-info/SOURCES.txt

@@ -37,6 +37,9 @@ src/general_manager/api/graphql_search.py
 src/general_manager/api/graphql_subscription_consumer.py
 src/general_manager/api/graphql_subscriptions.py
 src/general_manager/api/graphql_view.py
+src/general_manager/api/graphql_warmup.py
+src/general_manager/api/graphql_warmup_registry.py
+src/general_manager/api/graphql_warmup_tasks.py
 src/general_manager/api/mutation.py
 src/general_manager/api/property.py
 src/general_manager/api/registry.py
@@ -123,6 +126,8 @@ src/general_manager/interface/utils/__init__.py
 src/general_manager/interface/utils/database_interface_protocols.py
 src/general_manager/interface/utils/errors.py
 src/general_manager/interface/utils/models.py
+src/general_manager/management/commands/graphql_warmup.py
+src/general_manager/management/commands/graphql_warmup_refresh_due.py
 src/general_manager/management/commands/search_index.py
 src/general_manager/management/commands/search_reconcile.py
 src/general_manager/management/commands/seed_manager_landscape.py

generalmanager-0.56.0/src/general_manager/api/graphql_warmup.py

@@ -0,0 +1,302 @@
+"""GraphQL property warm-up execution helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import Any
+
+from django.core.cache import cache as django_cache
+from django.utils import timezone
+from django.utils.module_loading import import_string
+
+from general_manager.api.property import GraphQLProperty
+from general_manager.cache.run_context import CalculationRunContext
+from general_manager.conf import get_setting
+from general_manager.logging import get_logger
+from general_manager.utils.make_cache_key import make_cache_key
+from general_manager.api.graphql_warmup_registry import (
+    GraphQLWarmUpRecipe,
+    acquire_graphql_warmup_recipe_lock,
+    due_timeout_graphql_warmup_recipe_keys,
+    get_graphql_warmup_recipe,
+    register_graphql_warmup_recipe,
+    release_graphql_warmup_recipe_lock,
+)
+
+logger = get_logger("api.graphql_warmup")
+
+
+@dataclass(frozen=True, slots=True)
+class GraphQLWarmUpSummary:
+    """Counts from one warm-up execution."""
+
+    evaluated: int = 0
+    failed: int = 0
+    recipes: int = 0
+
+
+def warmable_graphql_properties(
+    manager_class: type[Any],
+    property_names: Iterable[str] | None = None,
+) -> dict[str, GraphQLProperty]:
+    """Return warm-up-eligible GraphQL properties for a manager class."""
+    interface_cls = getattr(manager_class, "Interface", None)
+    if interface_cls is None:
+        return {}
+    available_properties = interface_cls.get_graph_ql_properties()
+    selected = set(property_names) if property_names is not None else None
+    return {
+        name: prop
+        for name, prop in available_properties.items()
+        if isinstance(prop, GraphQLProperty)
+        and prop.warm_up
+        and prop.cache in {"dependency", "timeout"}
+        and (selected is None or name in selected)
+    }
+
+
+def warm_up_graphql_properties(
+    manager_classes: Iterable[type[Any]] | None = None,
+    property_names: Iterable[str] | None = None,
+) -> GraphQLWarmUpSummary:
+    """Warm opted-in GraphQL properties by enumerating each manager's ``.all()``."""
+    if not graphql_warmup_enabled():
+        return GraphQLWarmUpSummary()
+    classes = tuple(manager_classes) if manager_classes is not None else _all_managers()
+    batch_size = _positive_int_setting("GRAPHQL_WARMUP_BATCH_SIZE", 100)
+    warning_threshold = _positive_int_setting(
+        "GRAPHQL_WARMUP_WARNING_ITEMS_PER_MANAGER",
+        1000,
+    )
+    total_evaluated = 0
+    total_failed = 0
+    total_recipes = 0
+
+    for manager_class in classes:
+        properties = warmable_graphql_properties(manager_class, property_names)
+        if not properties:
+            continue
+        manager_path = _manager_path(manager_class)
+        instances_seen = 0
+        warned = False
+        batch: list[Any] = []
+        for instance in manager_class.all():
+            instances_seen += 1
+            if not warned and instances_seen > warning_threshold:
+                warned = True
+                logger.warning(
+                    "graphql warm-up manager crossed warning threshold",
+                    context={
+                        "manager": manager_class.__name__,
+                        "threshold": warning_threshold,
+                    },
+                )
+            batch.append(instance)
+            if len(batch) >= batch_size:
+                evaluated, failed, recipes = _warm_batch(
+                    batch,
+                    properties,
+                    manager_path,
+                )
+                total_evaluated += evaluated
+                total_failed += failed
+                total_recipes += recipes
+                batch = []
+        if batch:
+            evaluated, failed, recipes = _warm_batch(batch, properties, manager_path)
+            total_evaluated += evaluated
+            total_failed += failed
+            total_recipes += recipes
+
+    return GraphQLWarmUpSummary(
+        evaluated=total_evaluated,
+        failed=total_failed,
+        recipes=total_recipes,
+    )
+
+
+def warm_up_graphql_recipe(cache_key: str) -> bool:
+    """Reconstruct and warm one recipe-backed GraphQL property."""
+    if not graphql_warmup_enabled():
+        return False
+    recipe = get_graphql_warmup_recipe(cache_key)
+    if recipe is None:
+        return False
+    lock = acquire_graphql_warmup_recipe_lock(cache_key)
+    if lock is None:
+        return False
+    warmed = False
+    try:
+        manager_class = import_string(recipe.manager_path)
+        instance = manager_class(**recipe.identification)
+        interface_cls = getattr(manager_class, "Interface", None)
+        if interface_cls is not None:
+            prop = interface_cls.get_graph_ql_properties().get(recipe.property_name)
+            if isinstance(prop, GraphQLProperty):
+                if recipe.cache == "timeout":
+                    warmed = _refresh_timeout_recipe(instance, prop, recipe)
+                else:
+                    with CalculationRunContext():
+                        getattr(instance, recipe.property_name)
+                    register_graphql_warmup_recipe(
+                        _recipe_for(instance, prop, recipe.property_name)
+                    )
+                    warmed = True
+    except Exception:
+        logger.exception(
+            "graphql warm-up recipe failed",
+            context={"cache_key": cache_key},
+        )
+    finally:
+        release_graphql_warmup_recipe_lock(lock)
+    return warmed
+
+
+def refresh_due_graphql_warmup_recipes(limit: int | None = None) -> int:
+    """Refresh due timeout recipes and return the number refreshed."""
+    if not graphql_warmup_enabled():
+        return 0
+    refreshed = 0
+    for cache_key in due_timeout_graphql_warmup_recipe_keys(limit=limit):
+        if warm_up_graphql_recipe(cache_key):
+            refreshed += 1
+    return refreshed
+
+
+def enqueue_graphql_warmup(
+    manager_classes: Iterable[type[Any]] | None = None,
+) -> bool:
+    """Enqueue or run all-entry warm-up through the built-in task adapter."""
+    if not graphql_warmup_enabled():
+        return False
+    from general_manager.api.graphql_warmup_tasks import dispatch_graphql_warmup
+
+    return dispatch_graphql_warmup(manager_classes)
+
+
+def enqueue_graphql_recipe_warmup(cache_keys: Iterable[str]) -> bool:
+    """Enqueue recipe re-warm through the built-in task adapter."""
+    if not (
+        graphql_warmup_enabled()
+        and bool(get_setting("GRAPHQL_WARMUP_REWARM_AFTER_INVALIDATION", True))
+    ):
+        return False
+    keys = tuple(dict.fromkeys(cache_keys))
+    if not keys:
+        return False
+    from general_manager.api.graphql_warmup_tasks import dispatch_graphql_recipe_warmup
+
+    return dispatch_graphql_recipe_warmup(keys)
+
+
+def graphql_warmup_enabled() -> bool:
+    """Return whether framework-owned GraphQL warm-up behavior is enabled."""
+    return bool(get_setting("GRAPHQL_WARMUP_ENABLED", False))
+
+
+def _warm_batch(
+    instances: list[Any],
+    properties: dict[str, GraphQLProperty],
+    manager_path: str | None,
+) -> tuple[int, int, int]:
+    """Evaluate one batch of instances and persist successful recipes."""
+    candidates: list[GraphQLWarmUpRecipe] = []
+    evaluated = 0
+    failed = 0
+    with CalculationRunContext():
+        for instance in instances:
+            for property_name, prop in properties.items():
+                try:
+                    getattr(instance, property_name)
+                except Exception:
+                    failed += 1
+                    logger.exception(
+                        "graphql property warm-up failed",
+                        context={
+                            "manager": instance.__class__.__name__,
+                            "property": property_name,
+                            "identification": getattr(
+                                instance,
+                                "identification",
+                                None,
+                            ),
+                        },
+                    )
+                    continue
+                evaluated += 1
+                if manager_path is not None:
+                    candidates.append(_recipe_for(instance, prop, property_name))
+    for recipe in candidates:
+        register_graphql_warmup_recipe(recipe)
+    return evaluated, failed, len(candidates)
+
+
+def _refresh_timeout_recipe(
+    instance: Any,
+    prop: GraphQLProperty,
+    recipe: GraphQLWarmUpRecipe,
+) -> bool:
+    """Refresh one timeout-backed recipe without evicting the old value first."""
+    result = prop._raw_fget(instance)
+    django_cache.set(recipe.cache_key, result, recipe.timeout)
+    register_graphql_warmup_recipe(_recipe_for(instance, prop, recipe.property_name))
+    return True
+
+
+def _recipe_for(
+    instance: Any,
+    prop: GraphQLProperty,
+    property_name: str,
+) -> GraphQLWarmUpRecipe:
+    """Build the registry payload for a warmed instance/property pair."""
+    cache_key = make_cache_key(prop._get_cached_fget(), (instance,), {})
+    timeout = prop.timeout if prop.cache == "timeout" else None
+    refresh_at = None
+    if prop.cache == "timeout" and timeout is not None:
+        refresh_at = timezone.now() + timedelta(
+            seconds=timeout * _timeout_refresh_ratio()
+        )
+    return GraphQLWarmUpRecipe(
+        cache_key=cache_key,
+        manager_path=_manager_path(instance.__class__) or "",
+        property_name=property_name,
+        identification=dict(getattr(instance, "identification", {})),
+        cache="timeout" if prop.cache == "timeout" else "dependency",
+        timeout=timeout,
+        refresh_at=refresh_at,
+    )
+
+
+def _manager_path(manager_class: type[Any]) -> str | None:
+    """Return an import path for reconstructable manager classes."""
+    qualname = getattr(manager_class, "__qualname__", manager_class.__name__)
+    if "<locals>" in qualname:
+        return None
+    return f"{manager_class.__module__}.{qualname}"
+
+
+def _all_managers() -> tuple[type[Any], ...]:
+    """Return all manager classes registered with the GraphQL integration."""
+    from general_manager.api.graphql import GraphQL
+
+    return tuple(GraphQL.manager_registry.values())
+
+
+def _positive_int_setting(key: str, default: int) -> int:
+    """Read a positive integer warm-up setting with a fallback."""
+    raw = get_setting(key, default)
+    try:
+        return max(1, int(raw))
+    except (TypeError, ValueError):
+        return default
+
+
+def _timeout_refresh_ratio() -> float:
+    """Read and clamp the timeout refresh ratio setting."""
+    raw = get_setting("GRAPHQL_WARMUP_TIMEOUT_REFRESH_RATIO", 0.8)
+    try:
+        return max(0.0, min(1.0, float(raw)))
+    except (TypeError, ValueError):
+        return 0.8

generalmanager-0.56.0/src/general_manager/api/graphql_warmup_registry.py

@@ -0,0 +1,315 @@
+"""Cache-backed recipe registry for GraphQL property warm-up."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Literal
+import threading
+import time
+import uuid
+
+from django.core.cache import cache as django_cache
+
+GraphQLWarmUpCacheScope = Literal["dependency", "timeout"]
+
+RECIPE_VERSION = 1
+KEY_PREFIX = "general_manager:graphql_warmup"
+RECIPE_INDEX_KEY = f"{KEY_PREFIX}:recipes"
+TIMEOUT_RECIPE_INDEX_KEY = f"{KEY_PREFIX}:timeout_recipes"
+LOCK_PREFIX = f"{KEY_PREFIX}:lock"
+INDEX_LOCK_PREFIX = f"{LOCK_PREFIX}:index"
+DEFAULT_RECIPE_LOCK_TIMEOUT = 300
+DEFAULT_INDEX_LOCK_TIMEOUT = 30
+DEFAULT_INDEX_LOCK_WAIT_SECONDS = 1.0
+_LOCAL_LOCKS_GUARD = threading.Lock()
+_LOCAL_LOCKS: dict[str, threading.Lock] = {}
+
+
+@dataclass(frozen=True, slots=True)
+class GraphQLWarmUpRecipe:
+    """Information required to reconstruct one warmed GraphQL property entry."""
+
+    cache_key: str
+    manager_path: str
+    property_name: str
+    identification: dict[str, Any]
+    cache: GraphQLWarmUpCacheScope
+    timeout: int | None
+    refresh_at: datetime | None
+    version: int = RECIPE_VERSION
+
+
+@dataclass(frozen=True, slots=True)
+class GraphQLWarmUpRecipeLock:
+    """Token proving ownership of one recipe warm-up attempt."""
+
+    key: str
+    token: str
+
+
+class GraphQLWarmUpRecipeLockTimeoutError(TimeoutError):
+    """Raised when an index update lock cannot be acquired quickly."""
+
+    def __init__(self, lock_key: str) -> None:
+        """Build an error message for a cache lock acquisition timeout."""
+        super().__init__(f"Timed out acquiring GraphQL warm-up lock: {lock_key}")
+
+
+def register_graphql_warmup_recipe(
+    recipe: GraphQLWarmUpRecipe,
+    *,
+    cache_backend: Any = django_cache,
+) -> None:
+    """Persist one warm-up recipe and update registry indexes."""
+    cache_backend.set(_recipe_key(recipe.cache_key), recipe, None)
+    _add_index_member(RECIPE_INDEX_KEY, recipe.cache_key, cache_backend=cache_backend)
+    if recipe.cache == "timeout" and recipe.refresh_at is not None:
+        _add_index_member(
+            TIMEOUT_RECIPE_INDEX_KEY,
+            recipe.cache_key,
+            cache_backend=cache_backend,
+        )
+    else:
+        _remove_index_member(
+            TIMEOUT_RECIPE_INDEX_KEY,
+            recipe.cache_key,
+            cache_backend=cache_backend,
+        )
+
+
+def get_graphql_warmup_recipe(
+    cache_key: str,
+    *,
+    cache_backend: Any = django_cache,
+) -> GraphQLWarmUpRecipe | None:
+    """Return the recipe for *cache_key*, if it exists and matches this version."""
+    recipe = cache_backend.get(_recipe_key(cache_key))
+    if not isinstance(recipe, GraphQLWarmUpRecipe):
+        return None
+    if recipe.version != RECIPE_VERSION:
+        return None
+    return recipe
+
+
+def get_graphql_warmup_recipes(
+    cache_keys: Iterable[str],
+    *,
+    cache_backend: Any = django_cache,
+) -> dict[str, GraphQLWarmUpRecipe]:
+    """Return existing recipes for *cache_keys* keyed by cache key."""
+    recipes: dict[str, GraphQLWarmUpRecipe] = {}
+    for cache_key in tuple(dict.fromkeys(cache_keys)):
+        recipe = get_graphql_warmup_recipe(cache_key, cache_backend=cache_backend)
+        if recipe is not None:
+            recipes[cache_key] = recipe
+    return recipes
+
+
+def graphql_warmup_recipe_keys(
+    *,
+    cache_backend: Any = django_cache,
+) -> tuple[str, ...]:
+    """Return known recipe cache keys in deterministic order."""
+    return tuple(sorted(_read_index(RECIPE_INDEX_KEY, cache_backend=cache_backend)))
+
+
+def due_timeout_graphql_warmup_recipe_keys(
+    *,
+    now: datetime | None = None,
+    limit: int | None = None,
+    cache_backend: Any = django_cache,
+) -> tuple[str, ...]:
+    """Return timeout recipe keys whose refresh time has arrived."""
+    current_time = now or datetime.now(UTC)
+    due: list[tuple[datetime, str]] = []
+    for cache_key in _read_index(TIMEOUT_RECIPE_INDEX_KEY, cache_backend=cache_backend):
+        recipe = get_graphql_warmup_recipe(cache_key, cache_backend=cache_backend)
+        if recipe is None or recipe.cache != "timeout" or recipe.refresh_at is None:
+            _remove_index_member(
+                TIMEOUT_RECIPE_INDEX_KEY,
+                cache_key,
+                cache_backend=cache_backend,
+            )
+            continue
+        if recipe.refresh_at <= current_time:
+            due.append((recipe.refresh_at, cache_key))
+    ordered = tuple(cache_key for _refresh_at, cache_key in sorted(due))
+    if limit is None:
+        return ordered
+    return ordered[: max(0, limit)]
+
+
+def delete_graphql_warmup_recipe(
+    cache_key: str,
+    *,
+    cache_backend: Any = django_cache,
+) -> None:
+    """Remove one recipe and all index references to it."""
+    cache_backend.delete(_recipe_key(cache_key))
+    _remove_index_member(RECIPE_INDEX_KEY, cache_key, cache_backend=cache_backend)
+    _remove_index_member(
+        TIMEOUT_RECIPE_INDEX_KEY,
+        cache_key,
+        cache_backend=cache_backend,
+    )
+
+
+def acquire_graphql_warmup_recipe_lock(
+    cache_key: str,
+    *,
+    timeout: int = DEFAULT_RECIPE_LOCK_TIMEOUT,
+    cache_backend: Any = django_cache,
+) -> GraphQLWarmUpRecipeLock | None:
+    """Acquire a best-effort per-recipe execution lock."""
+    lock_key = _lock_key(cache_key)
+    token = uuid.uuid4().hex
+    if not cache_backend.add(lock_key, token, timeout):
+        return None
+    return GraphQLWarmUpRecipeLock(key=lock_key, token=token)
+
+
+def release_graphql_warmup_recipe_lock(
+    lock: GraphQLWarmUpRecipeLock,
+    *,
+    cache_backend: Any = django_cache,
+) -> None:
+    """Release a recipe lock without deleting another worker's newer lock."""
+    _delete_lock_if_owned(lock, cache_backend=cache_backend)
+
+
+def _recipe_key(cache_key: str) -> str:
+    """Return the cache key used for one recipe payload."""
+    return f"{KEY_PREFIX}:recipe:{cache_key}"
+
+
+def _lock_key(cache_key: str) -> str:
+    """Return the cache key used for one recipe lock."""
+    return f"{LOCK_PREFIX}:{cache_key}"
+
+
+def _read_index(index_key: str, *, cache_backend: Any) -> frozenset[str]:
+    """Read an index payload, ignoring malformed values."""
+    value = cache_backend.get(index_key, frozenset())
+    if not isinstance(value, frozenset):
+        return frozenset()
+    return value
+
+
+def _write_index(
+    index_key: str,
+    values: Iterable[str],
+    *,
+    cache_backend: Any,
+) -> None:
+    """Write an index payload as an immutable set."""
+    cache_backend.set(index_key, frozenset(values), None)
+
+
+def _add_index_member(
+    index_key: str,
+    member: str,
+    *,
+    cache_backend: Any,
+) -> None:
+    """Add one member to an index under an index-level cache lock."""
+    with _locked_index_update(index_key, cache_backend=cache_backend):
+        _write_index(
+            index_key,
+            _read_index(index_key, cache_backend=cache_backend) | frozenset((member,)),
+            cache_backend=cache_backend,
+        )
+
+
+def _remove_index_member(
+    index_key: str,
+    member: str,
+    *,
+    cache_backend: Any,
+) -> None:
+    """Remove one member from an index under an index-level cache lock."""
+    with _locked_index_update(index_key, cache_backend=cache_backend):
+        _write_index(
+            index_key,
+            _read_index(index_key, cache_backend=cache_backend) - frozenset((member,)),
+            cache_backend=cache_backend,
+        )
+
+
+@contextmanager
+def _locked_index_update(index_key: str, *, cache_backend: Any):
+    """Acquire and release the cache lock guarding one index update."""
+    lock = _acquire_cache_lock(
+        f"{INDEX_LOCK_PREFIX}:{index_key}",
+        timeout=DEFAULT_INDEX_LOCK_TIMEOUT,
+        wait_seconds=DEFAULT_INDEX_LOCK_WAIT_SECONDS,
+        cache_backend=cache_backend,
+    )
+    try:
+        yield
+    finally:
+        release_graphql_warmup_recipe_lock(lock, cache_backend=cache_backend)
+
+
+def _acquire_cache_lock(
+    lock_key: str,
+    *,
+    timeout: int,
+    wait_seconds: float,
+    cache_backend: Any,
+) -> GraphQLWarmUpRecipeLock:
+    """Acquire a cache-backed lock, waiting briefly for contention to clear."""
+    token = uuid.uuid4().hex
+    deadline = time.monotonic() + wait_seconds
+    while True:
+        if cache_backend.add(lock_key, token, timeout):
+            return GraphQLWarmUpRecipeLock(key=lock_key, token=token)
+        if time.monotonic() >= deadline:
+            raise GraphQLWarmUpRecipeLockTimeoutError(lock_key)
+        time.sleep(0.01)
+
+
+def _delete_lock_if_owned(
+    lock: GraphQLWarmUpRecipeLock,
+    *,
+    cache_backend: Any,
+) -> None:
+    """Delete a lock only when the backend still stores the caller's token."""
+    if _delete_redis_lock_if_owned(lock, cache_backend=cache_backend):
+        return
+    with _local_lock_for(lock.key):
+        if cache_backend.get(lock.key) != lock.token:
+            return
+        if cache_backend.get(lock.key) != lock.token:
+            return
+        cache_backend.delete(lock.key)
+
+
+def _delete_redis_lock_if_owned(
+    lock: GraphQLWarmUpRecipeLock,
+    *,
+    cache_backend: Any,
+) -> bool:
+    """Use Redis compare-and-delete when the cache backend exposes a client."""
+    client_holder = getattr(cache_backend, "client", None)
+    get_client = getattr(client_holder, "get_client", None)
+    if get_client is None:
+        return False
+    make_key = getattr(cache_backend, "make_key", lambda key: key)
+    script = (
+        "if redis.call('get', KEYS[1]) == ARGV[1] "
+        "then return redis.call('del', KEYS[1]) else return 0 end"
+    )
+    try:
+        get_client(write=True).eval(script, 1, make_key(lock.key), lock.token)
+    except (AttributeError, OSError, TypeError, ValueError):
+        return False
+    return True
+
+
+def _local_lock_for(lock_key: str) -> threading.Lock:
+    """Return a process-local fallback mutex for non-Redis cache backends."""
+    with _LOCAL_LOCKS_GUARD:
+        return _LOCAL_LOCKS.setdefault(lock_key, threading.Lock())