buildgrid 0.3.4__py3-none-any.whl → 0.4.0__py3-none-any.whl

buildgrid/server/app/cli.py

@@ -26,10 +26,10 @@ import logging
 import os
 import sys
 from typing import Any, cast
+from importlib.resources import files
 
 import click
 import grpc
-from importlib_resources import files
 
 from buildgrid.server.logging import JSONFormatter
 from buildgrid.server.metadata import ctx_grpc_request_id
@@ -64,9 +64,9 @@ class App(click.Group):
     def list_commands(self, ctx: Any) -> list[str]:
         """Lists available command names."""
         commands = []
-        for filename in os.listdir(cmd_folder):
-            if filename.endswith(".py") and filename.startswith("cmd_"):
-                command_name = filename[4:-3].replace("_", "-")
+        for path in cmd_folder.iterdir():
+            if path.name.endswith(".py") and path.name.startswith("cmd_"):
+                command_name = path.name[4:-3].replace("_", "-")
                 commands.append(command_name)
         commands.sort()
 

buildgrid/server/app/settings/parser.py

@@ -18,6 +18,7 @@ import sys
 from collections import defaultdict
 from typing import TYPE_CHECKING, Any, Callable, Hashable, Iterable, Sequence, TypedDict, TypeVar
 from urllib.parse import urlparse
+from importlib.resources import files
 
 import buildgrid_metering.client as metering
 import click
@@ -26,7 +27,6 @@ import jsonschema
 import requests
 import yaml
 from buildgrid_metering.client.exceptions import MeteringServiceClientError, MeteringServiceError
-from importlib_resources import files
 
 from buildgrid.server.actioncache.caches.action_cache_abc import ActionCacheABC
 from buildgrid.server.actioncache.caches.lru_cache import LruActionCache
@@ -99,6 +99,7 @@ if TYPE_CHECKING:
     from buildgrid.server.actioncache.caches.redis_cache import RedisActionCache
     from buildgrid.server.cas.storage.index.redis import RedisIndex
     from buildgrid.server.cas.storage.redis import RedisStorage
+    from buildgrid.server.cas.storage.redis_fmb_cache import RedisFMBCache
     from buildgrid.server.redis.provider import RedisProvider
 
 
@@ -459,6 +460,38 @@ def load_redis_index(storage: StorageABC, redis: "RedisProvider", prefix: str |
     return RedisIndex(redis=redis, storage=storage, prefix=prefix)
 
 
+@object_tag("!redis-fmb-cache")
+def load_redis_fmb_cache(
+    storage: StorageABC, redis: "RedisProvider", ttl_secs: int, prefix: str | None = None
+) -> "RedisFMBCache":
+    """Generates :class:`buildgrid.server.cas.storage.redis_fmb_cache.RedisFMBCache`
+    using the tag ``!redis-fmb-cache``.
+
+    Usage
+        .. code:: yaml
+
+            - !redis-fmb-cache
+              storage: *cas-storage
+              redis: *redis
+              ttl-secs: 3600  # 1 hour
+              prefix: "D"
+
+    Args:
+        storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`):
+            Instance of storage to use. This must be a storage object constructed using
+            a YAML tag ending in ``-storage``, for example ``!disk-storage``.
+        redis (:class:`buildgrid.server.redis.provider.RedisProvider`): A configured Redis
+            connection manager. This must be an object with an ``!redis-connection`` YAML tag.
+        prefix (str): An optional prefix to use to prefix keys written by this index. If not
+            specified a prefix of "C" is used.
+    """
+
+    # Import here so there is no global buildgrid dependency on redis
+    from buildgrid.server.cas.storage.redis_fmb_cache import RedisFMBCache
+
+    return RedisFMBCache(redis, storage, ttl_secs, prefix)
+
+
 @object_tag("!replicated-storage")
 def load_replicated_storage(
     storages: list[StorageABC],
@@ -1358,9 +1391,9 @@ def load_static_property_set(
 def load_sql_index(
     storage: StorageABC,
     sql: SqlProvider,
+    sql_ro: SqlProvider | None = None,
     window_size: int = 1000,
     inclause_limit: int = -1,
-    fallback_on_get: bool = False,
     max_inline_blob_size: int = 0,
     refresh_accesstime_older_than: int = 0,
 ) -> SQLIndex:
@@ -1371,13 +1404,12 @@ def load_sql_index(
         .. code:: yaml
 
             - !sql-index
-              # This assumes that a storage instance is defined elsewhere
-              # with a `&cas-storage` anchor
+              # Assuming the YAML anchors are defined elsewhere in the config file
               storage: *cas-storage
               sql: *sql
+              sql-ro: *readonly-sql
               window-size: 1000
               inclause-limit: -1
-              fallback-on-get: no
               max-inline-blob-size: 256
               refresh-accesstime-older-than: 0
 
@@ -1385,16 +1417,18 @@ def load_sql_index(
         storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`):
             Instance of storage to use. This must be a storage object constructed using
             a YAML tag ending in ``-storage``, for example ``!disk-storage``.
+        sql (:class:`buildgrid.server.sql.provider.SqlProvider`): A configured SQL
+            connection manager. This must be an object with an ``!sql-connection`` YAML tag.
+        sql_ro (:class:`buildgrid.server.sql.provider.SqlProvider`): Similar to `sql`,
+            but used for readonly backend transactions.
+            If set, it should be configured with a replica of main DB using an optional but
+            encouraged readonly role. Permission check is not executed by BuildGrid.
+            If not set, readonly transactions are executed by `sql` object.
         window_size (uint): Maximum number of blobs to fetch in one SQL operation
             (larger resultsets will be automatically split into multiple queries)
         inclause_limit (int): If nonnegative, overrides the default number of variables
             permitted per "in" clause. See the buildgrid.server.cas.storage.index.sql.SQLIndex
             comments for more details.
-        fallback_on_get (bool): By default, the SQL Index only fetches blobs from the
-            underlying storage if they're present in the index on ``get_blob``/``bulk_read_blobs``
-            requests to minimize interactions with the storage. If this is set, the index
-            instead checks the underlying storage directly on ``get_blob``/``bulk_read_blobs``
-            requests, then loads all blobs found into the index.
         max_inline_blob_size (int): Blobs of this size or smaller are stored directly in the index
             and not in the backing storage (must be nonnegative).
         refresh-accesstime-older-than (int): When reading a blob, its access timestamp will not be
@@ -1403,19 +1437,14 @@ def load_sql_index(
     """
 
     storage_type = type(storage).__name__
-    click.echo(
-        f"SQLIndex: storage={storage_type}, "
-        f"window_size={window_size}, "
-        f"inclause_limit={inclause_limit}, "
-        f"fallback_on_get={fallback_on_get}"
-    )
+    click.echo(f"SQLIndex: storage={storage_type}, window_size={window_size}, inclause_limit={inclause_limit}, ")
     click.echo(click.style(f"Creating an SQL CAS Index for {storage_type}\n", fg="green", bold=True))
     return SQLIndex(
         sql,
         storage,
+        sql_ro=sql_ro,
         window_size=window_size,
         inclause_limit=inclause_limit,
-        fallback_on_get=fallback_on_get,
         max_inline_blob_size=max_inline_blob_size,
         refresh_accesstime_older_than=refresh_accesstime_older_than,
     )

buildgrid/server/app/settings/schema.yml

@@ -24,7 +24,9 @@ properties:
   sentry:              { "$ref": "#/definitions/sentry" }
   limiter:             { "$ref": "#/definitions/limiter" }
 anyOf:
-  - required: [server, instances]
+  - anyOf:
+    - required: [server, instances]
+    - required: [server, services]
   - required: [cleanup]
 
 definitions:
@@ -172,6 +174,7 @@ definitions:
       - "$ref": "#/definitions/size-differentiated-storage"
       - "$ref": "#/definitions/sql-storage"
       - "$ref": "#/definitions/with-cache-storage"
+      - "$ref": "#/definitions/redis-fmb-cache"
       - "$ref": "#/definitions/index"
 
   index:
@@ -413,6 +416,17 @@ definitions:
       action-cache: { "$ref": "#/definitions/cache" }
     required: [kind, action-cache]
 
+  redis-fmb-cache:
+    type: object
+    propertyNames: { pattern: "^[A-Za-z0-9-]*$" }
+    properties:
+      kind:     { type: string, enum: ["!redis-fmb-cache"] }
+      storage:  { "$ref": "#/definitions/storage" }
+      redis:    { "$ref": "#/definitions/redis-connection" }
+      ttl-secs: { type: integer, minimum: 0 }
+      prefix:   { type: string }
+    required: [kind, storage, redis, ttl-secs]
+
   redis-index:
     type: object
     propertyNames: { pattern: "^[A-Za-z0-9-]*$" }
@@ -429,10 +443,10 @@ definitions:
     properties:
       kind:                           { type: string, enum: ["!sql-index"] }
       sql:                            { "$ref": "#/definitions/sql-connection" }
+      sql-ro:                         { "$ref": "#/definitions/sql-connection" }
       storage:                        { "$ref": "#/definitions/storage" }
       window-size:                    { type: integer, minimum: 1 }
       inclause-limit:                 { type: integer, minimum: 1 }
-      fallback-on-get:                { type: boolean }
       max-inline-blob-size:           { type: integer, minimum: 0, maximum: 1000000000 }
       refresh-accesstime-older-than:  { type: integer, minimum: 0 }
     required: [kind, sql, storage]

buildgrid/server/bots/service.py

@@ -117,33 +117,29 @@ class BaseBotsService:
 
         LOGGER.debug("Beginning initial lease synchronization.", tags=bot_log_tags(request.bot_session))
 
-        orig_leases_count = len(request.bot_session.leases)
         with scheduler.bot_notifier.subscription(request.bot_session.name) as event:
             capacity = get_bot_capacity(request.bot_session)
-            leases = scheduler.synchronize_bot_leases(
+            old_version = int(request.bot_session.version) if request.bot_session.version else 0
+            leases, new_version = scheduler.synchronize_bot_leases(
                 request.bot_session.name,
                 request.bot_session.bot_id,
                 request.bot_session.status,
+                old_version,
                 request.bot_session.leases,
                 lease_id_to_partial_execution_metadata,
                 max_capacity=capacity,
             )
             del request.bot_session.leases[:]
             request.bot_session.leases.extend(leases)
+            request.bot_session.version = str(new_version)
 
             LOGGER.debug("Completed initial lease synchronization.", tags=bot_log_tags(request.bot_session))
 
             capabilities = list(map(hash_from_dict, scheduler.property_set.worker_properties(request.bot_session)))
             scheduler.maybe_update_bot_platforms(request.bot_session.name, capabilities)
 
-            # Only block on lease assignment if we aren't currently working on anything, to avoid
-            # the client needing to deal with interrupting this wait when a current job is
-            # completed. Any extra assignments can be picked up when we next synchronize.
-            #
-            # This should also be skipped if we've removed a lease from the session, to mitigate
-            # situations where the scheduler is updated with the new state of a lease, but a fault
-            # thereafter causes the worker to retry the old UpdateBotSession call.
-            if not leases and not orig_leases_count:
+            # Block on lease assignment if we have space and an unchanged set of leases.
+            if len(leases) < capacity and old_version == new_version:
                 self._request_leases(
                     request.bot_session,
                     CancellationContext(context),
@@ -231,11 +227,18 @@ class BaseBotsService:
 
         # Synchronize the lease again to pick up db changes.
         LOGGER.debug("Synchronizing leases after job assignment wait.", tags=bot_log_tags(bot_session))
-        if leases := scheduler.synchronize_bot_leases(
-            bot_session.name, bot_session.bot_id, bot_session.status, bot_session.leases, max_capacity=capacity
+        if leases_version := scheduler.synchronize_bot_leases(
+            bot_session.name,
+            bot_session.bot_id,
+            bot_session.status,
+            int(bot_session.version) if bot_session.version else 0,
+            bot_session.leases,
+            max_capacity=capacity,
         ):
+            leases, new_version = leases_version
             del bot_session.leases[:]
             bot_session.leases.extend(leases)
+            bot_session.version = str(new_version)
 
 
 class BotsService(BotsServicer, BaseBotsService, InstancedServicer[BotsInterface]):

buildgrid/server/cas/storage/index/index_abc.py

@@ -34,13 +34,6 @@ from ..storage_abc import StorageABC
 
 
 class IndexABC(StorageABC):
-    @abc.abstractmethod
-    def __init__(self, *, fallback_on_get: bool = False) -> None:
-        # If fallback is enabled, the index is required to fetch blobs from
-        # storage on each get_blob and bulk_read_blobs request and update
-        # itself accordingly.
-        self._fallback_on_get = fallback_on_get
-
     @abc.abstractmethod
     def least_recent_digests(self) -> Iterator[Digest]:
         """Generator to iterate through the digests in LRU order"""