buildgrid 0.2.53__py3-none-any.whl → 0.3.1__py3-none-any.whl

buildgrid/server/app/settings/parser.py

@@ -1025,7 +1025,7 @@ def load_sql_scheduler(
         sql_ro = sql_ro or sql
         sql_notifier = sql_notifier or sql
 
-        logstream_url, logstream_credentials, logstream_instance = get_logstream_connection_info(logstream)
+        logstream_url, logstream_credentials = get_logstream_connection_info(logstream)
         logstream_channel: grpc.Channel | None = None
         if logstream_url is not None:
             logstream_credentials = logstream_credentials or {}
@@ -1081,7 +1081,6 @@ def load_sql_scheduler(
             ),
             bot_session_keepalive_timeout=bot_session_keepalive_timeout,
             logstream_channel=logstream_channel,
-            logstream_instance=logstream_instance,
             asset_client=asset_client,
             queued_action_retention_hours=queued_action_retention_hours,
             completed_action_retention_hours=completed_action_retention_hours,
@@ -2150,7 +2149,6 @@ def load_metering_service_client(
 def load_asset_client(
     url: str,
     credentials: ClientCredentials | None = None,
-    instance_name: str = "",
     request_timeout: float = 5.0,
     retries: int = 3,
 ) -> AssetClient:
@@ -2162,7 +2160,6 @@ def load_asset_client(
 
             - !asset-client
               url: https://remote-asset.com
-              instance-name: dev
               credentials:
                 tls-client-cert: /path/to/cert
                 auth-token: /path/to/token
@@ -2179,7 +2176,7 @@ def load_asset_client(
         server_cert=credentials.get("tls-server-cert"),
         timeout=request_timeout,
     )
-    return AssetClient(channel=channel, instance_name=instance_name, retries=retries)
+    return AssetClient(channel=channel, retries=retries)
 
 
 @object_tag("!introspection")
@@ -2378,18 +2375,16 @@ def _validate_server_credentials(credentials: dict[str, str] | None) -> None:
         sys.exit(-1)
 
 
-def get_logstream_connection_info(logstream: Any) -> tuple[str | None, dict[str, str] | None, str | None]:
+def get_logstream_connection_info(logstream: Any) -> tuple[str | None, dict[str, str] | None]:
     logstream_url = None
     credentials = None
-    logstream_instance_name = None
     if logstream:
         logstream_url = logstream["url"]
         credentials = logstream.get("credentials")
         if not _validate_url_and_credentials(logstream_url, credentials=credentials):
             sys.exit(-1)
-        logstream_instance_name = logstream.get("instance-name", "")
 
-    return logstream_url, credentials, logstream_instance_name
+    return logstream_url, credentials
 
 
 def get_schema(strict: bool = False) -> Any:

buildgrid/server/app/settings/schema.yml

@@ -472,7 +472,7 @@ definitions:
       priority-assignment-percentage:   { type: number, minimum: 0 }
       metering-service-client:          { "$ref": "#/definitions/metering-service-client" }
       metering-throttle-action:         { type: string, enum: [deprioritize, reject] }
-      logstream:                        { "$ref": "#/definitions/grpc-connection-options" }
+      logstream:                        { "$ref": "#/definitions/logstream-connection-options" }
       asset-client:                     { "$ref": "#/definitions/asset-client" }
       action-browser-url:               { type: string }
       poll-interval:                    { type: number, minimum: 0 }
@@ -645,12 +645,11 @@ definitions:
     properties:
       kind:            { type: string, enum: [ "!asset-client" ] }
       url:             { type: string }
-      instance-name:   { type: string }
       channel-options: { type: object, propertyNames: { pattern: "^[a-z0-9-]+$" } }
       request-timeout: { type: number }
       credentials:     { "$ref": "#/definitions/grpc-credentials" }
       retries:         { type: number }
-    required: [kind, url, instance-name]
+    required: [kind, url]
 
   metering-service-client:
     type: object
@@ -783,16 +782,15 @@ definitions:
       retry-limit:               { type: integer, min: 0 }
     required: [instance-name, index, high-watermark, low-watermark, batch-size]
 
-  grpc-connection-options:
+  logstream-connection-options:
     type: object
     propertyNames: { pattern: "^[A-Za-z0-9-]*$" }
     properties:
       url:              { type: string }
-      instance-name:    { type: string }
       channel-options:  { type: object, propertyNames: { pattern: "^[a-z0-9-]+$" } }
       request-timeout:  { type: number }
       credentials:      { "$ref": "#/definitions/grpc-credentials" }
-    required: [url, instance-name]
+    required: [url]
 
   grpc-credentials:
     type: object

buildgrid/server/cas/storage/index/sql.py

@@ -17,8 +17,7 @@
 SQLIndex
 ==================
 
-A SQL index implementation. This can be pointed to either a remote SQL server
-or a local SQLite database.
+A SQL index implementation. This must be pointed to a remote SQL server.
 
 """
 
@@ -35,6 +34,8 @@ from sqlalchemy.orm import InstrumentedAttribute, Session, load_only
 from sqlalchemy.orm.exc import StaleDataError
 from sqlalchemy.orm.query import Query
 from sqlalchemy.orm.session import Session as SessionType
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.sql.functions import coalesce
 
 from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest
 from buildgrid._protos.google.rpc import code_pb2
@@ -49,10 +50,8 @@ from buildgrid.server.sql.provider import SqlProvider
 
 from ..storage_abc import StorageABC
 from .index_abc import IndexABC
-from .sql_dialect_delegates import PostgreSQLDelegate, SQLiteDelegate
 
 LOGGER = buildgrid_logger(__name__)
-DIALECT_DELEGATES = {"postgresql": PostgreSQLDelegate, "sqlite": SQLiteDelegate}
 
 INLINE_BLOB_SIZE_HARD_MAXIMUM = 1000000000
 
@@ -151,9 +150,6 @@ class SQLIndex(IndexABC):
             unknown_args = kwargs_keys - available_options
             raise TypeError(f"Unknown keyword arguments: [{unknown_args}]")
 
-        # Dialect-specific initialization
-        self._dialect_delegate = DIALECT_DELEGATES.get(self._sql.dialect)
-
         if inclause_limit > 0:
             if inclause_limit > window_size:
                 LOGGER.warning(
@@ -377,14 +373,32 @@ class SQLIndex(IndexABC):
 
         digest_blob_pairs = sorted(digest_blob_pairs, key=lambda pair: (pair[0].hash, pair[0].size_bytes))
 
-        if self._dialect_delegate:
-            try:
-                self._dialect_delegate._save_digests_to_index(  # type: ignore
-                    digest_blob_pairs, session, self._max_inline_blob_size
-                )
-                return
-            except AttributeError:
-                pass
+        # See discussion of __table__ typing in https://github.com/sqlalchemy/sqlalchemy/issues/9130
+        index_table = cast(Table, IndexEntry.__table__)
+        update_time = datetime.utcnow()
+        new_rows = [
+            {
+                "digest_hash": digest.hash,
+                "digest_size_bytes": digest.size_bytes,
+                "accessed_timestamp": update_time,
+                "inline_blob": (blob if digest.size_bytes <= self._max_inline_blob_size else None),
+                "deleted": False,
+            }
+            for (digest, blob) in digest_blob_pairs
+        ]
+
+        base_insert_stmt = insert(index_table).values(new_rows)
+
+        update_stmt = base_insert_stmt.on_conflict_do_update(
+            index_elements=["digest_hash"],
+            set_={
+                "accessed_timestamp": update_time,
+                "inline_blob": coalesce(base_insert_stmt.excluded.inline_blob, index_table.c.inline_blob),
+                "deleted": False,
+            },
+        )
+
+        session.execute(update_stmt)
 
         update_time = datetime.utcnow()
         # Figure out which digests we can just update

buildgrid/server/cas/storage/sql.py

@@ -26,8 +26,7 @@ from io import BytesIO
 from typing import IO, Any, Iterator, Sequence, TypedDict, cast
 
 from sqlalchemy import CursorResult, delete, func, select
-from sqlalchemy.dialects.postgresql import insert as postgresql_insert
-from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import DBAPIError
 from sqlalchemy.orm.exc import StaleDataError
 
@@ -59,7 +58,7 @@ class SQLStorage(StorageABC):
         self._sql_ro = sql_ro_provider or sql_provider
         self._inclause_limit = self._sql.default_inlimit
 
-        supported_dialects = ["postgresql", "sqlite"]
+        supported_dialects = ["postgresql"]
 
         if self._sql.dialect not in supported_dialects:
             raise RuntimeError(
@@ -71,14 +70,6 @@ class SQLStorage(StorageABC):
         with self._sql.session() as session:
             session.query(BlobEntry).first()
 
-    def _sqlite_bulk_insert(self, new_rows: list[DigestRow]) -> None:
-        with self._sql.session() as session:
-            session.execute(sqlite_insert(BlobEntry).values(new_rows).on_conflict_do_nothing())
-
-    def _postgresql_bulk_insert(self, new_rows: list[DigestRow]) -> None:
-        with self._sql.session() as session:
-            session.execute(postgresql_insert(BlobEntry).values(new_rows).on_conflict_do_nothing())
-
     def _bulk_insert(self, digests: list[tuple[Digest, bytes]]) -> None:
         # Sort digests by hash to ensure consistent order to minimize deadlocks
         # when BatchUpdateBlobs requests have overlapping blobs
@@ -87,12 +78,8 @@ class SQLStorage(StorageABC):
             for (digest, blob) in sorted(digests, key=lambda x: x[0].hash)
         ]
 
-        if self._sql.dialect == "sqlite":
-            self._sqlite_bulk_insert(new_rows)
-        elif self._sql.dialect == "postgresql":
-            self._postgresql_bulk_insert(new_rows)
-        else:
-            raise RuntimeError(f"Unsupported dialect {self._sql.dialect} for bulk_insert")
+        with self._sql.session() as session:
+            session.execute(insert(BlobEntry).values(new_rows).on_conflict_do_nothing())
 
     @timed(METRIC.STORAGE.STAT_DURATION, type=TYPE)
     def has_blob(self, digest: Digest) -> bool:

buildgrid/server/client/asset.py

@@ -41,13 +41,11 @@ class AssetClient:
     def __init__(
         self,
         channel: grpc.Channel,
-        instance_name: str,
         retries: int = 0,
         max_backoff: int = 64,
         should_backoff: bool = True,
     ) -> None:
         self._channel = channel
-        self._instance_name = instance_name
         self._push_stub = PushStub(channel)
         self._fetch_stub = FetchStub(channel)
         self._retrier = GrpcRetrier(retries=retries, max_backoff=max_backoff, should_backoff=should_backoff)
@@ -68,6 +66,7 @@ class AssetClient:
         expire_at: datetime | None = None,
         referenced_blobs: Iterable[Digest] = [],
         referenced_directories: Iterable[Digest] = [],
+        instance_name: str,
     ) -> PushBlobResponse:
         def _push_blob() -> PushBlobResponse:
             qualifiers_pb = [Qualifier(name=name, value=value) for name, value in qualifiers.items()]
@@ -77,7 +76,7 @@ class AssetClient:
                 expire_at_pb.FromDatetime(expire_at)
 
             request = PushBlobRequest(
-                instance_name=self._instance_name,
+                instance_name=instance_name,
                 uris=uris,
                 qualifiers=qualifiers_pb,
                 expire_at=expire_at_pb,
@@ -98,6 +97,7 @@ class AssetClient:
         expire_at: datetime | None = None,
         referenced_blobs: Iterable[Digest] = [],
         referenced_directories: Iterable[Digest] = [],
+        instance_name: str,
     ) -> PushDirectoryResponse:
         def _push_directory() -> PushDirectoryResponse:
             qualifiers_pb = [Qualifier(name=name, value=value) for name, value in qualifiers.items()]
@@ -107,7 +107,7 @@ class AssetClient:
                 expire_at_pb.FromDatetime(expire_at)
 
             request = PushDirectoryRequest(
-                instance_name=self._instance_name,
+                instance_name=instance_name,
                 uris=uris,
                 qualifiers=qualifiers_pb,
                 expire_at=expire_at_pb,

buildgrid/server/scheduler/impl.py

@@ -31,7 +31,7 @@ from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
 from google.protobuf.timestamp_pb2 import Timestamp
 from grpc import Channel
 from sqlalchemy import ColumnExpressionArgument, CursorResult, and_, delete, func, insert, or_, select, text, update
-from sqlalchemy.dialects import postgresql, sqlite
+from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session, joinedload
 from sqlalchemy.sql.expression import Insert, Select
@@ -218,7 +218,6 @@ class Scheduler:
         metering_throttle_action: MeteringThrottleAction | None = None,
         bot_session_keepalive_timeout: int = 600,
         logstream_channel: Channel | None = None,
-        logstream_instance: str | None = None,
         asset_client: AssetClient | None = None,
         queued_action_retention_hours: float | None = None,
         completed_action_retention_hours: float | None = None,
@@ -268,7 +267,6 @@ class Scheduler:
         self.metering_throttle_action = metering_throttle_action or MeteringThrottleAction.DEPRIORITIZE
         self.bot_session_keepalive_timeout = bot_session_keepalive_timeout
         self.logstream_channel = logstream_channel
-        self.logstream_instance = logstream_instance
         self.asset_client = asset_client
         self.queued_action_retention_hours = queued_action_retention_hours
         self.completed_action_retention_hours = completed_action_retention_hours
@@ -414,7 +412,9 @@ class Scheduler:
                 LOGGER.exception("Checking ActionCache for action failed.", tags=dict(digest=action_digest))
 
         # Extend retention for action
-        self._update_action_retention(action, action_digest, self.queued_action_retention_hours)
+        self._update_action_retention(
+            action, action_digest, self.queued_action_retention_hours, instance_name=current_instance()
+        )
 
         return self.create_operation_for_new_job(
             action=action,
@@ -760,11 +760,10 @@ class Scheduler:
             )
 
     def _notify_job_updated(self, job_names: str | list[str], session: Session) -> None:
-        if self._sql.dialect == "postgresql":
-            if isinstance(job_names, str):
-                job_names = [job_names]
-            for job_name in job_names:
-                session.execute(text(f"NOTIFY {NotificationChannel.JOB_UPDATED.value}, '{job_name}';"))
+        if isinstance(job_names, str):
+            job_names = [job_names]
+        for job_name in job_names:
+            session.execute(text(f"NOTIFY {NotificationChannel.JOB_UPDATED.value}, '{job_name}';"))
 
     def _get_operation(self, operation_name: str, session: Session) -> OperationEntry | None:
         statement = (
@@ -776,7 +775,7 @@ class Scheduler:
 
     def _batch_timeout_jobs(self, job_select_stmt: Select[Any], status_code: int, message: str) -> int:
         """Timeout all jobs selected by a query"""
-        with self._sql.session(sqlite_lock_immediately=True, exceptions_to_not_raise_on=[Exception]) as session:
+        with self._sql.session(exceptions_to_not_raise_on=[Exception]) as session:
             # Get the full list of jobs to timeout
             job_entries = session.execute(job_select_stmt).scalars().all()
             jobs = []
@@ -838,7 +837,7 @@ class Scheduler:
             .limit(1)
             .with_for_update(skip_locked=True)
         )
-        with self._sql.session(sqlite_lock_immediately=True, exceptions_to_not_raise_on=[Exception]) as session:
+        with self._sql.session(exceptions_to_not_raise_on=[Exception]) as session:
             job = session.execute(stale_job_statement).scalar_one_or_none()
             if not job:
                 return False
@@ -916,22 +915,14 @@ class Scheduler:
         job.cancelled = True
 
         # If the job was assigned to a bot, we need to update the quota / capacity
-        if self._sql.dialect == "postgresql":
-            update_query = (
-                update(BotEntry)
-                .where(BotEntry.bot_id == job.worker_name)
-                .values(capacity=BotEntry.capacity + 1)
-                .returning(BotEntry.cohort)
-            )
-            if cohort := session.execute(update_query).scalar_one_or_none():
-                self._update_instance_quota_usage(session, cohort, job.instance_name, -1, guard=None)
-        else:
-            if bot := session.execute(
-                select(BotEntry).where(BotEntry.bot_id == job.worker_name).with_for_update()
-            ).scalar_one_or_none():
-                bot.capacity += 1
-                if bot.cohort:
-                    self._update_instance_quota_usage(session, bot.cohort, job.instance_name, -1, guard=None)
+        update_query = (
+            update(BotEntry)
+            .where(BotEntry.bot_id == job.worker_name)
+            .values(capacity=BotEntry.capacity + 1)
+            .returning(BotEntry.cohort)
+        )
+        if cohort := session.execute(update_query).scalar_one_or_none():
+            self._update_instance_quota_usage(session, cohort, job.instance_name, -1, guard=None)
 
         session.add(
             JobHistoryEntry(
@@ -1157,8 +1148,7 @@ class Scheduler:
         self._notify_job_updated(job.name, session)
 
         LOGGER.debug("Assigned job to bot", tags=log_tags)
-        if self._sql.dialect == "postgresql":
-            session.execute(text(f"NOTIFY {NotificationChannel.JOB_ASSIGNED.value}, '{bot.name}';"))
+        session.execute(text(f"NOTIFY {NotificationChannel.JOB_ASSIGNED.value}, '{bot.name}';"))
 
     def _match_bot_by_sampling(
         self, session: Session, query: Select[tuple[BotEntry]], sampling: SamplingConfig
@@ -1523,7 +1513,7 @@ class Scheduler:
             )
 
         updated = False
-        with self._sql.session(sqlite_lock_immediately=True) as session:
+        with self._sql.session() as session:
             job = session.execute(job_statement).scalar_one_or_none()
             if job is not None:
                 self._match_job_to_bot(session, job, failure_backoff, bot_assignment_fn, assigner_name)
@@ -1558,7 +1548,7 @@ class Scheduler:
             )
 
         updated = False
-        with self._sql.session(sqlite_lock_immediately=True) as session:
+        with self._sql.session() as session:
             job = session.execute(job_statement).scalar_one_or_none()
             if job is not None:
                 self._match_job_to_bot(session, job, failure_backoff, bot_assignment_fn, assigner_name)
@@ -1702,22 +1692,8 @@ class Scheduler:
         return num_rows_deleted
 
     def _insert_on_conflict_do_nothing(self, model: type[OrmBase]) -> Insert:
-        # `Insert.on_conflict_do_nothing` is a SQLAlchemy "generative method", it
-        # returns a modified copy of the statement it is called on. For
-        # some reason mypy can't understand this, so the errors are ignored here.
-        if self._sql.dialect == "sqlite":
-            sqlite_insert: sqlite.Insert = sqlite.insert(model)
-            return sqlite_insert.on_conflict_do_nothing()
-
-        elif self._sql.dialect == "postgresql":
-            insertion: postgresql.Insert = postgresql.insert(model)
-            return insertion.on_conflict_do_nothing()
-
-        else:
-            # Fall back to the non-specific insert implementation. This doesn't
-            # support `ON CONFLICT DO NOTHING`, so callers need to be careful to
-            # still catch IntegrityErrors if other database backends are possible.
-            return insert(model)
+        insertion: postgresql.Insert = postgresql.insert(model)
+        return insertion.on_conflict_do_nothing()
 
     def get_or_create_client_identity_in_store(
         self, session: Session, client_id: ClientIdentityEntry
@@ -1744,7 +1720,7 @@ class Scheduler:
         try:
             session.execute(insertion)
 
-        # Handle unique constraint violation when using an unsupported database (ie. not PostgreSQL or SQLite)
+        # Handle unique constraint violation when using an unsupported database (ie. not PostgreSQL)
         except IntegrityError:
             LOGGER.debug("Handled IntegrityError when inserting client identity.")
 
@@ -1777,7 +1753,7 @@ class Scheduler:
         try:
             session.execute(insertion)
 
-        # Handle unique constraint violation when using an unsupported database (ie. not PostgreSQL or SQLite)
+        # Handle unique constraint violation when using an unsupported database (ie. not PostgreSQL)
         except IntegrityError:
             LOGGER.debug("Handled IntegrityError when inserting request metadata.")
 
@@ -1999,11 +1975,11 @@ class Scheduler:
         return None
 
     def _create_logstream_for_job(self, job: JobEntry, log_tags: Tags) -> None:
-        if self.logstream_channel and self.logstream_instance is not None:
+        if self.logstream_channel:
             try:
                 action_digest = string_to_digest(job.action_digest)
                 parent_base = f"{action_digest.hash}_{action_digest.size_bytes}_{int(time())}"
-                with logstream_client(self.logstream_channel, self.logstream_instance) as ls_client:
+                with logstream_client(self.logstream_channel, job.instance_name) as ls_client:
                     stdout_stream = ls_client.create(f"{parent_base}_stdout")
                     stderr_stream = ls_client.create(f"{parent_base}_stderr")
                 job.stdout_stream_name = stdout_stream.name
@@ -2527,9 +2503,12 @@ class Scheduler:
             Action.FromString(job.action),
             string_to_digest(job.action_digest),
             retention_hours=self.completed_action_retention_hours,
+            instance_name=job.instance_name,
         )
         if action_result.ByteSize() > 0:
-            self._update_action_result_retention(action_result, retention_hours=self.action_result_retention_hours)
+            self._update_action_result_retention(
+                action_result, retention_hours=self.action_result_retention_hours, instance_name=job.instance_name
+            )
 
         worker_duration = None
         if job.worker_start_timestamp is not None and job.worker_completed_timestamp is not None:
@@ -2773,7 +2752,9 @@ class Scheduler:
         except Exception as exc:
             LOGGER.exception("Cannot publish resource usage.", tags=dict(job_name=job_name), exc_info=exc)
 
-    def _update_action_retention(self, action: Action, action_digest: Digest, retention_hours: float | None) -> None:
+    def _update_action_retention(
+        self, action: Action, action_digest: Digest, retention_hours: float | None, instance_name: str
+    ) -> None:
         if not self.asset_client or not retention_hours:
             return
         uri = DIGEST_URI_TEMPLATE.format(digest_hash=action_digest.hash)
@@ -2790,6 +2771,7 @@ class Scheduler:
                 expire_at=expire_at,
                 referenced_blobs=referenced_blobs,
                 referenced_directories=referenced_directories,
+                instance_name=instance_name,
             )
             LOGGER.debug(
                 "Extended the retention of action.", tags=dict(digest=action_digest, retention_hours=retention_hours)
@@ -2798,7 +2780,9 @@ class Scheduler:
             LOGGER.exception("Failed to push action as an asset.", tags=dict(digest=action_digest))
             # Not a fatal path, don't reraise here
 
-    def _update_action_result_retention(self, action_result: ActionResult, retention_hours: float | None) -> None:
+    def _update_action_result_retention(
+        self, action_result: ActionResult, retention_hours: float | None, instance_name: str
+    ) -> None:
         if not self.asset_client or not retention_hours:
             return
         digest = None
@@ -2837,6 +2821,7 @@ class Scheduler:
                 expire_at=expire_at,
                 referenced_blobs=referenced_blobs,
                 referenced_directories=referenced_directories,
+                instance_name=instance_name,
             )
             LOGGER.debug(
                 "Extended the retention of action result.", tags=dict(digest=digest, retention_hours=retention_hours)
@@ -2853,18 +2838,9 @@ class Scheduler:
         if self.bot_locality_hint_limit == 0:
             return
 
-        # Insert new hint with dialect-specific seq handling
-        if self._sql.dialect == "postgresql":
-            # For PostgreSQL, use the sequence to get the next seq number
-            next_seq = None
-        else:
-            # For SQLite, manually find the max seq number
-            max_seq = session.execute(
-                select(func.coalesce(func.max(BotLocalityHintEntry.sequence_number), 0)).where(
-                    BotLocalityHintEntry.bot_name == bot_name
-                )
-            ).scalar_one()
-            next_seq = max_seq + 1
+        # Insert new hint with seq handling
+        # For PostgreSQL, use the sequence to get the next seq number
+        next_seq = None
 
         new_hint = BotLocalityHintEntry(
             bot_name=bot_name,
@@ -2969,9 +2945,6 @@ class Scheduler:
         # `greatest(0,_)` is needed if this feature is released when there are already running jobs
         # TODO: remove the safe-guard after the next minor version bump
         new_usage: Any = func.greatest(0, InstanceQuota.current_usage + delta)
-        if self._sql.dialect == "sqlite":
-            # SQLite does not support `greatest`, so we use a simpler update for it.
-            new_usage = InstanceQuota.current_usage + delta
 
         update_usage_query = (
             update(InstanceQuota)

buildgrid/server/scheduler/notifier.py

@@ -25,7 +25,7 @@ from sqlalchemy import select as sql_select
 from sqlalchemy.orm import Session
 
 from buildgrid.server.logging import buildgrid_logger
-from buildgrid.server.sql.models import BotEntry, JobEntry
+from buildgrid.server.sql.models import BotEntry
 from buildgrid.server.sql.provider import SqlProvider
 from buildgrid.server.threading import ContextWorker
 
@@ -79,10 +79,7 @@ class Notifier(Generic[T]):
         while not shutdown_requested.is_set():
             try:
                 with self._sql.session() as session:
-                    if self._sql.dialect == "postgresql":
-                        self._listen_for_updates(shutdown_requested, session)
-                    else:
-                        self._poll_for_updates(shutdown_requested, session)
+                    self._listen_for_updates(shutdown_requested, session)
             except Exception as e:
                 LOGGER.warning(
                     f"OperationsNotifier encountered exception: {e}.",
@@ -122,23 +119,6 @@ class Notifier(Generic[T]):
                 notify = dbapi_connection.notifies.pop()
                 self.notify(notify.payload)
 
-    def _poll(self, names: list[str], session: Session) -> dict[str, T]:
-        raise NotImplementedError()
-
-    def _poll_for_updates(self, shutdown_requested: Event, session: Session) -> None:
-        prev_data: dict[str, T] = {}
-        while not shutdown_requested.is_set():
-            with self._lock:
-                names = list(self._listeners)
-
-            next_data: dict[str, T] = self._poll(names, session)
-            for name in next_data:
-                if name not in prev_data or prev_data[name] != next_data[name]:
-                    self.notify(name)
-
-            prev_data = next_data
-            shutdown_requested.wait(timeout=self.poll_interval)
-
     def notify(self, listener_name: str) -> None:
         with self._lock:
             if listener_name in self._listeners:
@@ -179,27 +159,11 @@ class OperationsNotifier(Notifier[tuple[bool, int]]):
         """
         super().__init__(sql_provider, NotificationChannel.JOB_UPDATED, "OperationsNotifier", poll_interval)
 
-    def _poll(self, names: list[str], session: Session) -> dict[str, tuple[bool, int]]:
-        # Only query for the minimal amount of data required.
-        # The subscribers can choose how they want to act (e.g. by querying the full job data).
-        statement = sql_select(JobEntry.name, JobEntry.cancelled, JobEntry.stage).where(JobEntry.name.in_(names))
-        next_data: dict[str, tuple[bool, int]] = {}
-        for [name, cancelled, stage] in session.execute(statement).all():
-            next_data[name] = (cancelled, stage)
-        return next_data
-
 
 class BotNotifier(Notifier[str]):
     def __init__(self, sql_provider: SqlProvider, poll_interval: float = 1.0) -> None:
         super().__init__(sql_provider, NotificationChannel.JOB_ASSIGNED, "BotNotifier", poll_interval)
 
-    def _poll(self, names: list[str], session: Session) -> dict[str, str]:
-        statement = sql_select(BotEntry.name, BotEntry.lease_id).where(BotEntry.name.in_(names))
-        next_data = {}
-        for name, lease_id in session.execute(statement).all():
-            next_data[name] = lease_id
-        return next_data
-
     def listener_count_for_instance(self, instance_name: str) -> int:
         with self._lock:
             stmt = sql_select(func.count(BotEntry.name)).where(

buildgrid/server/sql/alembic/versions/0c17a7cb2bc5_initial_database_state.py

@@ -119,7 +119,6 @@ def upgrade() -> None:
         ["worker_completed_timestamp"],
         unique=False,
         postgresql_where=sa.text("worker_completed_timestamp IS NOT NULL"),
-        sqlite_where=sa.text("worker_completed_timestamp IS NOT NULL"),
     )
     op.create_index(
         "ix_worker_start_timestamp",
@@ -127,7 +126,6 @@ def upgrade() -> None:
         ["worker_start_timestamp"],
         unique=False,
         postgresql_where=sa.text("worker_start_timestamp IS NOT NULL"),
-        sqlite_where=sa.text("worker_start_timestamp IS NOT NULL"),
     )
     op.create_table(
         "platform_properties",
@@ -220,13 +218,11 @@ def downgrade() -> None:
         "ix_worker_start_timestamp",
         table_name="jobs",
         postgresql_where=sa.text("worker_start_timestamp IS NOT NULL"),
-        sqlite_where=sa.text("worker_start_timestamp IS NOT NULL"),
     )
     op.drop_index(
         "ix_worker_completed_timestamp",
         table_name="jobs",
         postgresql_where=sa.text("worker_completed_timestamp IS NOT NULL"),
-        sqlite_where=sa.text("worker_completed_timestamp IS NOT NULL"),
     )
     op.drop_index(op.f("ix_jobs_worker_name"), table_name="jobs")
     op.drop_index("ix_jobs_stage_property_label", table_name="jobs")

buildgrid/server/sql/models.py

@@ -37,7 +37,8 @@ from buildgrid.server.enums import LeaseState, OperationStage
 
 bigint = Annotated[int, "bigint"]
 # This gives us something to reference in the type_annotation_map to specify the JSONB variant when
-# using postgresql. When we drop SQLite support this won't be necessary versus just using JSONB in
+# using postgresql.
+# TODO now SQLite support has been dropped this won't be necessary versus just using JSONB in
 # the model directly.
 json = Annotated[JSON, "json"]
 
@@ -133,7 +134,6 @@ class JobEntry(Base):
             "worker_completed_timestamp",
             unique=False,
             postgresql_where=worker_completed_timestamp.isnot(None),
-            sqlite_where=worker_completed_timestamp.isnot(None),
         ),
         Index(
             "ix_jobs_property_label_stage",

buildgrid/server/sql/provider.py

@@ -15,11 +15,10 @@
 
 from contextlib import contextmanager
 from datetime import timedelta
-from tempfile import NamedTemporaryFile
 from threading import Lock
 from typing import Any, Generator, Iterator
 
-from sqlalchemy import create_engine, event, text
+from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import Session, scoped_session, sessionmaker
 from sqlalchemy.pool import NullPool
@@ -37,9 +36,8 @@ from buildgrid.server.sql.models import Base
 
 from .utils import (
     SQLPoolDisposeHelper,
-    is_psycopg2_connection_string,
-    is_sqlite_connection_string,
-    is_sqlite_inmemory_connection_string,
+    is_postgresql_connection_string,
+    USE_POSTGRES_MESSAGE,
 )
 
 LOGGER = buildgrid_logger(__name__)
@@ -47,15 +45,13 @@ LOGGER = buildgrid_logger(__name__)
 # Each dialect has a limit on the number of bind parameters allowed. This
 # matters because it determines how large we can allow our IN clauses to get.
 #
-# SQLite: 1000 https://www.sqlite.org/limits.html#max_variable_number
 # PostgreSQL: 32767 (Int16.MAX_VALUE) https://www.postgresql.org/docs/9.4/protocol-message-formats.html
 #
 # We'll refer to this as the "inlimit" in the code. The inlimits are
 # set to 75% of the bind parameter limit of the implementation.
-DIALECT_INLIMIT_MAP = {"postgresql": 24000, "sqlite": 750}
+DIALECT_INLIMIT_MAP = {"postgresql": 24000}
 DEFAULT_INLIMIT = 100
 
-
 # NOTE: Obviously these type annotations are useless, but sadly they're what
 # is in the upstream sqlalchemy2-stubs[0].
 #
@@ -64,24 +60,6 @@ DEFAULT_INLIMIT = 100
 #
 # [0]: https://github.com/sqlalchemy/sqlalchemy2-stubs/blob/main/sqlalchemy-stubs/pool/events.pyi#L9
 # [1]: https://github.com/sqlalchemy/sqlalchemy/blob/main/lib/sqlalchemy/pool/events.py#L96-L100
-def _sqlite_on_connect(conn: Any, record: Any) -> None:
-    """SQLite ``PRAGMA`` statements to execute immediately upon connection.
-
-    These statements configure the behaviour of the database, and are specific
-    to SQLite.
-
-    See https://www.sqlite.org/pragma.html for details.
-
-    Args:
-        conn (DBAPIConnection): The DBAPI connection that was just connected.
-        record (_ConnectionRecord): The connection record which contains the
-            DBAPI connection.
-
-    """
-    # Use journal_mode=WAL to allow read/write concurrency, as well as the
-    # performance improvements it brings.
-    conn.execute("PRAGMA journal_mode=WAL")
-    conn.execute("PRAGMA synchronous=NORMAL")
 
 
 class SqlProvider:
@@ -93,17 +71,14 @@ class SqlProvider:
 
     Args:
         connection_string (str | None): The connection string to use when
-            creating a database connection. If ``None`` then a temporary
-            SQLite database will be created for the lifetime of this
-            object.
+            creating a database connection. Must be a valid postgres database.
 
         connection_timeout (int): The timeout to use when attempting to
             connect to the database, in seconds. Defaults to 5 seconds if
             unset.
 
         lock_timeout (int): The timeout to use when the connection
-        holds a lock in the database. This is supported only if the database
-        backend is PostgresQL.
+        holds a lock in the database.
 
         connect_args (dict[str, Any] | None): Dictionary of DBAPI
             connection arguments to pass to the engine. See the
@@ -132,7 +107,7 @@ class SqlProvider:
             publishing.
 
     Raises:
-        ValueError: when ``connection_string`` specifies an in-memory SQLite
+        ValueError: when ``connection_string`` doesn't specify a Postgresql
             database.
 
     .. _docs: https://docs.sqlalchemy.org/en/14/core/engines.html#use-the-connect-args-dictionary-parameter
@@ -155,16 +130,10 @@ class SqlProvider:
     ):
         """Initialize an SqlProvider."""
         self._database_tempfile = None
-        # If we don't have a connection string, we'll make a tempfile to use
-        # as an SQLite database. This tempfile needs to exist for the lifetime
-        # of the SqlProvider.
+        # If we don't have a connection string, we'll throw a ValueError and some info about setting up a
+        # postgres database.
         if not connection_string:
-            self._database_tempfile = NamedTemporaryFile(prefix="bgd-", suffix=".db")
-            LOGGER.warning(
-                "No connection string specified for the SQL provider, will use SQLite with tempfile.",
-                tags=dict(tempfile=self._database_tempfile.name),
-            )
-            connection_string = f"sqlite:///{self._database_tempfile.name}"
+            raise ValueError(f"No connection string specified for the SQL provider\n\n{USE_POSTGRES_MESSAGE}")
 
         # Set up database connection
         self._session_factory = sessionmaker(future=True)
@@ -224,8 +193,7 @@ class SqlProvider:
                 is applied.
 
             lock_timeout (int): The timeout to use when the connection
-            holds a lock in the database. This is supported only if the database
-            backend is PostgresQL.
+            holds a lock in the database.
 
             connect_args: Dictionary of DBAPI
                 connection arguments to pass to the engine. See the
@@ -257,35 +225,24 @@ class SqlProvider:
                 database defined by ``connection_string``.
 
         Raises:
-            ValueError: when attempting to connect to an in-memory SQLite
+            ValueError: when attempting to connect to a non Postgresql
                 database.
 
         .. _docs: https://docs.sqlalchemy.org/en/14/core/engines.html#use-the-connect-args-dictionary-parameter
 
         """
+        # Disallow sqlite for the scheduler db
+        # theres no reason to support a non production ready scheduler implementation
+
         # Disallow sqlite in-memory because multi-threaded access to it is
         # complex and potentially problematic at best
         # ref: https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#threading-pooling-behavior
-        if is_sqlite_inmemory_connection_string(connection_string):
-            raise ValueError(
-                "Cannot use SQLite in-memory with BuildGrid "
-                f"(connection_string=[{connection_string}]). Use a file or "
-                "leave the connection_string empty for a tempfile."
-            )
 
-        # Deprecate sqlite for the scheduler db
-        if is_sqlite_connection_string(connection_string):
-            LOGGER.warning(
-                "\n\n"
-                "Using deprecated dialect SQLite.\n"
-                "\n"
-                "Deployments, CI pipelines and developer setups will need to switch any SQLite storage"
-                "(cas storage, cas index, job scheduler) over to Postgresql before updating.\n"
-                "\n"
-                "For non production use a preconfigured docker buildgrid database is available from:"
-                "registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid-postgres:nightly"
-                "an example compose file for the database can be found at "
-                "https://gitlab.com/BuildGrid/buildgrid.hub.docker.com/-/blob/master/Composefile.buildbox.yml?ref_type=heads\n"  # noqa: E501
+        # Ensure only postgres is supported
+
+        if not is_postgresql_connection_string(connection_string):
+            raise ValueError(
+                f"Cannot use database (connection_string=[{connection_string}]).\n\n{USE_POSTGRES_MESSAGE}"
             )
 
         extra_engine_args: dict[str, Any] = {}
@@ -295,12 +252,9 @@ class SqlProvider:
             extra_engine_args["connect_args"] = {}
 
         if connection_timeout > 0:
-            if is_sqlite_connection_string(connection_string):
-                extra_engine_args["connect_args"]["timeout"] = connection_timeout
-            elif is_psycopg2_connection_string(connection_string):
-                extra_engine_args["connect_args"]["connect_timeout"] = connection_timeout
-        if lock_timeout > 0 and is_psycopg2_connection_string(connection_string):
-            # Additional postgres specific timeouts
+            extra_engine_args["connect_args"]["connect_timeout"] = connection_timeout
+        if lock_timeout > 0:
+            # Additional timeouts
             # Additional libpg options
             # Note that those timeouts are in milliseconds (so *1000)
             # User might specifically set options... do not override in this case.
@@ -326,10 +280,6 @@ class SqlProvider:
         engine = create_engine(connection_string, echo=False, future=True, **extra_engine_args)
         self._session_factory.configure(bind=engine)
 
-        # Register sqlite-specific connection callback.
-        if engine.dialect.name == "sqlite":
-            event.listen(engine, "connect", _sqlite_on_connect)
-
         return engine
 
     @property
@@ -347,7 +297,6 @@ class SqlProvider:
         self,
         *,
         scoped: bool = False,
-        sqlite_lock_immediately: bool = False,
         exceptions_to_not_raise_on: list[type[Exception]] | None = None,
         exceptions_to_not_rollback_on: list[type[Exception]] | None = None,
         expire_on_commit: bool = True,
@@ -362,10 +311,6 @@ class SqlProvider:
         if the underlying connection pool has recently been disposed of and
         refreshed due to connectivity issues.
 
-        When ``sqlite_lock_immediately`` is ``True``, the Session will not
-        yield until the database has been locked by entering into a write
-        transaction when using SQLite.
-
         If an Exception is raised whilst in the managed context, the ongoing
         database transaction is rolled back, and the Exception is reraised.
         Some Exceptions which suggest a transient connection issue with the
@@ -386,13 +331,6 @@ class SqlProvider:
                 session. This results in reuse of the underlying Session object
                 in a given thread.
 
-            sqlite_lock_immediately: If true, execute a ``BEGIN IMMEDIATE``
-                statement as soon as the session is created when using SQLite.
-                This allows locking for the lifetime of the ``Session`` within
-                this ContextManager, enabling similar behaviour to
-                ``SELECT ... FOR UPDATE`` in other dialects. Defaults to
-                ``False``.
-
             exceptions_to_not_raise_on: The list of error types to be suppressed
                 within the context rather than re-raised. Defaults to ``None``,
                 meaning all SQLAlchemyErrors will be re-raised.
@@ -442,8 +380,6 @@ class SqlProvider:
         # Try to obtain a session
         try:
             session = factory() if scoped else factory(expire_on_commit=expire_on_commit)
-            if sqlite_lock_immediately and session.bind.name == "sqlite":  # type: ignore
-                session.execute(text("BEGIN IMMEDIATE"))
         except Exception as e:
             LOGGER.error("Unable to obtain a database session.", exc_info=True)
             raise DatabaseError("Unable to obtain a database session.") from e
@@ -489,7 +425,6 @@ class SqlProvider:
     def scoped_session(
         self,
         *,
-        sqlite_lock_immediately: bool = False,
         exceptions_to_not_raise_on: list[type[Exception]] | None = None,
         exceptions_to_not_rollback_on: list[type[Exception]] | None = None,
     ) -> Generator[Session, None, None]:
@@ -512,7 +447,6 @@ class SqlProvider:
         """
         with self.session(
             scoped=True,
-            sqlite_lock_immediately=sqlite_lock_immediately,
             exceptions_to_not_raise_on=exceptions_to_not_raise_on,
             exceptions_to_not_rollback_on=exceptions_to_not_rollback_on,
         ) as session:

buildgrid/server/sql/utils.py

@@ -89,14 +89,16 @@ LIST_OPERATIONS_SORT_KEYS = {
     "command": SortKeySpec("command", JobEntry.__tablename__),
 }
 
-
-def is_sqlite_connection_string(connection_string: str) -> bool:
-    if connection_string:
-        return connection_string.startswith("sqlite")
-    return False
+USE_POSTGRES_MESSAGE = (
+    "For production use setup a postgresql database.\n"
+    "For CI and local development use the preconfigured docker buildgrid database from:\n"
+    "registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid-postgres:nightly\n"
+    "an example compose file for the database can be found at \n"
+    "https://gitlab.com/BuildGrid/buildgrid.hub.docker.com/-/blob/master/Composefile.buildbox.yml?ref_type=heads\n"  # noqa: E501
+)
 
 
-def is_psycopg2_connection_string(connection_string: str) -> bool:
+def is_postgresql_connection_string(connection_string: str) -> bool:
     if connection_string:
         if connection_string.startswith("postgresql:"):
             return True
@@ -105,35 +107,6 @@ def is_psycopg2_connection_string(connection_string: str) -> bool:
     return False
 
 
-def is_sqlite_inmemory_connection_string(full_connection_string: str) -> bool:
-    if is_sqlite_connection_string(full_connection_string):
-        # Valid connection_strings for in-memory SQLite which we don't support could look like:
-        # "sqlite:///file:memdb1?option=value&cache=shared&mode=memory",
-        # "sqlite:///file:memdb1?mode=memory&cache=shared",
-        # "sqlite:///file:memdb1?cache=shared&mode=memory",
-        # "sqlite:///file::memory:?cache=shared",
-        # "sqlite:///file::memory:",
-        # "sqlite:///:memory:",
-        # "sqlite:///",
-        # "sqlite://"
-        # ref: https://www.sqlite.org/inmemorydb.html
-        # Note that a user can also specify drivers, so prefix could become 'sqlite+driver:///'
-        connection_string = full_connection_string
-
-        uri_split_index = connection_string.find("?")
-        if uri_split_index != -1:
-            connection_string = connection_string[0:uri_split_index]
-
-        if connection_string.endswith((":memory:", ":///", "://")):
-            return True
-        elif uri_split_index != -1:
-            opts = full_connection_string[uri_split_index + 1 :].split("&")
-            if "mode=memory" in opts:
-                return True
-
-    return False
-
-
 class SQLPoolDisposeHelper:
     """Helper class for disposing of SQL session connections"""
 

buildgrid/server/version.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 
 
-__version__ = "0.2.53"
+__version__ = "0.3.1"

{buildgrid-0.2.53.dist-info → buildgrid-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildgrid
-Version: 0.2.53
+Version: 0.3.1
 Summary: A remote execution service
 License: Apache License, Version 2.0
 Project-URL: Homepage, https://buildgrid.build
@@ -42,7 +42,7 @@ Requires-Dist: fakeredis>=2.10.1; extra == "redis"
 Requires-Dist: redis>=4.5.1; extra == "redis"
 Requires-Dist: hiredis; extra == "redis"
 Provides-Extra: docs
-Requires-Dist: Sphinx; extra == "docs"
+Requires-Dist: Sphinx<=8; extra == "docs"
 Requires-Dist: sphinx-click; extra == "docs"
 Requires-Dist: sphinx-rtd-theme; extra == "docs"
 Requires-Dist: sphinxcontrib-apidoc; extra == "docs"
@@ -77,6 +77,7 @@ Requires-Dist: pytest-xdist; extra == "dev"
 Requires-Dist: memray; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: grpcio-tools; extra == "dev"
+Requires-Dist: bump4version; extra == "dev"
 Provides-Extra: mypy
 Requires-Dist: mypy; extra == "mypy"
 Requires-Dist: SQLAlchemy[mypy]; extra == "mypy"

{buildgrid-0.2.53.dist-info → buildgrid-0.3.1.dist-info}/RECORD

@@ -208,7 +208,7 @@ buildgrid/server/servicer.py,sha256=oqU9MaSxxHTDmSxobFTo9YmJctaUCklE2Dj-vfYWKkc,
 buildgrid/server/settings.py,sha256=Iy4eS9Putr5GroIVqYNeTcRY7gbvq29wgQoMDQgPJtA,5604
 buildgrid/server/threading.py,sha256=4QKQYev2KoO2Q-S_OyaoR9qpWyDTVzGMWVe9o2a1yIU,4743
 buildgrid/server/types.py,sha256=xG3bx64pbWMuEwXLuI0o8c2unt2rU2C4zsmUfmMT12c,1323
-buildgrid/server/version.py,sha256=zv7pfyNMnfQiih7uFjY-_UFi2yhATpDRA43o8Nz28fo,604
+buildgrid/server/version.py,sha256=n9eZStzsUHVKCm33O5kD4RgMgRgIUVgOkmGO0lowsqw,603
 buildgrid/server/actioncache/__init__.py,sha256=g9lb8Sn7NY5KOjkMr9GQoJovCVDEg_Fxz_EhdDbhP1I,579
 buildgrid/server/actioncache/instance.py,sha256=UCR7ZGkv4fJOXjeIILMAdTSFWcGgBSYlBg8fMaPJpaI,3139
 buildgrid/server/actioncache/service.py,sha256=WcikJAzFYOYX-tgiOfGGcOnPoubrCd4yP-EhKCHEW0c,2021
@@ -239,9 +239,9 @@ buildgrid/server/app/commands/rpc_utils.py,sha256=3C02_0Ba4Weksb2kX5mANSubmfWAFR
 buildgrid/server/app/settings/__init__.py,sha256=g9lb8Sn7NY5KOjkMr9GQoJovCVDEg_Fxz_EhdDbhP1I,579
 buildgrid/server/app/settings/config.py,sha256=3cmmnkvZtM44zXkc6meNpmA6rJoDBbrridHoj-Quapo,9649
 buildgrid/server/app/settings/mapper.py,sha256=sKD3LWyVXZWaFF16_JN1lbJkUE8vp5p06OJGOencxmU,752
-buildgrid/server/app/settings/parser.py,sha256=8hIlclUweGO9kcWdOEy9f_TeEshofjJhH56dwnKXFak,96742
+buildgrid/server/app/settings/parser.py,sha256=i8B1Zy6sR2o3n4z8H_e_QlszMTKOiLFPxonVE25MkOo,96439
 buildgrid/server/app/settings/reference.yml,sha256=DTXNJ6TkccgBLcvzj5264XAv6HFlvsc5Lx6iowixHPE,10703
-buildgrid/server/app/settings/schema.yml,sha256=1Jgvo-ruEVXQhDK_g323-OjSV7-QGZ_D_86ch-7pZCs,32765
+buildgrid/server/app/settings/schema.yml,sha256=DqWE6UaNYiJl0Ae5DSmDtU8CijLXeXAeSK9Qal0jwTA,32664
 buildgrid/server/auth/__init__.py,sha256=avyRxG9BSFd76mCHnGYInDDTqyqDn0UlhOuFzAkPQSs,579
 buildgrid/server/auth/config.py,sha256=sWcaSL9Oz7mLVnFBuFT7OaOGlxFd6hxEb-z30n3uTNU,2291
 buildgrid/server/auth/enums.py,sha256=O902XU3_C4rqWz23qVipEyQMcXXr1h8XCiPWmrs1_lc,1905
@@ -268,16 +268,13 @@ buildgrid/server/cas/storage/replicated.py,sha256=DF_oku2QJAigiRTz6358ZBy4LzQiIH
 buildgrid/server/cas/storage/s3.py,sha256=fHsbNBYBN6x2DnlG22_UA4GMjcqGEMlOU6yIHueE5mc,20058
 buildgrid/server/cas/storage/sharded.py,sha256=WYehvpn1AD-pvGsZDjzIZQRLjCyw7eEEjHymYoMzg2Q,7076
 buildgrid/server/cas/storage/size_differentiated.py,sha256=puT7xMhT_0T1hKGJf_kjjbCcYsmfhQnsEvsRClmk59Y,8223
-buildgrid/server/cas/storage/sql.py,sha256=T4huxHnZMxR5JnalonzSI45H_tWgICxF9dl3LOjWpYo,10274
+buildgrid/server/cas/storage/sql.py,sha256=ERiHfqF9DA1Tu75QxHP2okAtQB3oA-DCFA95D6KQ6Js,9589
 buildgrid/server/cas/storage/storage_abc.py,sha256=BTLNiAr31amzz22mcLy8ctM9lCcNypL79iD3hPmIUMI,7728
 buildgrid/server/cas/storage/with_cache.py,sha256=IB-pq5S6R6V037RQiS4jGk9Jm5Wj_Qdy7WChs0xiDqI,8132
 buildgrid/server/cas/storage/index/__init__.py,sha256=adgShFqjP778F2KJoM-z7stkT9V_3BPuc3uodlf1buw,579
 buildgrid/server/cas/storage/index/index_abc.py,sha256=JaFHjnePcazCWqCwsrlBYybArc4d4KBM6Utv4xyK7gI,3189
 buildgrid/server/cas/storage/index/redis.py,sha256=nJqZ9HEk65zxNnrD-7sQ6aS5PGy5l55p6-Rs-oSRSZY,16556
-buildgrid/server/cas/storage/index/sql.py,sha256=EAYZ9rhbaLbPDtu7m0At5wmxn5-If99sBY16fA3KSZI,40485
-buildgrid/server/cas/storage/index/sql_dialect_delegates/__init__.py,sha256=2J1pwjLJgO2wScTgQ5p7KaqGxdfKZXf4ouu6oKn5SVM,146
-buildgrid/server/cas/storage/index/sql_dialect_delegates/postgresqldelegate.py,sha256=tIXpp2wYgc3Hd5oXdZEAuhGLHrbDUe4EmKjgZPkik7Y,2316
-buildgrid/server/cas/storage/index/sql_dialect_delegates/sqlitedelegate.py,sha256=Pq7QAkT0GeHrTyDnUFd-9nJ6moFqynAEHDiGBBjjKcc,2300
+buildgrid/server/cas/storage/index/sql.py,sha256=b01u2DdalQqMWMf9_SCE6l7i2Vp4_61pmeul7Qk8bWU,41016
 buildgrid/server/cleanup/__init__.py,sha256=TTer4pMMV4HOzR6rYI9yPzDlMXEWnY_SXczlHiFZCng,579
 buildgrid/server/cleanup/cleanup.py,sha256=Yyhc9LZpvIZnQNVn-mgnMfdPBrd1x9KMsqrHwmRhzf8,13606
 buildgrid/server/cleanup/janitor/__init__.py,sha256=XaoGqSD-oQwbfSl8OcR0U4pScRcBueB4sU1tpcZNGtk,579
@@ -289,7 +286,7 @@ buildgrid/server/cleanup/janitor/types.py,sha256=R3F9OcF1OsRncmHPz6k7G2iSdOxqOw9
 buildgrid/server/cleanup/janitor/utils.py,sha256=eYxuJZFJ-7Hhlrj3B3qJvL9X3Se0GEAlkNlXf88swYU,1410
 buildgrid/server/client/__init__.py,sha256=g9lb8Sn7NY5KOjkMr9GQoJovCVDEg_Fxz_EhdDbhP1I,579
 buildgrid/server/client/actioncache.py,sha256=2Xd7HzDRzMWwyiWN1nno1glkGEXrQtjmIFzsa0SHfyQ,5021
-buildgrid/server/client/asset.py,sha256=3iEaf1ggGH6xYnU9xdprXyPpvqyVId5IJGWDX54X9tA,4454
+buildgrid/server/client/asset.py,sha256=eZdMuqwrJgPy9U7Q2Qe98P1Vzrry0jZUgxXqH18ru_A,4426
 buildgrid/server/client/auth_token_loader.py,sha256=B95uDkAsHg5p98-jZwBWZAkXEfLVWjgWEhdAC1k8fDU,1604
 buildgrid/server/client/authentication.py,sha256=6J-5PKR5AhBm_CTXWegAsHjUrmwny1wYVCEgfbMs9TM,5140
 buildgrid/server/client/capabilities.py,sha256=2fKyjy-BZK5szC_wQRmGc7bfMx2-G_zl5VZ2uHt6RA4,1687
@@ -334,18 +331,18 @@ buildgrid/server/scheduler/__init__.py,sha256=arCg8LWFATeX1tj-s0keVYP8p3wwrrUlCV
 buildgrid/server/scheduler/assigner.py,sha256=wHPAhyiQxYABZJXaUc2g5yFzM78Z0U5nvGV3X9h5pCM,10512
 buildgrid/server/scheduler/cohorts.py,sha256=L_5YZRiVOwPPGStfqnnQXknO5Ja-SC0vq0xjw4XgP-I,1426
 buildgrid/server/scheduler/events.py,sha256=cM7Z7Htr2pYKhltJxfg1YRo0q524yZaGm8yXvRehivk,1453
-buildgrid/server/scheduler/impl.py,sha256=MR3Hp0hxuYTpFu0hnriin2GcY1xgri_HAok4EaeToow,137584
-buildgrid/server/scheduler/notifier.py,sha256=uypoIXZowpAIsDQ728VCuFJ4MN2zJld1npnanCWHTrw,8854
+buildgrid/server/scheduler/impl.py,sha256=jqwf4HU6otf0XLpq0hFUC8JhOgyFKPKYd2-x23biz1M,135712
+buildgrid/server/scheduler/notifier.py,sha256=22ZsKwyf2oQirAjrwROkvgvr4C_TMUNyhOmtro4uM4I,7121
 buildgrid/server/scheduler/properties.py,sha256=2GydX8KUy9MFv1_JznIkGfWE_wOS0m_XapSv6Gp4pCM,11260
 buildgrid/server/sql/__init__.py,sha256=zbeeRP9BEeDzR-Mx2Ip6SUr49J8eeXsuREgljJTrHkk,579
-buildgrid/server/sql/models.py,sha256=Pd43qOhmtDDSV0AaHiZDeCPP09W4osiLMBLW3v_wU8w,14424
-buildgrid/server/sql/provider.py,sha256=xRF5_GDkz1u1VawcNzySLEqTpIV6FSzRsZIo8iDYEzM,22544
-buildgrid/server/sql/utils.py,sha256=yXG6wWIqKk2AP5nRpZ6AmuH8vuSQy-Zk3L_NeoRNKCg,17414
+buildgrid/server/sql/models.py,sha256=jAuNLLvdlBWf7uZWlPLMSC1dTWHYlqlk9Pn5A04rQcg,14374
+buildgrid/server/sql/provider.py,sha256=CXOpjcUY-InwmGnZTpCd4_ziOdROiDY34_SRpqsBwwk,18965
+buildgrid/server/sql/utils.py,sha256=j76Z_qtGawz6O7vO4-zgCzBV4ylhKszcd_6iY0gV4W4,16470
 buildgrid/server/sql/alembic/README,sha256=MVlc9TYmr57RbhXET6QxgyCcwWP7w-vLkEsirENqiIQ,38
 buildgrid/server/sql/alembic/env.py,sha256=vRRLEpyPS_a-jeKKYvORE0NW2goBnjN7v62n7ix_t28,4534
 buildgrid/server/sql/alembic/script.py.mako,sha256=z6re8oZ_Qk3kZBPMRWWl0b0_sb29J9V7ruHwBlODDHw,1090
 buildgrid/server/sql/alembic/versions/0596ea8f5c61_add_bot_locality_hints_table_and_.py,sha256=lFYk8PPZs2TNBDKbDlYKcDlxqt4CQUO2Jts_sPymeao,2909
-buildgrid/server/sql/alembic/versions/0c17a7cb2bc5_initial_database_state.py,sha256=l2F2549VRygDOi_xuZ4Zal0S411JToWYlSy_L3eju-s,11713
+buildgrid/server/sql/alembic/versions/0c17a7cb2bc5_initial_database_state.py,sha256=UlIiOq9d0D5pTXsdIr8k454yYvCv14uQ2XCYFzb2V3E,11433
 buildgrid/server/sql/alembic/versions/12992085e81a_add_a_job_index_on_worker_name_and_.py,sha256=Dg5kxHvR4REXmhzpd72EXriMZRCFzO9XRTPRKmWcluY,1681
 buildgrid/server/sql/alembic/versions/1f959c3834d3_drop_the_leases_table.py,sha256=V0X1XVVB_iq3nrwnVWDM7rhVrFDG3T8byZChsNwjcFA,1645
 buildgrid/server/sql/alembic/versions/22cc661efef9_add_instance_quotas_table.py,sha256=zBf23GKRMJq5scFjWC83oMHvKy7K55zOTyWYrbTuJtI,1702
@@ -368,9 +365,9 @@ buildgrid/server/utils/async_lru_cache.py,sha256=iLKeRPoZtZb1wC5AtcyQm8Wt0Bx-KZm
 buildgrid/server/utils/bots.py,sha256=c8hn7tbCecru-m2wicRmtKU5v5rSZPGlk97Yc6eUHgQ,1729
 buildgrid/server/utils/cancellation.py,sha256=pNETzKNoXg0AsXOXKCcLWlFl7SVKdkKinlqWl7MesRA,1703
 buildgrid/server/utils/digests.py,sha256=YNrWeHdbNp7OVTcsInjs30C33z_t9GQ_noMd14bpqPQ,2424
-buildgrid-0.2.53.dist-info/licenses/LICENSE,sha256=swa3Vs7GgALaG9p-e05M-WLkhd_U9QknacNkyVZ85xA,11338
-buildgrid-0.2.53.dist-info/METADATA,sha256=U9a7b52eyU6pWfE8PIMYE-Rl4TvhbC9d_Po0MEHnXG4,7040
-buildgrid-0.2.53.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-buildgrid-0.2.53.dist-info/entry_points.txt,sha256=uyFAXiR9d6EDfSA5vWT8xskz6xalt4PdTuRruT6Q8rk,49
-buildgrid-0.2.53.dist-info/top_level.txt,sha256=T6TYhI_k6NTm2871tIxGCyBIqzlKxylgF9KDLU0Hi7o,10
-buildgrid-0.2.53.dist-info/RECORD,,
+buildgrid-0.3.1.dist-info/licenses/LICENSE,sha256=swa3Vs7GgALaG9p-e05M-WLkhd_U9QknacNkyVZ85xA,11338
+buildgrid-0.3.1.dist-info/METADATA,sha256=_Tt0d76-FzhNXCN5Zm7GNYMl0GgVS3br_ok2uQABmgA,7086
+buildgrid-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+buildgrid-0.3.1.dist-info/entry_points.txt,sha256=uyFAXiR9d6EDfSA5vWT8xskz6xalt4PdTuRruT6Q8rk,49
+buildgrid-0.3.1.dist-info/top_level.txt,sha256=T6TYhI_k6NTm2871tIxGCyBIqzlKxylgF9KDLU0Hi7o,10
+buildgrid-0.3.1.dist-info/RECORD,,

buildgrid/server/cas/storage/index/sql_dialect_delegates/__init__.py

@@ -1,4 +0,0 @@
-from .postgresqldelegate import PostgreSQLDelegate
-from .sqlitedelegate import SQLiteDelegate
-
-__all__ = ["PostgreSQLDelegate", "SQLiteDelegate"]

buildgrid/server/cas/storage/index/sql_dialect_delegates/postgresqldelegate.py

@@ -1,65 +0,0 @@
-# Copyright (C) 2020 Bloomberg LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  <http://www.apache.org/licenses/LICENSE-2.0>
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-PostgreSQLDelegate
-==================
-
-Extra functionality for the SQL index when using a PostgreSQL backend.
-
-"""
-
-from datetime import datetime
-from typing import cast
-
-from sqlalchemy import Table
-from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy.sql.functions import coalesce
-
-from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest
-from buildgrid.server.sql.models import IndexEntry
-
-
-class PostgreSQLDelegate:
-    @staticmethod
-    def _save_digests_to_index(
-        digest_blob_pairs: list[tuple[Digest, bytes | None]], session: SessionType, max_inline_blob_size: int
-    ) -> None:
-        # See discussion of __table__ typing in https://github.com/sqlalchemy/sqlalchemy/issues/9130
-        index_table = cast(Table, IndexEntry.__table__)
-        update_time = datetime.utcnow()
-        new_rows = [
-            {
-                "digest_hash": digest.hash,
-                "digest_size_bytes": digest.size_bytes,
-                "accessed_timestamp": update_time,
-                "inline_blob": (blob if digest.size_bytes <= max_inline_blob_size else None),
-                "deleted": False,
-            }
-            for (digest, blob) in digest_blob_pairs
-        ]
-
-        base_insert_stmt = insert(index_table).values(new_rows)
-
-        update_stmt = base_insert_stmt.on_conflict_do_update(
-            index_elements=["digest_hash"],
-            set_={
-                "accessed_timestamp": update_time,
-                "inline_blob": coalesce(base_insert_stmt.excluded.inline_blob, index_table.c.inline_blob),
-                "deleted": False,
-            },
-        )
-
-        session.execute(update_stmt)

buildgrid/server/cas/storage/index/sql_dialect_delegates/sqlitedelegate.py

@@ -1,65 +0,0 @@
-# Copyright (C) 2020 Bloomberg LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  <http://www.apache.org/licenses/LICENSE-2.0>
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-SQLiteDelegate
-==================
-
-Extra functionality for the SQL index when using a SQLite backend.
-
-"""
-
-from datetime import datetime
-from typing import cast
-
-from sqlalchemy import Table
-from sqlalchemy.dialects.sqlite import insert
-from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy.sql.functions import coalesce
-
-from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest
-from buildgrid.server.sql.models import IndexEntry
-
-
-class SQLiteDelegate:
-    @staticmethod
-    def _save_digests_to_index(
-        digest_blob_pairs: list[tuple[Digest, bytes | None]], session: SessionType, max_inline_blob_size: int
-    ) -> None:
-        # See discussion of __table__ typing in https://github.com/sqlalchemy/sqlalchemy/issues/9130
-        index_table = cast(Table, IndexEntry.__table__)
-        update_time = datetime.utcnow()
-        new_rows = [
-            {
-                "digest_hash": digest.hash,
-                "digest_size_bytes": digest.size_bytes,
-                "accessed_timestamp": update_time,
-                "inline_blob": (blob if digest.size_bytes <= max_inline_blob_size else None),
-                "deleted": False,
-            }
-            for (digest, blob) in digest_blob_pairs
-        ]
-
-        base_insert_stmt = insert(index_table).values(new_rows)
-
-        update_stmt = base_insert_stmt.on_conflict_do_update(
-            index_elements=["digest_hash"],
-            set_={
-                "accessed_timestamp": update_time,
-                "inline_blob": coalesce(base_insert_stmt.excluded.inline_blob, index_table.c.inline_blob),
-                "deleted": False,
-            },
-        )
-
-        session.execute(update_stmt)