seamless-database 2.0.1__tar.gz → 2.1.0__tar.gz

{seamless_database-2.0.1 → seamless_database-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: seamless-database
-Version: 2.0.1
+Version: 2.1.0
 Summary: SQLite-backed metadata database service for Seamless
 Author: Sjoerd de Vries
 License-Expression: MIT
@@ -13,7 +13,6 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries
@@ -70,6 +69,27 @@ seamless-database seamless.db --port-range 5520 5530 --writable
 seamless-database seamless.db --port 5522
 ```
 
+If `--port` and `--port-range` are both omitted, `seamless-database` picks a random free port in the dynamic/private range (`49152-65535`).
+
+### Status-file protocol
+
+`seamless-database` does not require a status file. If `--status-file` is omitted, it runs independently.
+
+If `--status-file` is provided, the file is used for two things:
+
+1. Report the chosen port, especially when `--port-range` is used.
+2. Report whether startup succeeded (`"running"`) or failed (`"failed"`).
+
+The status-file protocol is simple:
+
+1. Wait for the status file to exist and parse it as JSON.
+2. Reuse the existing JSON object as the base payload. An empty JSON object `{}` is sufficient.
+3. Choose or validate its listening port.
+4. Once the HTTP server is up, rewrite the same file with `"status": "running"` and the selected `"port"`.
+5. If startup fails before the server is running, rewrite the file with `"status": "failed"` instead.
+
+If `remote-http-launcher` is used, it may pre-populate the JSON with fields such as the PID, workdir, or `"status": "starting"`. `seamless-database` preserves such fields when it writes back the final status.
+
 ### CLI options
 
 | Option | Description |

{seamless_database-2.0.1 → seamless_database-2.1.0}/README.md

@@ -46,6 +46,27 @@ seamless-database seamless.db --port-range 5520 5530 --writable
 seamless-database seamless.db --port 5522
 ```
 
+If `--port` and `--port-range` are both omitted, `seamless-database` picks a random free port in the dynamic/private range (`49152-65535`).
+
+### Status-file protocol
+
+`seamless-database` does not require a status file. If `--status-file` is omitted, it runs independently.
+
+If `--status-file` is provided, the file is used for two things:
+
+1. Report the chosen port, especially when `--port-range` is used.
+2. Report whether startup succeeded (`"running"`) or failed (`"failed"`).
+
+The status-file protocol is simple:
+
+1. Wait for the status file to exist and parse it as JSON.
+2. Reuse the existing JSON object as the base payload. An empty JSON object `{}` is sufficient.
+3. Choose or validate its listening port.
+4. Once the HTTP server is up, rewrite the same file with `"status": "running"` and the selected `"port"`.
+5. If startup fails before the server is running, rewrite the file with `"status": "failed"` instead.
+
+If `remote-http-launcher` is used, it may pre-populate the JSON with fields such as the PID, workdir, or `"status": "starting"`. `seamless-database` preserves such fields when it writes back the final status.
+
 ### CLI options
 
 | Option | Description |

{seamless_database-2.0.1 → seamless_database-2.1.0}/database.py

@@ -20,9 +20,13 @@ from database_models import (
     SyntacticToSemantic,
     Expression,
     MetaData,
+    BucketProbe,
     IrreproducibleTransformation,
 )
 
+DEFAULT_RANDOM_PORT_START = 49152
+DEFAULT_RANDOM_PORT_END = 65535
+
 
 STATUS_FILE_WAIT_TIMEOUT = 20.0
 INACTIVITY_CHECK_INTERVAL = 1.0
@@ -135,6 +139,105 @@ class DatabaseError(Exception):
     pass
 
 
+def _conflict_response(message: str) -> web.Response:
+    return web.Response(status=409, text="ERROR: " + message)
+
+
+def _normalize_metadata_value(value):
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, str):
+        value = json.loads(value)
+        if isinstance(value, dict):
+            return value
+    raise TypeError(type(value))
+
+
+def _normalize_freshness_tokens(value):
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, str):
+        value = json.loads(value)
+        if isinstance(value, dict):
+            return value
+    raise TypeError(type(value))
+
+
+def _validate_execution_record_identity(
+    checksum: str, result: str, value: dict
+) -> None:
+    schema_version = value.get("schema_version")
+    if not isinstance(schema_version, int):
+        raise ValueError("schema_version")
+    if value.get("tf_checksum") != checksum:
+        raise ValueError("tf_checksum")
+    if value.get("result_checksum") != result:
+        raise ValueError("result_checksum")
+    checksum_fields = value.get("checksum_fields")
+    if checksum_fields is not None:
+        if not isinstance(checksum_fields, list):
+            raise ValueError("checksum_fields")
+        for field in checksum_fields:
+            if not isinstance(field, str) or not field:
+                raise ValueError("checksum_fields")
+
+
+def _validate_bucket_probe_request(
+    bucket_kind: str,
+    label: str,
+    bucket_checksum: str,
+    freshness_tokens: dict,
+    captured_at: str,
+) -> None:
+    if bucket_kind not in BUCKET_KINDS:
+        raise ValueError("bucket_kind")
+    if not isinstance(label, str) or not label:
+        raise ValueError("label")
+    parse_checksum(bucket_checksum, as_bytes=False)
+    if not isinstance(freshness_tokens, dict):
+        raise ValueError("freshness_tokens")
+    if not isinstance(captured_at, str) or not captured_at:
+        raise ValueError("captured_at")
+
+
+def _get_transformation_row(checksum: str):
+    try:
+        return Transformation[checksum]
+    except DoesNotExist:
+        return None
+
+
+def _get_metadata_row(checksum: str):
+    try:
+        return MetaData[checksum]
+    except DoesNotExist:
+        return None
+
+
+def _ensure_rev_transformation_row(checksum: str, result: str) -> None:
+    query = RevTransformation.select().where(
+        RevTransformation.checksum == checksum, RevTransformation.result == result
+    )
+    if query.exists():
+        return
+    RevTransformation.create(checksum=checksum, result=result)
+
+
+def _delete_rev_transformation_rows(checksum: str, result: str) -> None:
+    (
+        RevTransformation.delete()
+        .where(RevTransformation.checksum == checksum, RevTransformation.result == result)
+        .execute()
+    )
+
+
+def _irreproducible_exists(checksum: str) -> bool:
+    query = IrreproducibleTransformation.select().where(
+        IrreproducibleTransformation.checksum == checksum
+    )
+    return query.exists()
+
+
 def is_port_in_use(address, port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         return s.connect_ex((address, port)) == 0
@@ -244,8 +347,9 @@ types = (
     "semantic_to_syntactic",
     "transformation",
     "metadata",
+    "bucket_probe",
     "expression",
-    "irreproducible",  # only PUT
+    "irreproducible",
     "rev_expression",  # only GET
     "rev_transformations",  # only GET
 )
@@ -273,7 +377,7 @@ def format_response(response, *, none_as_404=False):
 
 class DatabaseServer:
     future = None
-    PROTOCOL = ("seamless", "database", "2.0")
+    PROTOCOL = ("seamless", "database", "2.2")
 
     def __init__(
         self,
@@ -387,13 +491,15 @@ class DatabaseServer:
                     type_ = rq["type"]
                     if type_ not in types:
                         raise KeyError
-                    if type_ != "protocol":
+                    if type_ not in ("protocol", "bucket_probe"):
                         checksum = rq["checksum"]
                 except KeyError:
                     raise DatabaseError("Malformed request") from None
 
                 if type_ == "protocol":
                     response = list(self.PROTOCOL)
+                elif type_ == "bucket_probe":
+                    response = await self._get(type_, None, rq)
                 else:
                     try:
                         checksum = parse_checksum(checksum, as_bytes=False)
@@ -445,21 +551,27 @@ class DatabaseServer:
                     type_ = rq["type"]
                     if type_ not in types:
                         raise KeyError
-                    checksum = rq["checksum"]
+                    if type_ != "bucket_probe":
+                        checksum = rq["checksum"]
+                    else:
+                        checksum = None
                 except KeyError:
                     # import traceback; traceback.print_exc()
                     raise DatabaseError("Malformed request") from None
 
-                try:
-                    checksum = parse_checksum(checksum, as_bytes=False)
-                except ValueError:
-                    # import traceback; traceback.print_exc()
-                    raise DatabaseError("Malformed request") from None
+                if type_ != "bucket_probe":
+                    try:
+                        checksum = parse_checksum(checksum, as_bytes=False)
+                    except ValueError:
+                        # import traceback; traceback.print_exc()
+                        raise DatabaseError("Malformed request") from None
 
                 response = await self._put(type_, checksum, rq)
             except DatabaseError as exc:
                 status = 400
                 response = "ERROR: " + exc.args[0]
+            if isinstance(response, web.Response):
+                return response
             status2, response = format_response(response)
             if status == 200 and status2 is not None:
                 status = status2
@@ -524,13 +636,59 @@ class DatabaseServer:
             except DoesNotExist:
                 return None  # None is also a valid response
 
+        elif type_ == "bucket_probe":
+            try:
+                bucket_kind = request["bucket_kind"]
+                label = request["label"]
+            except KeyError:
+                raise DatabaseError("Malformed bucket_probe request") from None
+            try:
+                row = BucketProbe.get(
+                    BucketProbe.bucket_kind == bucket_kind,
+                    BucketProbe.label == label,
+                )
+            except DoesNotExist:
+                return None
+            return {
+                "bucket_kind": row.bucket_kind,
+                "label": row.label,
+                "bucket_checksum": row.bucket_checksum,
+                "captured_at": row.captured_at,
+                "freshness_tokens": row.freshness_tokens,
+            }
+
+        elif type_ == "irreproducible":
+            result_checksum = request.get("result")
+            if result_checksum is not None:
+                try:
+                    result_checksum = parse_checksum(result_checksum, as_bytes=False)
+                except ValueError:
+                    raise DatabaseError("Malformed irreproducible request") from None
+            rows = (
+                IrreproducibleTransformation.select()
+                .where(IrreproducibleTransformation.checksum == checksum)
+                .execute()
+            )
+            result = []
+            for row in rows:
+                if result_checksum is not None and row.result != result_checksum:
+                    continue
+                result.append(
+                    {
+                        "checksum": row.checksum,
+                        "result": row.result,
+                        "metadata": row.metadata,
+                    }
+                )
+            if not result:
+                return None
+            return result
+
         elif type_ == "expression":
             try:
                 celltype = request["celltype"]
                 path = json.dumps(request["path"])
-                hash_pattern = json.dumps(request.get("hash_pattern", ""))
                 target_celltype = request["target_celltype"]
-                target_hash_pattern = json.dumps(request.get("target_hash_pattern", ""))
             except KeyError:
                 raise DatabaseError("Malformed expression request")
             result = (
@@ -539,9 +697,7 @@ class DatabaseServer:
                     Expression.input_checksum == checksum,
                     Expression.path == path,
                     Expression.celltype == celltype,
-                    Expression.hash_pattern == hash_pattern,
                     Expression.target_celltype == target_celltype,
-                    Expression.target_hash_pattern == target_hash_pattern,
                 )
                 .execute()
             )
@@ -565,9 +721,7 @@ class DatabaseServer:
                     "checksum": expression.input_checksum,
                     "path": json.loads(expression.path),
                     "celltype": expression.celltype,
-                    "hash_pattern": json.loads(expression.hash_pattern),
                     "target_celltype": expression.target_celltype,
-                    "target_hash_pattern": json.loads(expression.target_hash_pattern),
                     "result": checksum,
                 }
                 result.append(expr)
@@ -645,9 +799,7 @@ class DatabaseServer:
                 value = parse_checksum(request["value"], as_bytes=False)
                 celltype = request["celltype"]
                 path = json.dumps(request["path"])
-                hash_pattern = json.dumps(request.get("hash_pattern", ""))
                 target_celltype = request["target_celltype"]
-                target_hash_pattern = json.dumps(request.get("target_hash_pattern", ""))
             except KeyError:
                 raise DatabaseError("Malformed expression request")
             try:
@@ -656,9 +808,7 @@ class DatabaseServer:
                 if len(request["path"]):
                     assert celltype in ("mixed", "plain", "binary")
                 assert len(celltype) <= 20
-                assert len(hash_pattern) <= 20
                 assert len(target_celltype) <= 20
-                assert len(target_hash_pattern) <= 20
             except AssertionError:
                 raise DatabaseError(
                     "Malformed expression request (constraint violation)"
@@ -667,51 +817,124 @@ class DatabaseServer:
                 input_checksum=checksum,
                 path=path,
                 celltype=celltype,
-                hash_pattern=hash_pattern,
                 target_celltype=target_celltype,
-                target_hash_pattern=target_hash_pattern,
                 result=value,
             )
 
         elif type_ == "metadata":
             try:
-                value = request["value"]
-                value = json.loads(value)
-            except (KeyError, ValueError):
+                value = _normalize_metadata_value(request["value"])
+                result = parse_checksum(request["result"], as_bytes=False)
+                _validate_execution_record_identity(checksum, result, value)
+            except (KeyError, ValueError, TypeError):
                 raise DatabaseError("Malformed PUT metadata request") from None
-            MetaData.create(checksum=checksum, metadata=value)
+            with db_atomic():
+                if _irreproducible_exists(checksum):
+                    return _conflict_response(
+                        "Transformation already has irreproducible records"
+                    )
+                tf_row = _get_transformation_row(checksum)
+                if tf_row is not None:
+                    tf_result = parse_checksum(tf_row.result, as_bytes=False)
+                    if tf_result != result:
+                        return _conflict_response(
+                            "Transformation result does not match execution record result"
+                        )
+                metadata_row = _get_metadata_row(checksum)
+                if metadata_row is not None:
+                    metadata_result = parse_checksum(metadata_row.result, as_bytes=False)
+                    if metadata_result != result:
+                        return _conflict_response(
+                            "Execution record result does not match stored metadata result"
+                        )
+                    if metadata_row.metadata != value:
+                        return _conflict_response(
+                            "Execution record already exists with different metadata"
+                        )
+                    if tf_row is None:
+                        Transformation.create(checksum=checksum, result=result)
+                    _ensure_rev_transformation_row(checksum, result)
+                    return "OK"
+
+                if tf_row is None:
+                    Transformation.create(checksum=checksum, result=result)
+                    _ensure_rev_transformation_row(checksum, result)
+                else:
+                    _ensure_rev_transformation_row(checksum, result)
+                MetaData.create(checksum=checksum, result=result, metadata=value)
+
+        elif type_ == "bucket_probe":
+            try:
+                bucket_kind = request["bucket_kind"]
+                label = request["label"]
+                bucket_checksum = parse_checksum(
+                    request["bucket_checksum"], as_bytes=False
+                )
+                freshness_tokens = _normalize_freshness_tokens(
+                    request["freshness_tokens"]
+                )
+                captured_at = request["captured_at"]
+                _validate_bucket_probe_request(
+                    bucket_kind,
+                    label,
+                    bucket_checksum,
+                    freshness_tokens,
+                    captured_at,
+                )
+            except (KeyError, ValueError, TypeError):
+                raise DatabaseError("Malformed PUT bucket_probe request") from None
+            with db_atomic():
+                query = BucketProbe.select().where(
+                    BucketProbe.bucket_kind == bucket_kind,
+                    BucketProbe.label == label,
+                )
+                if query.exists():
+                    row = query.get()
+                    row.bucket_checksum = bucket_checksum
+                    row.captured_at = captured_at
+                    row.freshness_tokens = freshness_tokens
+                    row.save()
+                else:
+                    BucketProbe.create(
+                        bucket_kind=bucket_kind,
+                        label=label,
+                        bucket_checksum=bucket_checksum,
+                        captured_at=captured_at,
+                        freshness_tokens=freshness_tokens,
+                    )
 
         elif type_ == "irreproducible":
             try:
                 result = parse_checksum(request["result"], as_bytes=False)
             except (KeyError, ValueError):
                 raise DatabaseError("Malformed 'irreproducible' request") from None
-            in_transformations = False
-            try:
-                tf = Transformation[checksum]
-                tf_result = parse_checksum(tf.result, as_bytes=False)
-                in_transformations = True
-            except DoesNotExist:
-                pass
-            if in_transformations:
-                if tf_result != result:
-                    return web.Response(
-                        status=404,
-                        reason="Transformation does not have the irreproducible result",
-                    )
-            try:
-                metadata = MetaData[checksum].metadata
-                in_metadata = True
-            except DoesNotExist:
-                metadata = ""
-                in_metadata = False
-            IrreproducibleTransformation.create(
-                checksum=checksum, result=result, metadata=metadata
-            )
-            if in_transformations:
-                tf.delete_instance()
-            if in_metadata:
-                MetaData[checksum].delete_instance()
+            with db_atomic():
+                tf_row = _get_transformation_row(checksum)
+                if tf_row is not None:
+                    tf_result = parse_checksum(tf_row.result, as_bytes=False)
+                    if tf_result != result:
+                        return web.Response(
+                            status=404,
+                            text="ERROR: Transformation does not have the irreproducible result",
+                        )
+                metadata_row = _get_metadata_row(checksum)
+                if metadata_row is not None:
+                    metadata_result = parse_checksum(metadata_row.result, as_bytes=False)
+                    if metadata_result != result:
+                        return _conflict_response(
+                            "Stored metadata result does not match irreproducible result"
+                        )
+                    metadata = metadata_row.metadata
+                else:
+                    metadata = ""
+                IrreproducibleTransformation.create(
+                    checksum=checksum, result=result, metadata=metadata
+                )
+                if tf_row is not None:
+                    tf_row.delete_instance()
+                    _delete_rev_transformation_rows(checksum, result)
+                if metadata_row is not None:
+                    metadata_row.delete_instance()
         else:
             raise DatabaseError("Unknown request type")
         return "OK"
@@ -775,21 +998,28 @@ If it doesn't exist, a new file is created.""",
             create_tables=False,
         )
 
-    selected_port = args.port if args.port is not None else 5522
-    status_file_path = args.status_file
-    status_tracker = None
-    if status_file_path:
-        status_file_contents = wait_for_status_file(status_file_path)
-        status_tracker = StatusFileTracker(
-            status_file_path, status_file_contents, args.port
-        )
-
     if args.port_range:
         start, end = args.port_range
         try:
             selected_port = pick_random_free_port(args.host, start, end)
         except BaseException as exc:
             raise_startup_error(exc)
+    elif args.port is not None:
+        selected_port = args.port
+    else:
+        try:
+            selected_port = pick_random_free_port(
+                args.host, DEFAULT_RANDOM_PORT_START, DEFAULT_RANDOM_PORT_END
+            )
+        except BaseException as exc:
+            raise_startup_error(exc)
+    status_file_path = args.status_file
+    status_tracker = None
+    if status_file_path:
+        status_file_contents = wait_for_status_file(status_file_path)
+        status_tracker = StatusFileTracker(
+            status_file_path, status_file_contents, selected_port
+        )
     if status_tracker:
         status_tracker.port = selected_port
 
@@ -831,3 +1061,7 @@ If it doesn't exist, a new file is created.""",
         raise
     finally:
         loop.run_until_complete(database_server.stop())
+BUCKET_KINDS = frozenset(
+    ("node", "environment", "node_env", "queue", "queue_node")
+)
+

{seamless_database-2.0.1 → seamless_database-2.1.0}/database_models.py

@@ -7,6 +7,7 @@ from peewee import (
     CompositeKey,
     IntegrityError,
 )
+import sqlite3
 from playhouse.sqlite_ext import JSONField
 
 
@@ -134,9 +135,23 @@ class MetaData(BaseModel):
     # - execution time (also if failed)
     # - last recorded progress (if failed)
     checksum = ChecksumField(primary_key=True)
+    result = ChecksumField(index=True, unique=False)
     metadata = JSONField()
 
 
+class BucketProbe(BaseModel):
+    bucket_kind = CharField(max_length=20)
+    label = TextField()
+    bucket_checksum = ChecksumField(index=True, unique=False)
+    captured_at = TextField()
+    freshness_tokens = JSONField()
+
+    class Meta:
+        database = _db
+        legacy_table_names = False
+        primary_key = CompositeKey("bucket_kind", "label")
+
+
 class IrreproducibleTransformation(BaseModel):
     result = ChecksumField(index=True, unique=False)
     checksum = ChecksumField(index=True, unique=False)
@@ -150,6 +165,7 @@ _model_classes = [
     SyntacticToSemantic,
     Expression,
     MetaData,
+    BucketProbe,
     IrreproducibleTransformation,
 ]
 _primary = {}
@@ -158,6 +174,8 @@ for model_class in _model_classes:
         model_class is Expression
         or model_class is SyntacticToSemantic
         or model_class is RevTransformation
+        or model_class is MetaData
+        or model_class is BucketProbe
     ):
         continue
     for fieldname, field in model_class._meta.fields.items():
@@ -175,6 +193,8 @@ def db_init(
     *,
     create_tables: bool = True,
 ):
+    if not _db.is_closed():
+        _db.close()
     if init_parameters is None:
         init_parameters = {}
     if connection_parameters is None:
@@ -182,7 +202,47 @@ def db_init(
     _db.init(filename, **init_parameters)
     _db.connect(**connection_parameters)
     if create_tables:
+        _ensure_meta_data_schema()
         _db.create_tables(_model_classes, safe=True)
 
 
 db_atomic = _db.atomic
+
+
+def _table_columns(table_name: str) -> list[str]:
+    cursor = _db.execute_sql(f"PRAGMA table_info({table_name})")
+    rows = cursor.fetchall()
+    return [row[1] for row in rows]
+
+
+def _table_rowcount(table_name: str) -> int:
+    cursor = _db.execute_sql(f"SELECT COUNT(*) FROM {table_name}")
+    row = cursor.fetchone()
+    assert row is not None
+    return int(row[0])
+
+
+def _ensure_meta_data_schema() -> None:
+    table_name = MetaData._meta.table_name
+    try:
+        tables = set(_db.get_tables())
+    except sqlite3.OperationalError:
+        tables = set()
+    if table_name not in tables:
+        return
+
+    columns = _table_columns(table_name)
+    column_set = set(columns)
+    if len(columns) == 3 and column_set == {"checksum", "result", "metadata"}:
+        return
+    if len(columns) == 2 and column_set == {"checksum", "metadata"}:
+        if _table_rowcount(table_name) == 0:
+            _db.execute_sql(f"DROP TABLE {table_name}")
+            return
+        raise RuntimeError(
+            "Existing legacy 'meta_data' table is non-empty and cannot be upgraded automatically"
+        )
+
+    raise RuntimeError(
+        f"Unsupported schema for '{table_name}': columns={columns!r}"
+    )

{seamless_database-2.0.1 → seamless_database-2.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "seamless-database"
-version = "2.0.1"
+version = "2.1.0"
 description = "SQLite-backed metadata database service for Seamless"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -21,7 +21,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.12",
     "Operating System :: OS Independent",
     "Topic :: Scientific/Engineering",
     "Topic :: Software Development :: Libraries",

{seamless_database-2.0.1 → seamless_database-2.1.0}/seamless_database.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: seamless-database
-Version: 2.0.1
+Version: 2.1.0
 Summary: SQLite-backed metadata database service for Seamless
 Author: Sjoerd de Vries
 License-Expression: MIT
@@ -13,7 +13,6 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries
@@ -70,6 +69,27 @@ seamless-database seamless.db --port-range 5520 5530 --writable
 seamless-database seamless.db --port 5522
 ```
 
+If `--port` and `--port-range` are both omitted, `seamless-database` picks a random free port in the dynamic/private range (`49152-65535`).
+
+### Status-file protocol
+
+`seamless-database` does not require a status file. If `--status-file` is omitted, it runs independently.
+
+If `--status-file` is provided, the file is used for two things:
+
+1. Report the chosen port, especially when `--port-range` is used.
+2. Report whether startup succeeded (`"running"`) or failed (`"failed"`).
+
+The status-file protocol is simple:
+
+1. Wait for the status file to exist and parse it as JSON.
+2. Reuse the existing JSON object as the base payload. An empty JSON object `{}` is sufficient.
+3. Choose or validate its listening port.
+4. Once the HTTP server is up, rewrite the same file with `"status": "running"` and the selected `"port"`.
+5. If startup fails before the server is running, rewrite the file with `"status": "failed"` instead.
+
+If `remote-http-launcher` is used, it may pre-populate the JSON with fields such as the PID, workdir, or `"status": "starting"`. `seamless-database` preserves such fields when it writes back the final status.
+
 ### CLI options
 
 | Option | Description |

{seamless_database-2.0.1 → seamless_database-2.1.0}/seamless_database.egg-info/SOURCES.txt

@@ -8,4 +8,6 @@ seamless_database.egg-info/SOURCES.txt
 seamless_database.egg-info/dependency_links.txt
 seamless_database.egg-info/entry_points.txt
 seamless_database.egg-info/requires.txt
-seamless_database.egg-info/top_level.txt
+seamless_database.egg-info/top_level.txt
+tests/test_execution_records.py
+tests/test_random_port.py

seamless_database-2.1.0/tests/test_execution_records.py

@@ -0,0 +1,313 @@
+import asyncio
+import sqlite3
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+DATABASE_DIR = ROOT / "seamless-database"
+if str(DATABASE_DIR) not in sys.path:
+    sys.path.insert(0, str(DATABASE_DIR))
+
+from database import DatabaseServer  # noqa: E402
+from database_models import (  # noqa: E402
+    BucketProbe,
+    MetaData,
+    RevTransformation,
+    Transformation,
+    IrreproducibleTransformation,
+    _db,
+    db_init,
+)
+
+
+TF_CHECKSUM = "1" * 64
+RESULT_CHECKSUM = "2" * 64
+BUCKET_CHECKSUM = "3" * 64
+BUCKET_CHECKSUM_2 = "4" * 64
+
+
+def _close_db():
+    if not _db.is_closed():
+        _db.close()
+
+
+def _init_db(path: Path):
+    _close_db()
+    db_init(str(path))
+
+
+def _record(
+    tf_checksum: str = TF_CHECKSUM,
+    result_checksum: str = RESULT_CHECKSUM,
+    schema_version=1,
+):
+    return {
+        "schema_version": schema_version,
+        "checksum_fields": ["node", "environment"],
+        "tf_checksum": tf_checksum,
+        "result_checksum": result_checksum,
+    }
+
+
+def _minimal_record(
+    tf_checksum: str = TF_CHECKSUM,
+    result_checksum: str = RESULT_CHECKSUM,
+):
+    return {
+        "schema_version": 1,
+        "tf_checksum": tf_checksum,
+        "result_checksum": result_checksum,
+        "seamless_version": "test",
+        "execution_mode": "process",
+        "remote_target": None,
+        "wall_time_seconds": 0.5,
+        "cpu_time_user_seconds": 0.2,
+        "cpu_time_system_seconds": 0.1,
+        "memory_peak_bytes": 12345,
+        "gpu_memory_peak_bytes": None,
+    }
+
+
+def test_db_init_recreates_empty_legacy_metadata_table(tmp_path):
+    dbfile = tmp_path / "legacy-empty.db"
+    con = sqlite3.connect(dbfile)
+    try:
+        con.execute("CREATE TABLE meta_data (checksum TEXT PRIMARY KEY, metadata TEXT)")
+        con.commit()
+    finally:
+        con.close()
+
+    _init_db(dbfile)
+    try:
+        con = sqlite3.connect(dbfile)
+        try:
+            columns = [
+                row[1]
+                for row in con.execute("PRAGMA table_info(meta_data)").fetchall()
+            ]
+        finally:
+            con.close()
+        assert columns == ["checksum", "result", "metadata"]
+    finally:
+        _close_db()
+
+
+def test_db_init_rejects_nonempty_legacy_metadata_table(tmp_path):
+    dbfile = tmp_path / "legacy-nonempty.db"
+    con = sqlite3.connect(dbfile)
+    try:
+        con.execute("CREATE TABLE meta_data (checksum TEXT PRIMARY KEY, metadata TEXT)")
+        con.execute(
+            "INSERT INTO meta_data(checksum, metadata) VALUES(?, ?)",
+            (TF_CHECKSUM, "{}"),
+        )
+        con.commit()
+    finally:
+        con.close()
+
+    with pytest.raises(RuntimeError, match="legacy 'meta_data' table is non-empty"):
+        _init_db(dbfile)
+    _close_db()
+
+
+def test_put_metadata_auto_creates_and_gets_record(tmp_path):
+    dbfile = tmp_path / "records.db"
+    _init_db(dbfile)
+    server = DatabaseServer("127.0.0.1", 0)
+    record = _record()
+    request = {
+        "type": "metadata",
+        "checksum": TF_CHECKSUM,
+        "result": RESULT_CHECKSUM,
+        "value": record,
+    }
+
+    try:
+        result = asyncio.run(server._put("metadata", TF_CHECKSUM, request))
+        assert result == "OK"
+        assert Transformation[TF_CHECKSUM].result == RESULT_CHECKSUM
+        assert MetaData[TF_CHECKSUM].result == RESULT_CHECKSUM
+        assert (
+            asyncio.run(
+                server._get(
+                    "metadata", TF_CHECKSUM, {"type": "metadata", "checksum": TF_CHECKSUM}
+                )
+            )
+            == record
+        )
+    finally:
+        _close_db()
+
+
+def test_put_metadata_accepts_minimal_execution_record(tmp_path):
+    dbfile = tmp_path / "minimal-records.db"
+    _init_db(dbfile)
+    server = DatabaseServer("127.0.0.1", 0)
+    record = _minimal_record()
+    request = {
+        "type": "metadata",
+        "checksum": TF_CHECKSUM,
+        "result": RESULT_CHECKSUM,
+        "value": record,
+    }
+
+    try:
+        result = asyncio.run(server._put("metadata", TF_CHECKSUM, request))
+        assert result == "OK"
+        assert MetaData[TF_CHECKSUM].metadata == record
+    finally:
+        _close_db()
+
+
+def test_put_metadata_is_idempotent_and_rejects_conflicts(tmp_path):
+    dbfile = tmp_path / "records-conflict.db"
+    _init_db(dbfile)
+    server = DatabaseServer("127.0.0.1", 0)
+    record = _record()
+    request = {
+        "type": "metadata",
+        "checksum": TF_CHECKSUM,
+        "result": RESULT_CHECKSUM,
+        "value": record,
+    }
+    bad_request = {
+        "type": "metadata",
+        "checksum": TF_CHECKSUM,
+        "result": RESULT_CHECKSUM,
+        "value": {
+            **record,
+            "checksum_fields": ["node"],
+        },
+    }
+
+    try:
+        assert asyncio.run(server._put("metadata", TF_CHECKSUM, request)) == "OK"
+        assert asyncio.run(server._put("metadata", TF_CHECKSUM, request)) == "OK"
+        response = asyncio.run(server._put("metadata", TF_CHECKSUM, bad_request))
+        assert response.status == 409
+    finally:
+        _close_db()
+
+
+def test_put_irreproducible_moves_metadata_and_deletes_normal_rows(tmp_path):
+    dbfile = tmp_path / "irreproducible.db"
+    _init_db(dbfile)
+    server = DatabaseServer("127.0.0.1", 0)
+    record = _record()
+
+    try:
+        asyncio.run(
+            server._put(
+                "metadata",
+                TF_CHECKSUM,
+                {
+                    "type": "metadata",
+                    "checksum": TF_CHECKSUM,
+                    "result": RESULT_CHECKSUM,
+                    "value": record,
+                },
+            )
+        )
+        assert RevTransformation.select().count() == 1
+        result = asyncio.run(
+            server._put(
+                "irreproducible",
+                TF_CHECKSUM,
+                {
+                    "type": "irreproducible",
+                    "checksum": TF_CHECKSUM,
+                    "result": RESULT_CHECKSUM,
+                },
+            )
+        )
+        assert result == "OK"
+        assert Transformation.select().count() == 0
+        assert MetaData.select().count() == 0
+        assert RevTransformation.select().count() == 0
+        rows = asyncio.run(
+            server._get(
+                "irreproducible",
+                TF_CHECKSUM,
+                {"type": "irreproducible", "checksum": TF_CHECKSUM},
+            )
+        )
+        assert rows == [
+            {
+                "checksum": TF_CHECKSUM,
+                "result": RESULT_CHECKSUM,
+                "metadata": record,
+            }
+        ]
+        response = asyncio.run(
+            server._put(
+                "metadata",
+                TF_CHECKSUM,
+                {
+                    "type": "metadata",
+                    "checksum": TF_CHECKSUM,
+                    "result": RESULT_CHECKSUM,
+                    "value": record,
+                },
+            )
+        )
+        assert response.status == 409
+        assert IrreproducibleTransformation.select().count() == 1
+    finally:
+        _close_db()
+
+
+def test_bucket_probe_roundtrip_and_overwrite(tmp_path):
+    dbfile = tmp_path / "bucket-probe.db"
+    _init_db(dbfile)
+    server = DatabaseServer("127.0.0.1", 0)
+    request = {
+        "type": "bucket_probe",
+        "bucket_kind": "environment",
+        "label": "conda:/envs/seamless1",
+        "bucket_checksum": BUCKET_CHECKSUM,
+        "captured_at": "2026-04-26T12:00:00Z",
+        "freshness_tokens": {"conda_meta_mtime": 123},
+    }
+    expected_probe = {k: v for k, v in request.items() if k != "type"}
+    updated_request = {
+        **request,
+        "bucket_checksum": BUCKET_CHECKSUM_2,
+        "captured_at": "2026-04-26T12:05:00Z",
+        "freshness_tokens": {"conda_meta_mtime": 456},
+    }
+
+    try:
+        assert asyncio.run(server._put("bucket_probe", None, request)) == "OK"
+        probe = asyncio.run(
+            server._get(
+                "bucket_probe",
+                None,
+                {
+                    "type": "bucket_probe",
+                    "bucket_kind": request["bucket_kind"],
+                    "label": request["label"],
+                },
+            )
+        )
+        assert probe == expected_probe
+        assert BucketProbe.select().count() == 1
+
+        assert asyncio.run(server._put("bucket_probe", None, updated_request)) == "OK"
+        probe = asyncio.run(
+            server._get(
+                "bucket_probe",
+                None,
+                {
+                    "type": "bucket_probe",
+                    "bucket_kind": request["bucket_kind"],
+                    "label": request["label"],
+                },
+            )
+        )
+        assert probe == {k: v for k, v in updated_request.items() if k != "type"}
+        assert BucketProbe.select().count() == 1
+    finally:
+        _close_db()

seamless_database-2.1.0/tests/test_random_port.py

@@ -0,0 +1,82 @@
+import json
+import os
+import signal
+import subprocess
+import time
+from pathlib import Path
+from urllib.error import URLError
+from urllib.request import urlopen
+
+
+DEFAULT_RANDOM_PORT_START = 49152
+DEFAULT_RANDOM_PORT_END = 65535
+
+
+def wait_for_status_file(status_file: Path, timeout: float = 10.0):
+    deadline = time.monotonic() + timeout
+    last_error = None
+    while time.monotonic() < deadline:
+        try:
+            payload = json.loads(status_file.read_text())
+        except (FileNotFoundError, json.JSONDecodeError) as exc:
+            last_error = exc
+            time.sleep(0.1)
+            continue
+        if payload.get("status") == "running" and isinstance(payload.get("port"), int):
+            return payload
+        last_error = payload
+        time.sleep(0.1)
+    raise RuntimeError(f"Database status file did not report running state: {last_error}")
+
+
+def wait_for_server(port: int, timeout: float = 10.0):
+    deadline = time.monotonic() + timeout
+    url = f"http://127.0.0.1:{port}/healthcheck"
+    last_error = None
+    while time.monotonic() < deadline:
+        try:
+            with urlopen(url, timeout=1.0) as response:  # noqa: S310 - local test server
+                if response.status == 200 and response.read().decode() == "OK":
+                    return
+                last_error = response.status
+        except URLError as exc:
+            last_error = exc
+        time.sleep(0.1)
+    raise RuntimeError(f"Database server at {url} did not become ready: {last_error}")
+
+
+def test_random_port_default(tmp_path):
+    database_file = tmp_path / "seamless.db"
+    status_file = tmp_path / "status.json"
+    status_file.write_text("{}\n")
+    command = [
+        "seamless-database",
+        str(database_file),
+        "--writable",
+        "--status-file",
+        str(status_file),
+    ]
+    process = subprocess.Popen(  # noqa: S603,S607 - controlled test command
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=os.environ.copy(),
+    )
+    try:
+        payload = wait_for_status_file(status_file)
+        port = payload["port"]
+        assert DEFAULT_RANDOM_PORT_START <= port <= DEFAULT_RANDOM_PORT_END, port
+        assert port != 5522
+        wait_for_server(port)
+    finally:
+        if process.poll() is None:
+            process.send_signal(signal.SIGINT)
+        try:
+            stdout, _ = process.communicate(timeout=5)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            stdout, _ = process.communicate()
+        if stdout:
+            print("Server logs:")
+            print(stdout, end="" if stdout.endswith("\n") else "\n")