folio-migration-tools 1.10.0__py3-none-any.whl → 1.10.0b1__py3-none-any.whl

folio_migration_tools/migration_tasks/batch_poster.py

@@ -4,15 +4,14 @@ import json
 import logging
 import re
 import sys
+import time
 import traceback
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Annotated, List, Optional
+from typing import Annotated, List, Optional
 from uuid import uuid4
 
-import folioclient
-
-if TYPE_CHECKING:
-    from httpx import Response
+import httpx
+import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
 from pydantic import Field
 
@@ -20,7 +19,6 @@ from folio_migration_tools.custom_exceptions import (
     TransformationProcessError,
     TransformationRecordFailedError,
 )
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.library_configuration import (
     FileDefinition,
     LibraryConfiguration,
@@ -75,7 +73,7 @@ class BatchPoster(MigrationTaskBase):
                 description=(
                     "The type of object being migrated"
                     "Examples of possible values: "
-                    "'Extradata', 'Instances', 'Holdings', 'Items'"
+                    "'Extradata', 'SRS', Instances', 'Holdings', 'Items'"
                 ),
             ),
         ]
@@ -247,80 +245,90 @@ class BatchPoster(MigrationTaskBase):
         self.failed_fields: set = set()
         self.num_failures = 0
         self.num_posted = 0
+        self.okapi_headers = self.folio_client.okapi_headers
+        self.http_client = None
         self.starting_record_count_in_folio: Optional[int] = None
         self.finished_record_count_in_folio: Optional[int] = None
 
     def do_work(self):  # noqa: C901
-        with open(
-            self.folder_structure.failed_recs_path, "w", encoding="utf-8"
-        ) as failed_recs_file:
-            self.get_starting_record_count()
-            try:
-                batch = []
-                for idx, file_def in enumerate(self.task_configuration.files):  # noqa: B007
-                    path = self.folder_structure.results_folder / file_def.file_name
-                    with open(path) as rows:
-                        logging.info("Running %s", path)
-                        last_row = ""
-                        for self.processed, row in enumerate(rows, start=1):
-                            last_row = row
-                            if row.strip():
-                                try:
-                                    if self.task_configuration.object_type == "Extradata":
-                                        self.post_extra_data(row, self.processed, failed_recs_file)
-                                    elif not self.api_info["is_batch"]:
-                                        self.post_single_records(
-                                            row, self.processed, failed_recs_file
+        with self.folio_client.get_folio_http_client() as httpx_client:
+            self.http_client = httpx_client
+            with open(
+                self.folder_structure.failed_recs_path, "w", encoding="utf-8"
+            ) as failed_recs_file:
+                self.get_starting_record_count()
+                try:
+                    batch = []
+                    if self.task_configuration.object_type == "SRS":
+                        self.create_snapshot()
+                    for idx, file_def in enumerate(self.task_configuration.files):  # noqa: B007
+                        path = self.folder_structure.results_folder / file_def.file_name
+                        with open(path) as rows:
+                            logging.info("Running %s", path)
+                            last_row = ""
+                            for self.processed, row in enumerate(rows, start=1):
+                                last_row = row
+                                if row.strip():
+                                    try:
+                                        if self.task_configuration.object_type == "Extradata":
+                                            self.post_extra_data(
+                                                row, self.processed, failed_recs_file
+                                            )
+                                        elif not self.api_info["is_batch"]:
+                                            self.post_single_records(
+                                                row, self.processed, failed_recs_file
+                                            )
+                                        else:
+                                            batch = self.post_record_batch(
+                                                batch, failed_recs_file, row
+                                            )
+                                    except UnicodeDecodeError as unicode_error:
+                                        self.handle_unicode_error(unicode_error, last_row)
+                                    except TransformationProcessError as tpe:
+                                        self.handle_generic_exception(
+                                            tpe,
+                                            last_row,
+                                            batch,
+                                            self.processed,
+                                            failed_recs_file,
                                         )
-                                    else:
-                                        batch = self.post_record_batch(
-                                            batch, failed_recs_file, row
+                                        batch = []
+                                        raise
+                                    except TransformationRecordFailedError as exception:
+                                        self.handle_generic_exception(
+                                            exception,
+                                            last_row,
+                                            batch,
+                                            self.processed,
+                                            failed_recs_file,
                                         )
-                                except UnicodeDecodeError as unicode_error:
-                                    self.handle_unicode_error(unicode_error, last_row)
-                                except TransformationProcessError as tpe:
-                                    self.handle_generic_exception(
-                                        tpe,
-                                        last_row,
-                                        batch,
-                                        self.processed,
-                                        failed_recs_file,
-                                    )
-                                    batch = []
-                                    raise
-                                except TransformationRecordFailedError as exception:
-                                    self.handle_generic_exception(
-                                        exception,
-                                        last_row,
-                                        batch,
-                                        self.processed,
-                                        failed_recs_file,
-                                    )
-                                    batch = []
-                                except (FileNotFoundError, PermissionError) as ose:
-                                    logging.error("Error reading file: %s", ose)
-
-            except Exception as ee:
-                if "idx" in locals() and self.task_configuration.files[idx:]:
-                    for file_def in self.task_configuration.files[idx:]:
-                        path = self.folder_structure.results_folder / file_def.file_name
+                                        batch = []
+                                    except (FileNotFoundError, PermissionError) as ose:
+                                        logging.error("Error reading file: %s", ose)
+
+                except Exception as ee:
+                    if "idx" in locals() and self.task_configuration.files[idx:]:
+                        for file_def in self.task_configuration.files[idx:]:
+                            path = self.folder_structure.results_folder / file_def.file_name
+                            try:
+                                with open(path, "r") as failed_file:
+                                    failed_file.seek(self.processed)
+                                    failed_recs_file.write(failed_file.read())
+                                    self.processed = 0
+                            except (FileNotFoundError, PermissionError) as ose:
+                                logging.error("Error reading file: %s", ose)
+                    raise ee
+                finally:
+                    if self.task_configuration.object_type != "Extradata" and any(batch):
                         try:
-                            with open(path, "r") as failed_file:
-                                failed_file.seek(self.processed)
-                                failed_recs_file.write(failed_file.read())
-                                self.processed = 0
-                        except (FileNotFoundError, PermissionError) as ose:
-                            logging.error("Error reading file: %s", ose)
-                raise ee
-            finally:
-                if self.task_configuration.object_type != "Extradata" and any(batch):
-                    try:
-                        self.post_batch(batch, failed_recs_file, self.processed)
-                    except Exception as exception:
-                        self.handle_generic_exception(
-                            exception, last_row, batch, self.processed, failed_recs_file
-                        )
-                logging.info("Done posting %s records. ", self.processed)
+                            self.post_batch(batch, failed_recs_file, self.processed)
+                        except Exception as exception:
+                            self.handle_generic_exception(
+                                exception, last_row, batch, self.processed, failed_recs_file
+                            )
+                    logging.info("Done posting %s records. ", self.processed)
+                    if self.task_configuration.object_type == "SRS":
+                        self.commit_snapshot()
 
     @staticmethod
     def set_consortium_source(json_rec):
@@ -358,26 +366,26 @@ class BatchPoster(MigrationTaskBase):
         fetch_batch_size = 90
         fetch_tasks = []
         existing_records = {}
-
-        for i in range(0, len(batch), fetch_batch_size):
-            batch_slice = batch[i : i + fetch_batch_size]
-            fetch_tasks.append(
-                self.get_with_retry(
-                    query_api,
-                    params={
-                        "query": (
-                            f"id==({' OR '.join([r['id'] for r in batch_slice if 'id' in r])})"
-                        ),
-                        "limit": fetch_batch_size,
-                    },
+        async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
+            for i in range(0, len(batch), fetch_batch_size):
+                batch_slice = batch[i : i + fetch_batch_size]
+                fetch_tasks.append(
+                    self.get_with_retry(
+                        client,
+                        query_api,
+                        params={
+                            "query": (
+                                f"id==({' OR '.join([r['id'] for r in batch_slice if 'id' in r])})"
+                            ),
+                            "limit": fetch_batch_size,
+                        },
+                    )
                 )
-            )
-
-        responses = await asyncio.gather(*fetch_tasks)
 
-        for response in responses:
-            self.collect_existing_records_for_upsert(object_type, response, existing_records)
+            responses = await asyncio.gather(*fetch_tasks)
 
+            for response in responses:
+                self.collect_existing_records_for_upsert(object_type, response, existing_records)
         for record in batch:
             if record["id"] in existing_records:
                 self.prepare_record_for_upsert(record, existing_records[record["id"]])
@@ -413,18 +421,18 @@ class BatchPoster(MigrationTaskBase):
 
     @staticmethod
     def collect_existing_records_for_upsert(
-        object_type: str, response_json: dict, existing_records: dict
+        object_type: str, response: httpx.Response, existing_records: dict
     ):
-        """
-        Collects existing records from API response into existing_records dict.
-
-        Args:
-            object_type: The key in response containing the records array
-            response_json: Parsed JSON response from API
-            existing_records: Dict to populate with {record_id: record_data}
-        """
-        for record in response_json.get(object_type, []):
-            existing_records[record["id"]] = record
+        if response.status_code == 200:
+            response_json = response.json()
+            for record in response_json[object_type]:
+                existing_records[record["id"]] = record
+        else:
+            logging.error(
+                "Failed to fetch current records. HTTP %s\t%s",
+                response.status_code,
+                response.text,
+            )
 
     def handle_upsert_for_statistical_codes(self, updates: dict, keep_existing: dict):
         if not self.task_configuration.preserve_statistical_codes:
@@ -502,66 +510,31 @@ class BatchPoster(MigrationTaskBase):
             updates.update(keep_new)
             new_record.update(updates)
 
-    async def get_with_retry(self, url: str, params=None):
-        """
-        Wrapper around folio_get_async with selective retry logic.
-
-        Retries on:
-        - Connection errors (FolioConnectionError): Always retry
-        - Server errors (5xx): Transient failures
-        - Rate limiting (429): Too many requests
-
-        Does NOT retry on:
-        - Client errors (4xx except 429): Bad request, won't succeed on retry
-        """
+    async def get_with_retry(self, client: httpx.AsyncClient, url: str, params=None):
         if params is None:
             params = {}
         retries = 3
-
         for attempt in range(retries):
             try:
-                return await self.folio_client.folio_get_async(url, query_params=params)
-
-            except folioclient.FolioConnectionError as e:
-                # Network/connection errors - always retry
+                response = await client.get(
+                    url, params=params, headers=self.folio_client.okapi_headers
+                )
+                response.raise_for_status()
+                return response
+            except httpx.HTTPError as e:
                 if attempt < retries - 1:
-                    wait_time = 2**attempt
-                    logging.warning(
-                        f"Connection error, retrying in {wait_time}s "
-                        f"(attempt {attempt + 1}/{retries}): {e}"
-                    )
-                    await asyncio.sleep(wait_time)
-                else:
-                    logging.error(f"Connection failed after {retries} attempts: {e}")
-                    raise
-
-            except folioclient.FolioHTTPError as e:
-                # HTTP errors - selective retry based on status code
-                status_code = e.response.status_code
-                should_retry = status_code >= 500 or status_code == 429
-
-                if should_retry and attempt < retries - 1:
-                    # Longer wait for rate limiting
-                    wait_time = 5 if status_code == 429 else 2**attempt
-                    logging.warning(
-                        f"HTTP {status_code} error, retrying in {wait_time}s "
-                        f"(attempt {attempt + 1}/{retries}): {e}"
-                    )
-                    await asyncio.sleep(wait_time)
+                    logging.warning(f"Retrying due to {e}")
+                    await asyncio.sleep(2**attempt)
                 else:
-                    # Either not retryable or out of attempts
-                    if should_retry:
-                        logging.error(
-                            f"HTTP {status_code} error persisted after {retries} attempts: {e}"
-                        )
-                    else:
-                        logging.error(f"HTTP {status_code} error (not retryable): {e}")
+                    logging.error(f"Failed to connect after {retries} attempts: {e}")
                     raise
 
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
         if self.task_configuration.object_type == "ShadowInstances":
             self.set_consortium_source(json_rec)
+        if self.task_configuration.object_type == "SRS":
+            json_rec["snapshotId"] = self.snapshot_id
         if self.processed == 1:
             logging.info(json.dumps(json_rec, indent=True))
         batch.append(json_rec)
@@ -572,29 +545,22 @@ class BatchPoster(MigrationTaskBase):
 
     def post_extra_data(self, row: str, num_records: int, failed_recs_file):
         (object_name, data) = row.split("\t")
-        url = self.get_extradata_endpoint(self.task_configuration, object_name, data)
+        endpoint = self.get_extradata_endpoint(self.task_configuration, object_name, data)
+        url = f"{self.folio_client.gateway_url}/{endpoint}"
         body = data
-        try:
-            _ = self.folio_client.folio_post(url, payload=body)
+        response = self.post_objects(url, body)
+        if response.status_code == 201:
             self.num_posted += 1
-        except folioclient.FolioHTTPError as fhe:
-            if fhe.response.status_code == 422:
-                self.num_failures += 1
-                error_msg = json.loads(fhe.response.text)["errors"][0]["message"]
-                logging.error(
-                    "Row %s\tHTTP %s\t %s", num_records, fhe.response.status_code, error_msg
-                )
-                if (
-                    "id value already exists"
-                    not in json.loads(fhe.response.text)["errors"][0]["message"]
-                ):
-                    failed_recs_file.write(row)
-            else:
-                self.num_failures += 1
-                logging.error(
-                    "Row %s\tHTTP %s\t%s", num_records, fhe.response.status_code, fhe.response.text
-                )
+        elif response.status_code == 422:
+            self.num_failures += 1
+            error_msg = json.loads(response.text)["errors"][0]["message"]
+            logging.error("Row %s\tHTTP %s\t %s", num_records, response.status_code, error_msg)
+            if "id value already exists" not in json.loads(response.text)["errors"][0]["message"]:
                 failed_recs_file.write(row)
+        else:
+            self.num_failures += 1
+            logging.error("Row %s\tHTTP %s\t%s", num_records, response.status_code, response.text)
+            failed_recs_file.write(row)
         if num_records % 50 == 0:
             logging.info(
                 "%s records posted successfully. %s failed",
@@ -634,41 +600,41 @@ class BatchPoster(MigrationTaskBase):
     def post_single_records(self, row: str, num_records: int, failed_recs_file):
         if self.api_info["is_batch"]:
             raise TypeError("This record type supports batch processing, use post_batch method")
-        url = self.api_info.get("api_endpoint")
-        try:
-            _ = self.folio_client.folio_post(url, payload=row)
+        api_endpoint = self.api_info.get("api_endpoint")
+        url = f"{self.folio_client.gateway_url}{api_endpoint}"
+        response = self.post_objects(url, row)
+        if response.status_code == 201:
             self.num_posted += 1
-        except folioclient.FolioHTTPError as fhe:
-            if fhe.response.status_code == 422:
-                self.num_failures += 1
-                error_msg = json.loads(fhe.response.text)["errors"][0]["message"]
-                logging.error(
-                    "Row %s\tHTTP %s\t %s", num_records, fhe.response.status_code, error_msg
-                )
-                if (
-                    "id value already exists"
-                    not in json.loads(fhe.response.text)["errors"][0]["message"]
-                ):
-                    failed_recs_file.write(row)
-            else:
-                self.num_failures += 1
-                logging.error(
-                    "Row %s\tHTTP %s\t%s",
-                    num_records,
-                    fhe.response.status_code,
-                    fhe.response.text,
-                )
+        elif response.status_code == 422:
+            self.num_failures += 1
+            error_msg = json.loads(response.text)["errors"][0]["message"]
+            logging.error("Row %s\tHTTP %s\t %s", num_records, response.status_code, error_msg)
+            if "id value already exists" not in json.loads(response.text)["errors"][0]["message"]:
                 failed_recs_file.write(row)
-            if num_records % 50 == 0:
-                logging.info(
-                    "%s records posted successfully. %s failed",
-                    self.num_posted,
-                    self.num_failures,
-                )
+        else:
+            self.num_failures += 1
+            logging.error("Row %s\tHTTP %s\t%s", num_records, response.status_code, response.text)
+            failed_recs_file.write(row)
+        if num_records % 50 == 0:
+            logging.info(
+                "%s records posted successfully. %s failed",
+                self.num_posted,
+                self.num_failures,
+            )
+
+    def post_objects(self, url, body):
+        if self.http_client and not self.http_client.is_closed:
+            return self.http_client.post(
+                url, data=body.encode("utf-8"), headers=self.folio_client.okapi_headers
+            )
+        else:
+            return httpx.post(
+                url, headers=self.okapi_headers, data=body.encode("utf-8"), timeout=None
+            )
 
     def handle_generic_exception(self, exception, last_row, batch, num_records, failed_recs_file):
         logging.error("%s", exception)
-        self.migration_report.add("Details", i18n_t("Generic exceptions (see log for details)"))
+        self.migration_report.add("Details", i18n.t("Generic exceptions (see log for details)"))
         # logging.error("Failed row: %s", last_row)
         self.failed_batches += 1
         self.num_failures += len(batch)
@@ -681,7 +647,7 @@ class BatchPoster(MigrationTaskBase):
             sys.exit(1)
 
     def handle_unicode_error(self, unicode_error, last_row):
-        self.migration_report.add("Details", i18n_t("Encoding errors"))
+        self.migration_report.add("Details", i18n.t("Encoding errors"))
         logging.info("=========ERROR==============")
         logging.info(
             "%s Posting failed. Encoding error reading file",
@@ -696,7 +662,7 @@ class BatchPoster(MigrationTaskBase):
         traceback.logging.info_exc()  # type: ignore
         logging.info("=======================")
 
-    def post_batch(self, batch, failed_recs_file, num_records):
+    def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):
         if self.query_params.get("upsert", False) and self.api_info.get("query_endpoint", ""):
             self.set_version(batch, self.api_info["query_endpoint"], self.api_info["object_name"])
         response = self.do_post(batch)
@@ -767,6 +733,24 @@ class BatchPoster(MigrationTaskBase):
             # Likely a json parsing error
             logging.error(response.text)
             raise TransformationProcessError("", "HTTP 400. Something is wrong. Quitting")
+        elif self.task_configuration.object_type == "SRS" and response.status_code >= 500:
+            logging.info(
+                "Post failed. Size: %s Waiting 30s until reposting. Number of tries: %s of 5",
+                get_req_size(response),
+                recursion_depth,
+            )
+            logging.info(response.text)
+            time.sleep(30)
+            if recursion_depth > 4:
+                raise TransformationRecordFailedError(
+                    "",
+                    f"HTTP {response.status_code}\t"
+                    f"Request size: {get_req_size(response)}"
+                    f"{datetime.now(timezone.utc).isoformat()}\n",
+                    response.text,
+                )
+            else:
+                self.post_batch(batch, failed_recs_file, num_records, recursion_depth + 1)
         elif (
             response.status_code == 413 and "DB_ALLOW_SUPPRESS_OPTIMISTIC_LOCKING" in response.text
         ):
@@ -791,36 +775,50 @@ class BatchPoster(MigrationTaskBase):
             )
 
     def do_post(self, batch):
-        with self.folio_client.get_folio_http_client() as http_client:
-            url = self.api_info["api_endpoint"]
-            if self.api_info["object_name"] == "users":
-                payload = {self.api_info["object_name"]: list(batch), "totalRecords": len(batch)}
-            elif self.api_info["total_records"]:
-                payload = {"records": list(batch), "totalRecords": len(batch)}
-            else:
-                payload = {self.api_info["object_name"]: batch}
-            return http_client.post(
+        path = self.api_info["api_endpoint"]
+        url = self.folio_client.gateway_url + path
+        if self.api_info["object_name"] == "users":
+            payload = {self.api_info["object_name"]: list(batch), "totalRecords": len(batch)}
+        elif self.api_info["total_records"]:
+            payload = {"records": list(batch), "totalRecords": len(batch)}
+        else:
+            payload = {self.api_info["object_name"]: batch}
+        if self.http_client and not self.http_client.is_closed:
+            return self.http_client.post(
+                url,
+                json=payload,
+                headers=self.folio_client.okapi_headers,
+                params=self.query_params,
+            )
+        else:
+            return httpx.post(
                 url,
+                headers=self.okapi_headers,
                 json=payload,
                 params=self.query_params,
+                timeout=None,
             )
 
     def get_current_record_count_in_folio(self):
         if "query_endpoint" in self.api_info:
-            url = self.api_info["query_endpoint"]
+            url = f"{self.folio_client.gateway_url}{self.api_info['query_endpoint']}"
             query_params = {"query": "cql.allRecords=1", "limit": 0}
-            try:
-                res = self.folio_client.folio_get(url, query_params=query_params)
-                return res["totalRecords"]
-            except folioclient.FolioHTTPError as fhe:
-                logging.error(
-                    "Failed to get current record count. HTTP %s", fhe.response.status_code
+            if self.http_client and not self.http_client.is_closed:
+                res = self.http_client.get(
+                    url, headers=self.folio_client.okapi_headers, params=query_params
                 )
+            else:
+                res = httpx.get(url, headers=self.okapi_headers, params=query_params, timeout=None)
+            try:
+                res.raise_for_status()
+                return res.json()["totalRecords"]
+            except httpx.HTTPStatusError:
+                logging.error("Failed to get current record count. HTTP %s", res.status_code)
                 return 0
             except KeyError:
                 logging.error(
                     "Failed to get current record count. "
-                    f"No 'totalRecords' in response: {json.dumps(res, indent=2)}"
+                    f"No 'totalRecords' in response: {res.json()}"
                 )
                 return 0
         else:
@@ -852,6 +850,8 @@ class BatchPoster(MigrationTaskBase):
     def wrap_up(self):
         logging.info("Done. Wrapping up")
         self.extradata_writer.flush()
+        if self.task_configuration.object_type == "SRS":
+            self.commit_snapshot()
         if self.task_configuration.object_type != "Extradata":
             logging.info(
                 (
@@ -938,6 +938,69 @@ class BatchPoster(MigrationTaskBase):
                 str(self.folder_structure.failed_recs_path),
             )
 
+    def create_snapshot(self):
+        snapshot = {
+            "jobExecutionId": self.snapshot_id,
+            "status": "PARSING_IN_PROGRESS",
+            "processingStartedDate": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
+        }
+        try:
+            url = f"{self.folio_client.gateway_url}/source-storage/snapshots"
+            if self.http_client and not self.http_client.is_closed:
+                res = self.http_client.post(
+                    url, json=snapshot, headers=self.folio_client.okapi_headers
+                )
+            else:
+                res = httpx.post(url, headers=self.okapi_headers, json=snapshot, timeout=None)
+            res.raise_for_status()
+            logging.info("Posted Snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
+            get_url = (
+                f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
+            )
+            got = False
+            while not got:
+                logging.info("Sleeping while waiting for the snapshot to get created")
+                time.sleep(5)
+                if self.http_client and not self.http_client.is_closed:
+                    res = self.http_client.get(get_url, headers=self.folio_client.okapi_headers)
+                else:
+                    res = httpx.get(get_url, headers=self.okapi_headers, timeout=None)
+                if res.status_code == 200:
+                    got = True
+                else:
+                    logging.info(res.status_code)
+        except httpx.HTTPStatusError as exc:
+            logging.exception("HTTP error occurred while posting the snapshot: %s", exc)
+            sys.exit(1)
+        except Exception as exc:
+            logging.exception("Could not post the snapshot: %s", exc)
+            sys.exit(1)
+
+    def commit_snapshot(self):
+        snapshot = {"jobExecutionId": self.snapshot_id, "status": "COMMITTED"}
+        try:
+            url = f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
+            if self.http_client and not self.http_client.is_closed:
+                res = self.http_client.put(
+                    url, json=snapshot, headers=self.folio_client.okapi_headers
+                )
+            else:
+                res = httpx.put(url, headers=self.okapi_headers, json=snapshot, timeout=None)
+            res.raise_for_status()
+            logging.info("Posted Committed snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
+        except httpx.HTTPStatusError as exc:
+            logging.exception("HTTP error occurred while posting the snapshot: %s", exc)
+            sys.exit(1)
+        except Exception as exc:
+            logging.exception(
+                "Could not commit snapshot with id %s. Post this to /source-storage/snapshots/%s:",
+                self.snapshot_id,
+                self.snapshot_id,
+                exc,
+            )
+            logging.info("%s", json.dumps(snapshot, indent=4))
+            sys.exit(1)
+
 
 def get_api_info(object_type: str, use_safe: bool = True):
     choices = {
@@ -999,6 +1062,22 @@ def get_api_info(object_type: str, use_safe: bool = True):
             "addSnapshotId": False,
             "supports_upsert": True,
         },
+        "Authorities": {
+            "object_name": "",
+            "api_endpoint": "/authority-storage/authorities",
+            "is_batch": False,
+            "total_records": False,
+            "addSnapshotId": False,
+            "supports_upsert": False,
+        },
+        "SRS": {
+            "object_name": "records",
+            "api_endpoint": "/source-storage/batch/records",
+            "is_batch": True,
+            "total_records": True,
+            "addSnapshotId": True,
+            "supports_upsert": False,
+        },
         "Users": {
             "object_name": "users",
             "api_endpoint": "/user-import",
@@ -1045,7 +1124,7 @@ def get_human_readable(size, precision=2):
     return "%.*f%s" % (precision, size, suffixes[suffix_index])
 
 
-def get_req_size(response: "Response"):
+def get_req_size(response: httpx.Response):
     size = response.request.method
     size += str(response.request.url)
     size += "\r\n".join(f"{k}{v}" for k, v in response.request.headers.items())