smart-tests-cli 2.16.3__tar.gz → 2.16.4__tar.gz

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/.github/workflows/test.yml

@@ -65,6 +65,16 @@ jobs:
 
       - name: Lint with flake8
         run: uv run poe lint
+
+      - name: Check CLI reference docs are up to date
+        run: |
+          uv run tools/generate_reference.py smart_tests/docs/modules/references/pages/cli-reference.adoc
+          git diff --exit-code smart_tests/docs/modules/references/pages/cli-reference.adoc || {
+            echo "::error::cli-reference.adoc is stale. Run 'tools/generate_reference.py smart_tests/docs/modules/references/pages/cli-reference.adoc' and commit the updated doc."
+            exit 1
+          }
+        shell: bash
+
       - name: Pull request validation
         run: |
           # Install Smart Tests CLI from this repo's code as a global tool

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: smart-tests-cli
-Version: 2.16.3
+Version: 2.16.4
 Summary: Smart Tests CLI
 Author-email: CloudBees <support@cloudbees.com>
 License: Apache Software License v2

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/scripts/update_docs.sh

@@ -13,3 +13,7 @@ git clone --depth=1 "$REPO" "$tmpdir"
 rm -rf "$DOCS_DST"
 cp -r "$tmpdir/docs" "$DOCS_DST"
 echo "Docs updated at $DOCS_DST"
+
+CLI_REFERENCE="$DOCS_DST/modules/references/pages/cli-reference.adoc"
+"$(dirname "$0")/../tools/generate_reference.py" "$CLI_REFERENCE"
+echo "Regenerated CLI reference tables in $CLI_REFERENCE"

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/commands/record/__init__.py

@@ -3,6 +3,7 @@ from ...app import Application
 from .attachment import attachment
 from .build import build
 from .commit import commit
+from .coverage import coverage
 from .deployment import deployment
 from .session import session
 from .tests import tests
@@ -18,4 +19,5 @@ record.add_command(commit)
 record.add_command(tests)
 record.add_command(session)
 record.add_command(attachment)
+record.add_command(coverage)
 record.add_command(deployment)

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/commands/record/case_event.py

@@ -1,6 +1,6 @@
 import datetime
 import sys
-from typing import Any, Callable, Dict, Generator
+from typing import Any, Callable, Dict, Generator, List, Optional, Tuple
 
 import dateutil.parser
 from dateutil.tz import tzlocal
@@ -12,7 +12,13 @@ from ...testpath import FilePathNormalizer, TestPath
 
 POSSIBLE_RESULTS = (Failure, Error, Skipped)
 
-CaseEventType = Dict[str, str]
+# Surefire's rerun/flaky extension tags. junitparser's TestCase.ITER_TYPES (and thus
+# case.result/case.system_out/case.system_err) doesn't know about these, so they're
+# read directly off the raw element.
+RERUN_TAGS = ("rerunFailure", "rerunError")
+FLAKY_TAGS = ("flakyFailure", "flakyError")
+
+CaseEventType = Dict[str, Any]
 CaseEventGenerator = Generator[CaseEventType, None, None]
 
 # function that computes TestPath from a test case
@@ -83,17 +89,15 @@ class CaseEvent:
         suite: TestSuite,
         report_file: str,
         data_builder: DataBuilder
-    ) -> Dict:
-        "Builds a JSON representation of CaseEvent from JUnitPaser objects"
-
-        # TODO: reconsider the initial value of the status.
-        status = CaseEvent.TEST_PASSED
-        for r in case.result:
-            if any(isinstance(r, s) for s in (Failure, Error)):
-                status = CaseEvent.TEST_FAILED
-                break
-            elif isinstance(r, Skipped):
-                status = CaseEvent.TEST_SKIPPED
+    ) -> List[Dict]:
+        """
+        Builds JSON representations of CaseEvent(s) from JUnitParser objects.
+
+        Normally a <testcase> maps to a single event. But Surefire's rerun/flaky
+        reports pack multiple attempts into one <testcase> (see RERUN_TAGS/FLAKY_TAGS);
+        each attempt is expanded into its own FAILED event here, in addition to the
+        testcase's own final PASSED/FAILED/SKIPPED result.
+        """
 
         def path_canonicalizer(test_path: TestPath) -> TestPath:
             if sys.platform == "win32":
@@ -114,6 +118,12 @@ class CaseEvent:
 
             return ""
 
+        # TODO(Improvement): when <system-err> is present, it currently replaces the failure's own
+        # text/message below instead of being appended to it. A test can both fail AND
+        # print to stderr (e.g. a logged warning before the assertion), so this can hide
+        # the actual failure detail behind unrelated captured output. Consider
+        # concatenating (failure text/message first, then system-err) instead of
+        # returning early here.
         def stderr(case: TestCase) -> str:
             """
             case for:
@@ -142,15 +152,64 @@ class CaseEvent:
 
             return stderr
 
-        return CaseEvent.create(
-            test_path=path_canonicalizer(path_builder(case, suite, report_file)),
-            duration_secs=case.time,
-            status=status,
-            stdout=stdout(case),
-            stderr=stderr(case),
-            timestamp=suite.timestamp,
-            data=data_builder(case),
-        )
+        def final_result() -> Tuple[int, str, str, float]:
+            """
+            The testcase's own final result: unchanged from before this method supported reruns.
+            <testcase time> is defined by Surefire as the duration of THIS run (the first failing
+            run, or the last successful run), so this is the one attempt that keeps it.
+            """
+            # TODO: reconsider the initial value of the status.
+            status = CaseEvent.TEST_PASSED
+            for r in case.result:
+                if any(isinstance(r, s) for s in (Failure, Error)):
+                    status = CaseEvent.TEST_FAILED
+                    break
+                elif isinstance(r, Skipped):
+                    status = CaseEvent.TEST_SKIPPED
+            return status, stdout(case), stderr(case), case.time
+
+        def attempt_result(elem) -> Tuple[int, str, str, float]:
+            """
+            One <flakyFailure>/<flakyError>/<rerunFailure>/<rerunError> attempt: always FAILED.
+            Surefire doesn't record a separate duration for these, so duration is 0 (avoids
+            double-counting the testcase's total time against final_result()'s duration).
+            """
+            def child_text(tag: str) -> Optional[str]:
+                child = elem.find(tag)
+                return child.text if child is not None and child.text else None
+
+            attempt_stdout = child_text("system-out") or ""
+            attempt_stderr = child_text("system-err") or child_text("stackTrace") or elem.attrib.get("message") or ""
+            return CaseEvent.TEST_FAILED, attempt_stdout, attempt_stderr, 0.0
+
+        rerun_elems = [c for c in case._elem if c.tag in RERUN_TAGS]
+        flaky_elems = [c for c in case._elem if c.tag in FLAKY_TAGS]
+
+        if rerun_elems:
+            # failed on the first attempt and on every rerun
+            attempts = [final_result()] + [attempt_result(e) for e in rerun_elems]
+        elif flaky_elems:
+            # failed on earlier attempts, passed on the last one
+            attempts = [attempt_result(e) for e in flaky_elems] + [final_result()]
+        else:
+            # no reruns: unchanged, single-event behavior
+            attempts = [final_result()]
+
+        test_path = path_canonicalizer(path_builder(case, suite, report_file))
+        data = data_builder(case)
+
+        return [
+            CaseEvent.create(
+                test_path=test_path,
+                duration_secs=attempt_duration,
+                status=attempt_status,
+                stdout=attempt_stdout,
+                stderr=attempt_stderr,
+                timestamp=suite.timestamp,
+                data=data,
+            )
+            for attempt_status, attempt_stdout, attempt_stderr, attempt_duration in attempts
+        ]
 
     @classmethod
     def create(cls, test_path: TestPath, duration_secs: float, status,

smart_tests_cli-2.16.4/smart_tests/commands/record/coverage.py

@@ -0,0 +1,135 @@
+import glob
+import os
+from io import BytesIO
+from pathlib import Path
+from typing import Annotated, List, Optional, Tuple
+
+import click
+from tabulate import tabulate
+
+import smart_tests.args4p.typer as typer
+from smart_tests.utils.session import SessionId, get_session
+
+from ... import args4p
+from ...app import Application
+from ...utils.smart_tests_client import SmartTestsClient
+
+
+class CoverageStatus:
+    SUCCESS = "✓ Recorded successfully"
+    FAILED = "⚠ Failed to record"
+
+
+@args4p.command(help="Record coverage data (LCOV format) related to a test session. "
+                "Passing COVERAGE_FILES is optional for bazel; "
+                "files are auto discovered when the command is run inside project root. "
+                "Other test runners must pass their coverage report file(s) directly.")
+def coverage(
+    app: Application,
+    session: Annotated[SessionId, SessionId.as_option()],
+    coverage_files: Annotated[List[str], typer.Argument(
+        multiple=True,
+        required=False,
+        help="Coverage report files to upload (LCOV format)"
+    )] = [],
+):
+    client = SmartTestsClient(app=app)
+    sources: List[str] = []
+
+    try:
+        # Assert test session exists
+        get_session(session, client)
+
+        if coverage_files:
+            sources = list(coverage_files)
+            blocks = [_read(f) for f in sources]
+        else:
+            # No files given: auto-discover from the current directory
+            # Auto discovery is Bazel only for now.
+            sources, blocks = discover_bazel_coverage(Path("."))
+
+        # LCOV tracefiles are just a sequence of SF:...end_of_record blocks, so concatenating
+        # multiple files is itself a valid tracefile. Send them as one upload rather than one
+        # request per file. The server already unions repeated SF: blocks for the same file within
+        # one upload, so this is correct even when the inputs cover overlapping files (e.g. one
+        # report from a unit-test run, another from an integration-test run touching the same
+        # source file)
+        content = b"\n".join(blocks)
+        status, files_recorded, is_reupload = post_coverage(client, session, content)
+    except Exception as e:
+        client.print_exception_and_recover(e)
+        status, files_recorded, is_reupload = CoverageStatus.FAILED, None, False
+
+    display_summary(sources, status, files_recorded, is_reupload)
+
+
+def discover_bazel_coverage(workspace: "str | Path") -> Tuple[List[str], List[bytes]]:
+    """
+    Finds every per-target coverage.dat under <workspace>/bazel-testlogs
+
+    Bazel's own coverage.dat carries no target identity on its own, though TN (Test Name) is a standard LCOV field.
+    So we derive TN by file path (same approach used by `record tests bazel`
+    (bazel-testlogs/<pkg>/<target>/coverage.dat -> TN://<pkg>:<target>).
+    """
+    base = Path(workspace).joinpath('bazel-testlogs')
+    if not base.exists():
+        raise click.UsageError(
+            "No such directory: %s. "
+            "Pass coverage report file(s) directly as arguments instead." % str(base))
+
+    sources: List[str] = []
+    blocks: List[bytes] = []
+    for report_file in sorted(glob.iglob(os.path.join(str(base), '**', 'coverage.dat'), recursive=True)):
+        pkg_and_target = report_file[len(str(base)) + 1:-len("/coverage.dat")]
+        package, target = os.path.dirname(pkg_and_target), os.path.basename(pkg_and_target)
+        label = "//{}:{}".format(package.replace(os.sep, '/'), target)
+        sources.append(report_file)
+        blocks.append(_tag_with_test_name(_read(report_file), label))
+
+    if not blocks:
+        raise click.UsageError("No coverage.dat files found under %s" % str(base))
+
+    return sources, blocks
+
+
+def _tag_with_test_name(content: bytes, label: str) -> bytes:
+    tn_line = "TN:{}\n".format(label).encode('utf-8')
+    tagged = []
+    for line in content.splitlines(keepends=True):
+        if line.startswith(b"SF:"):
+            tagged.append(tn_line)
+        tagged.append(line)
+    return b"".join(tagged)
+
+
+def _read(path: str) -> bytes:
+    with open(path, mode='rb') as f:
+        return f.read()
+
+
+def post_coverage(client: SmartTestsClient, session: SessionId, content: bytes) -> Tuple[str, Optional[int], bool]:
+    try:
+        res = client.request(
+            "post", session.subpath("coverage"), compress=True, payload=BytesIO(content),
+            additional_headers={"Content-Type": "text/plain"})
+        res.raise_for_status()
+        summary = res.json()
+        return CoverageStatus.SUCCESS, summary.get("sourceFilesRecorded"), summary.get("isReupload", False)
+    except Exception as e:
+        click.echo("Failed to upload coverage report: {}".format(str(e)), err=True)
+        return CoverageStatus.FAILED, None, False
+
+
+def display_summary(sources: List[str], status: str, source_files: Optional[int], is_reupload: bool = False):
+    if sources:
+        click.echo(tabulate([[f] for f in sources], ["Coverage files found"], tablefmt="github"))
+        click.echo()
+    rows = [[len(sources), source_files if source_files is not None else "-", status]]
+    click.echo(tabulate(rows, ["Coverage Files", "Source Files", "Status"], tablefmt="github"))
+    if is_reupload:
+        click.echo(
+            click.style(
+                "⚠ This session already had coverage recorded. The previous upload was overwritten. "
+                "Refer to --help if you intended to record multiple files",
+                fg="yellow"),
+            err=True)

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/commands/record/tests.py

@@ -6,7 +6,7 @@ import xml.etree.ElementTree as ET
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from time import time_ns
-from typing import Annotated, Callable, Dict, Generator, List, Tuple, Union
+from typing import Annotated, Callable, Dict, Generator, Iterable, List, Tuple, Union
 
 import click
 from dateutil.parser import ParserError, parse
@@ -118,7 +118,8 @@ class RecordTests:
             try:
                 for suite in testsuites:
                     for case in suite:
-                        yield CaseEvent.from_case_and_suite(self.path_builder, case, suite, report, self.metadata_builder)
+                        yield from CaseEvent.from_case_and_suite(
+                            self.path_builder, case, suite, report, self.metadata_builder)
             except Exception as e:
                 warn_and_exit_if_fail_fast_mode(
                     "Warning: error parsing JUnitXml file {filename}: {error}".format(
@@ -388,23 +389,6 @@ class RecordTests:
             nonlocal is_observation
             is_observation = res.json().get("testSession", {}).get("isObservation", False)
 
-        def recorded_result() -> Tuple[int, int, int, float]:
-            test_count = 0
-            success_count = 0
-            fail_count = 0
-            duration = float(0)
-
-            for tc in testcases(self.reports):
-                test_count += 1
-                status = tc.get("status")
-                if status == 0:
-                    fail_count += 1
-                elif status == 1:
-                    success_count += 1
-                duration += float(tc.get("duration") or 0)  # sec
-
-            return test_count, success_count, fail_count, duration / 60   # sec to min
-
         try:
             start = time_ns()
             tc = testcases(self.reports)
@@ -478,7 +462,7 @@ class RecordTests:
                 return
 
         file_count = len(self.reports)
-        test_count, success_count, fail_count, duration = recorded_result()
+        test_count, success_count, fail_count, duration = summarize_case_events(testcases(self.reports))
 
         click.echo(
             f"Smart Tests recorded tests for build "
@@ -516,6 +500,36 @@ class RecordTests:
 tests = Group(name="tests", callback=RecordTests, help="Record test results")
 
 
+def summarize_case_events(events: Iterable[CaseEventType]) -> Tuple[int, int, int, float]:
+    """
+    Summarizes a stream of case events into (test_count, success_count, fail_count, duration_minutes),
+    for the "Tests found / Tests passed / Tests failed" table printed after `record tests` runs.
+
+    Grouped by testPath rather than counted per event: a flaky/rerun testcase produces multiple
+    events (see CaseEvent.from_case_and_suite), but it's still a single test. A test that ever
+    passed counts as passed (even if it also failed on earlier attempts); a test that only ever
+    failed counts as failed.
+    """
+    tests: Dict[Tuple, Dict[str, bool]] = {}
+    duration = float(0)
+
+    for tc in events:
+        key = tuple((p["type"], p["name"]) for p in tc.get("testPath", []))
+        entry = tests.setdefault(key, {"passed": False, "failed": False})
+        status = tc.get("status")
+        if status == 1:
+            entry["passed"] = True
+        elif status == 0:
+            entry["failed"] = True
+        duration += float(tc.get("duration") or 0)  # sec
+
+    test_count = len(tests)
+    success_count = sum(1 for e in tests.values() if e["passed"])
+    fail_count = sum(1 for e in tests.values() if e["failed"] and not e["passed"])
+
+    return test_count, success_count, fail_count, duration / 60   # sec to min
+
+
 # if we fail to determine the timestamp of the build, we err on the side of collecting more test reports
 # than no test reports, so we use the 'epoch' timestamp
 INVALID_TIMESTAMP = datetime.datetime.fromtimestamp(0)

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/docs/modules/ROOT/nav.adoc

@@ -3,7 +3,10 @@ include::ROOT:partial$abbr.adoc[]
 * xref:ROOT:index.adoc[Overview]
 * xref:send-data-to-smart-tests:send-data-to-smart-tests.adoc[Send data to {PRODUCT}]
 ** xref:send-data-to-smart-tests:set-up-smart-tests/getting-started.adoc[Set up smart tests]
-*** xref:send-data-to-smart-tests:set-up-smart-tests/migration-to-github-oidc-auth.adoc[Update tokenless authentication to use GitHub OIDC]
+*** xref:send-data-to-smart-tests:set-up-smart-tests/migration-from-cli-v1-to-v2.adoc[Migrate from CLI v1 to v2]
+*** xref:send-data-to-smart-tests:set-up-smart-tests/oidc-authentication.adoc[Authenticate without an API key (OIDC)]
+*** xref:send-data-to-smart-tests:set-up-smart-tests/oidc-pipeline-authentication.adoc[Authenticate a pipeline with OIDC]
+*** xref:send-data-to-smart-tests:set-up-smart-tests/migration-to-github-oidc-auth.adoc[Migrate from the legacy GitHub Actions OIDC flow]
 *** xref:send-data-to-smart-tests:set-up-smart-tests/use-the-cli-with-docker-image.adoc[Use the CLI with Docker image]
 ** xref:send-data-to-smart-tests:record-builds/record-builds.adoc[Record builds]
 *** xref:send-data-to-smart-tests:record-builds/choose-a-value-for-build-name.adoc[Choose a value for build name]
@@ -67,7 +70,6 @@ include::ROOT:partial$abbr.adoc[]
 ** xref:references:integrations/raw.adoc['raw' profile for custom test runners]
 
 * xref:references:supported-languages.adoc[Supported languages]
-* xref:references:supported-test-frameworks.adoc[Supported test frameworks]
 * xref:references:faq.adoc[FAQ]
 
 .Administration

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/docs/modules/administration/pages/data-privacy-and-protection.adoc

@@ -17,17 +17,17 @@ CloudBees Smart Tests is designed to make your test runs faster and more reliabl
 | Data collected | What exactly, and why is it needed?
 
 | *Contents of source code & test files*
-| To generate a vector embedding (see in more detail below), so Smart Tests can understand the semantic relationships between how the source code relates to your tests. 
+| To generate a vector embedding (see in more detail below), so Smart Tests can understand the semantic relationships between how the source code relates to your tests.
 
 This data is temporarily processed by Smart Tests to generate vector embeddings – and not persisted.
 
 | *Metadata about the code change under test*
-| Git commit SHAs, branch names, timestamps, Git author details and the list of files changed in each build. 
+| Git commit SHAs, branch names, timestamps, Git author details and the list of files changed in each build.
 
 These are used to associate a code change with the tests that should run for it.
 
 | *Metadata about the test cases that were run*
-| Test paths & names, pass/fail/skip outcomes, durations, and associations with test suites. 
+| Test paths & names, pass/fail/skip outcomes, durations, and associations with test suites.
 
 These are used to train the prediction model that ranks tests by relevance for each code change, and to power flaky-test detection and other test health & insights features.
 |===
@@ -40,27 +40,45 @@ Think of it this way, the embedding does not contain the original code any more
 
 Smart Tests uses embeddings to answer one question: when files change in a commit, which tests are most likely to exercise the behaviour that changed? Comparing embeddings makes that comparison tractable, accurate, and even language-agnostic – Smart Tests can reason about code relationships across languages and frameworks without parsing them.
 
-== What embeddings don't let us do
+What embeddings don't let us do:
 
 * *They do not let us read your code*. The transformation from source to embedding is mathematically lossy; the original content cannot be recovered from a stored vector in any practical sense.
 * *They do not let us execute your code*, extract secrets from it or inspect its behaviour in any way.
 * *They do not enable cross-tenant access*. Every Smart Tests tenant's data – including embeddings – is isolated using row-level security in our database.
-* *They are not used to train public AI models*. Source content sent for embedding is processed to produce the vector and nothing else. It does not flow into the training data for any CloudBees or third-party foundation model.
 
-== Data security and handling practices
+== Model training
 
-* *We use OpenAI to generate vector embeddings*. Source code is transmitted to OpenAI solely for this purpose and not stored with Smart Tests thereafter.
-* *In transit* – All data is transmitted over HTTPS using TLS 1.3 or higher, end-to-end between your CI environment and CloudBees infrastructure, and onward to OpenAI for embedding generation.
-* *During processing* – Source code is held only for the duration of embedding generation:
-** First-time ingestion (full repository scan): ~3–4 minutes for large enterprise repositories.
-** Subsequent commits (incremental updates): typically a few seconds.
-+
-After embedding generation completes, source code is permanently and irreversibly deleted. Only the embedding vectors are retained.
-* *Security reviews & Penetration testing* – CloudBees' production infrastructure is reviewed on a recurring basis as part of our standard security program. Smart Tests has already completed a security review, with future reviews scheduled accordingly.
+Your source code is not used to train any AI model – not a CloudBees model, not OpenAI's, and not any other third-party foundation model, for any customer. Source content sent for embedding is processed to produce the vector, and nothing else; it never becomes part of a model's training data.
+
+CloudBees does train its own prediction models – the ones that rank tests by relevance and power flaky-test detection. Those are trained on the metadata described above (commit and test-result data), not on source code content.
+
+== Data retention
+
+*CloudBees*: Source code is held only for the duration of embedding generation:
+
+* First-time ingestion (full repository scan): ~3-4 minutes for large enterprise repositories.
+* Subsequent commits (incremental updates): typically a few seconds.
+
+After embedding generation completes, source code is permanently and irreversibly deleted. Only the embedding vectors and the metadata described above are retained, for as long as the customer's account is active. Customers have the option to have their data deleted; CloudBees will delete data based on a customer request to do so.
 
-== Data storage and retention
+*OpenAI (subprocessor)*: Source code is transmitted to OpenAI's /v1/embeddings endpoint (model: text-embedding-3-small) solely to compute the embedding, and is not stored with Smart Tests thereafter. Per link:https://developers.openai.com/api/docs/guides/your-data#storage-requirements-and-retention-controls-per-endpoint[OpenAI's published data-control policy] for that endpoint:
 
-* *Encryption* – Personal information is encrypted both in transit and at rest.
-* *Tenant isolation* – CloudBees Smart Tests is a multi-tenant SaaS product, with each customer's data kept separate from one another.
+* Data sent to the OpenAI API is *not used to train or improve OpenAI's models*.
+* The endpoint retains *no application state* after the response is returned.
+* *Zero Data Retention (ZDR) is enabled* for this workload – CloudBees' OpenAI organization is excluded from OpenAI's default abuse-monitoring log retention for this endpoint.
+
+== Subprocessors
+
+OpenAI is the only subprocessor that receives source code, and only for the purpose described above. Standard Contractual Clauses covering the transfer are included within OpenAI's link:https://openai.com/policies/data-processing-addendum[Data Processing Agreement], which CloudBees has agreed to.
+
+All AI-related vendors that process personal data and are relevant to the services provided are listed within our link:https://www.cloudbees.com/company/trust-center[Trust Center].
+
+== Access & security
+
+* *Repository access* – Smart Tests does not require a standing connection to your source repository. The CLI runs inside your own CI pipeline and sends the code already checked out for that build – access is scoped to whatever that CI job already has, not a separate CloudBees integration into your SCM.
+* *In transit* – All data is transmitted over HTTPS using TLS 1.3 or higher, end-to-end between your CI environment and CloudBees infrastructure, and onward to OpenAI for embedding generation.
+* *At rest* – Data is encrypted at rest.
+* *Tenant isolation* – CloudBees Smart Tests is a multi-tenant SaaS product; each customer's data – including embeddings – is isolated using row-level security in the database.
 * *Hosting location* – CloudBees Smart Tests is hosted on AWS' US-West region.
-* *Retention and deletion* – Customers have the option to have their data deleted. CloudBees will delete data based on a customer request to do so.
+* *Certifications* – CloudBees Smart Tests is SOC 2 Type 1 compliant (as of August 2023). The underlying AWS infrastructure carries SOC 2 Type 2, ISO 27001, and ISO 27017.
+* *Security reviews & penetration testing* – CloudBees' production infrastructure is reviewed on a recurring basis as part of our standard security program. Smart Tests has already completed a security review, with future reviews scheduled accordingly.

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/docs/modules/concepts/pages/build.adoc

@@ -37,6 +37,6 @@ This command collects the details of the changes in each commit in your reposito
 By default, `smart-tests record build` runs `smart-tests record commit` , but these operations can be separated.
 --
 
-== More complex build/test pipelines
+== Build from multiple repositories
 
 However, in many other cases, the software being tested might be a single binary built from several repositories. Furthermore, the tested software might combine several services deployed to a single testing environment. The xref:send-data-to-smart-tests:record-builds/record-builds-from-multiple-repositories.adoc[record builds from multiple repositories] page outlines how to instrument your pipeline in these situations.

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/docs/modules/concepts/pages/subset.adoc

@@ -32,6 +32,7 @@ When you request a subset of tests to run in your CI process, you pass in the na
 `smart-tests subset --session $SESSION_NAME [other options...]`
 
 This is important so that the Predictive Test Selection service can analyze the changes in the build and select tests appropriately.
+Changes from all repositories in the build are considered, regardless of which repository the tests belong to.
 
 === Optimization target
 
@@ -177,3 +178,45 @@ In general, don't worry about creating the input test list. However, understandi
 --
 With xref:send-data-to-smart-tests:subset/zero-input-subsetting.adoc[Zero input subsetting], {PRODUCT} generates the input test list automatically.
 --
+
+== Fallback mode on Smart Tests failures
+
+When the subset API is unavailable or the model is untrained, the CLI falls back according to the `--fallback-mode` option. By default, the CLI runs all tests so that CI is never disrupted.
+
+Fallback is triggered by either of the following conditions:
+
+* The subset API call fails (network error, HTTP error, timeout, etc.).
+* When the {PRODUCT} model is untrained and the service returns `isBrainless: true`, this means it is still learning from your recorded test data and cannot yet produce an optimized subset.
+
+Use the `--fallback-mode` option to choose the fallback behavior:
+
+|===
+|Mode |Behavior |When to use
+
+|`run-all` (default)
+|Returns all tests unchanged. CI continues normally.
+|Safest option; no disruption to CI when the service is unavailable.
+
+|`stop`
+|Exits with a non-zero status. CI halts immediately.
+|When running an unoptimized test suite on failure is unacceptable.
+
+|`random-sample`
+|Picks a random local subset at the ratio specified by `--fallback-sampling-target`. No ML duration estimates are used.
+|When you want partial test coverage even if the service is down.
+|===
+
+When using `--fallback-mode=random-sample`, you must also specify `--fallback-sampling-target` as a percentage (for example, `50%`). This option is not valid with any other fallback mode.
+
+[source]
+----
+smart-tests subset \
+    --fallback-mode random-sample \
+    --fallback-sampling-target 50% \
+    [other options...]
+----
+
+[NOTE]
+--
+When the model is untrained and `--fallback-mode=random-sample` is set, the server-side result is used as-is. In this state the server also splits tests, so the local random sample is not applied.
+--

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/docs/modules/concepts/pages/test-session.adoc

@@ -43,7 +43,7 @@ In many cases, builds and test sessions map 1:1. For example, a developer pushes
 
 The CLI handles this default case without any extra steps. Just run `smart-tests record tests` at the end of your run to capture test results in a single session.
 
-. Test session definition 1
+.Test session definition 1
 image::ROOT:test-session-definition.png[Test session definition 1,role="screenshot"]
 
 === Run tests in different environments

smart_tests_cli-2.16.4/smart_tests/docs/modules/features/pages/test-notifications/github-app-for-test-sessions.adoc

@@ -0,0 +1,40 @@
+include::ROOT:partial$abbr.adoc[]
+= GitHub app for test sessions
+:slug: github-app-for-test-sessions
+
+The GitHub app for test sessions gives developers convenient, timely access to test results from their GitHub pull requests. Developers can start investigating failures immediately, without waiting for the entire CI process to finish. Test results are delivered as comments on pull requests.
+
+Access to associated workflows and test files via links provided in the {PRODUCT} web app is also available.
+
+[#configuration-and-features]
+== Configuration and features
+The following steps are required to enable the GitHub app for test sessions:
+
+. Install the link:https://github.com/apps/cloudbees-smart-tests/installations/new[{PRODUCT} GitHub app]. Your GitHub org admin might have to approve that process.
+
+. Verify that your test session is associated with a GitHub pull request.
+
+. Use the link:https://cloudbees.io/[{PRODUCT} webapp] to find a test session associated with a GitHub pull request.
+
+. Once the dropdown appears, the app is fully integrated. This association happens automatically when the {PRODUCT} CLI detects environment variables that are set by typical CI systems to indicate that the test is running for a GitHub pull request.
+
+. Select *View workflow run* to get to the workflow in GitHub Actions where this test session was triggered.
+
+. Select *View pull request* to go directly to the pull request page for this session.
++
+Additionally, once the sendGitHub app is integrated, you can jump directly to failing tests associated with files stored in sendGitHub repositories.
+
+[#accessing-failing-tests]
+== Accessing failing tests in sendGitHub
+
+To access failing tests, follow these steps:
+
+. Navigate to a test session page with a failure.
+
+. Navigate to the *Failed tests* column at the bottom of the page.
+
+. Select the ellipses for the desired failed test.
+
+. Select the *Open in sendGitHub* link. This allows you to open this test file directly in sendGitHub.
+
+image::ROOT:screenshot-2024-04-29-at-1-48-30-pm.png[]

{smart_tests_cli-2.16.3 → smart_tests_cli-2.16.4}/smart_tests/docs/modules/references/pages/cli-reference.adoc

@@ -503,7 +503,7 @@ The `--session` parameter is required and must use the session name that was reg
 
 // GENERATED. MODIFY IN CLI SOURCE CODE
 |`--fallback-mode` FALLBACKMODE
-|Behavior when the subset API is unavailable or the model is untrained. 'run-all' (default) runs all tests as usual; 'stop' exits with a non-zero status so CI halts; 'random-sample' picks a random subset locally based on the count derived from --fallback-sampling-target (no duration estimates are available in this path).
+|Behavior when the subset API is unavailable or the model is untrained. 'run-all' (default) runs all tests as usual; 'stop' exits with a non-zero status so CI halts; 'random-sample' picks a random subset locally based on thecount derived from --fallback-sampling-target (no duration estimates are available in this path). Exception: if the model is untrained (brainless mode), theserver has already split the tests, so 'random-sample' keeps that server-provided split as-is instead of sampling locally; the local random sample only applies when the subset API request itself fails.
 |No
 
 // GENERATED. MODIFY IN CLI SOURCE CODE