corgea-cli 1.14.0__tar.gz → 1.14.2__tar.gz

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/Cargo.lock

@@ -369,7 +369,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "corgea"
-version = "1.14.0"
+version = "1.14.2"
 dependencies = [
  "chrono",
  "clap",

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "corgea"
-version = "1.14.0"
+version = "1.14.2"
 edition = "2021"
 readme = "README.md"
 

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corgea-cli
-Version: 1.14.0
+Version: 1.14.2
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/README.md

@@ -48,20 +48,36 @@ evaluated; override with `CORGEA_BLOCKING_RULES_TIMEOUT_SECONDS`.
 trips: both are written before `--fail`/`--block-on` are evaluated, so a scan
 that exits 1 on a blocking rule still leaves its report behind to ingest.
 
-### Gateway errors are retried, not surfaced
-
-Every Corgea API call the CLI makes — uploads included — replays itself when the
-platform's proxy answers `502 Bad Gateway`, waiting 10s, then 30s, then 50s. A
-request that is still answered 502 after those three retries fails the command
-in the usual way, so a pipeline exits non-zero on a real outage and rides out the
-blips a busy platform produces under parallel scans. Each retry is logged, and
-the count belongs to a single request: any successful call starts the next one
-with the full three retries again.
-
-The one place this stops early is `corgea upload`'s per-file source upload: a 502
-that outlives the retries there is taken as the platform being unavailable rather
-than one bad file, so the remaining paths are reported as unsent instead of each
-spending another 90 seconds.
+### Rate limits and gateway errors are retried
+
+Neither `429 Too Many Requests` nor `502 Bad Gateway` is Corgea rejecting a
+request on its merits, so both are retried rather than failed: the CLI waits
+10s, then 30s, then 50s, and a pipeline rides out the blips a busy platform
+produces under parallel scans instead of failing on them. A request still
+answered the same way after those three retries fails the command in the usual
+way, so a real outage still exits non-zero. Each retry is logged, and the count
+belongs to a single request: any other answer starts the next one with the full
+three retries again. A `429` that names a `Retry-After` in seconds is honored,
+up to two minutes for any one pause, and never shortens the pause below the
+schedule.
+
+Which requests get retried depends on which of the two it is:
+
+- A `429` retries everything, `POST` and `PATCH` included. The rate limiter
+  declines the request before the API sees it, so nothing was created and
+  sending it again finishes the same work.
+- A `502` retries reads only. It comes from the proxy rather than from Corgea,
+  so it is equally the answer for "the request never arrived" and for "the
+  request was processed and the reply was lost coming back" — and every write
+  the CLI sends creates something, so re-sending one does not finish the first
+  scan, it starts a second. A write's 502 goes straight to the caller.
+
+Writes still retry network errors, where nothing reached Corgea at all.
+
+`corgea upload` treats either status on a source upload — a 502, or a rate limit
+that outlived its retries — as the platform being unavailable rather than one bad
+file, and reports the remaining paths as unsent instead of collecting the same
+answer once per path.
 
 ### Skipping a re-scan of the same commit
 

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/src/main.rs

@@ -16,6 +16,7 @@ mod scanners {
     pub mod blast;
     pub mod fortify;
     pub mod parsers;
+    pub mod report_paths;
 }
 mod utils {
     pub mod api;

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/src/scan.rs

@@ -1,8 +1,10 @@
 use crate::cicd::*;
 use crate::log::debug;
 use crate::scanners::parsers::ScanParserFactory;
+use crate::scanners::report_paths;
 use crate::{utils, Config};
 use reqwest::header;
+use reqwest::Method;
 use serde_json::Value;
 use std::collections::HashSet;
 use std::io::{self, Read};
@@ -57,14 +59,46 @@ fn find_corgea_policy_files(root: &Path) -> Vec<String> {
     found
 }
 
-fn merge_corgea_policy_files(mut paths: Vec<String>, root: &Path) -> Vec<String> {
+/// One file to send to `code-upload`.
+struct SourceUpload {
+    /// Sent as `path=`, exactly as the report wrote it. The engine matches the
+    /// report's paths against this, so it stays as written even when the file
+    /// itself was found somewhere else.
+    report_path: String,
+    /// Where the file actually is on this machine.
+    local_path: PathBuf,
+}
+
+/// Pair every path a report names with the file to read for it, and append the
+/// repo's `corgea.yaml` policy files.
+///
+/// The prefix search runs on the report's paths alone, before the policy files
+/// are added: those are found by walking `root`, so they always resolve, and
+/// one of them counting as a path that "already resolves" would call off the
+/// search for a report that matches nothing.
+fn plan_source_uploads(root: &Path, report_paths: &[String]) -> Vec<SourceUpload> {
+    let prefix = report_paths::find_report_path_prefix(root, report_paths);
+
+    let mut uploads: Vec<SourceUpload> = report_paths
+        .iter()
+        .map(|path| SourceUpload {
+            report_path: path.clone(),
+            local_path: report_paths::local_report_path(root, path, &prefix),
+        })
+        .collect();
+
     for yaml in find_corgea_policy_files(root) {
-        if !paths.iter().any(|path| path == &yaml) {
-            debug(&format!("Including repo policy file: {yaml}"));
-            paths.push(yaml);
+        if uploads.iter().any(|upload| upload.report_path == yaml) {
+            continue;
         }
+        debug(&format!("Including repo policy file: {yaml}"));
+        uploads.push(SourceUpload {
+            local_path: root.join(&yaml),
+            report_path: yaml,
+        });
     }
-    paths
+
+    uploads
 }
 
 pub fn run_command(base_cmd: &String, mut command: Command) -> String {
@@ -280,7 +314,7 @@ pub fn upload_scan(
     project_name: Option<String>,
 ) -> Option<ScanUploadResult> {
     let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
-    let paths = merge_corgea_policy_files(paths, &cwd);
+    let uploads = plan_source_uploads(&cwd, &paths);
     let in_ci = running_in_ci();
     let ci_platform = which_ci();
     let github_env_vars = get_github_env_vars();
@@ -328,13 +362,25 @@ pub fn upload_scan(
     let mut uploaded_paths = HashSet::new();
     let mut uploaded_count = 0;
     let mut upload_error_count = 0;
-    let mut gateway_gave_up = false;
-
-    'files: for path in &paths {
-        if !Path::new(&path).exists() {
+    let mut platform_declined = false;
+
+    'files: for upload in &uploads {
+        let path = &upload.report_path;
+        let fp = upload.local_path.as_path();
+
+        if !fp.exists() {
+            // Name where the file was looked for whenever that is not the path
+            // in the report, so a prefix that resolved most of the report but
+            // not this path is visible rather than mystifying.
+            let looked_in = if fp.as_os_str() == path.as_str() {
+                String::new()
+            } else {
+                format!(" (looked for it at '{}')", fp.display())
+            };
             log::error!(
-                "Required file {} not found which is required for the scan, exiting.",
-                path
+                "Required file {}{} not found which is required for the scan, exiting.",
+                path,
+                looked_in
             );
             std::process::exit(1);
         }
@@ -348,14 +394,13 @@ pub fn upload_scan(
             base_url, api_base, run_id, path
         );
         debug(&format!("Uploading file: {}", path));
-        let fp = Path::new(&path);
 
         let mut attempts = 0;
         let mut success = false;
 
         while attempts < 3 && !success {
             debug(&format!("POST: {}", src_upload_url));
-            let res = utils::api::send_with_retries("a source file upload", || {
+            let res = utils::api::send_with_retries("a source file upload", &Method::POST, || {
                 let form = reqwest::blocking::multipart::Form::new()
                     .file("file", fp)
                     .expect("Failed to read file");
@@ -373,18 +418,16 @@ pub fn upload_scan(
                             "Code upload failed with status: {}. Response body: {}",
                             status, body
                         ));
-                        // A 502 that outlived the retry schedule is the platform
-                        // being unavailable, not this one file. Retrying it here
-                        // would spend the schedule twice over, and walking the
-                        // remaining paths would spend a fresh 90 seconds on each
-                        // of them, so stop uploading source files altogether.
-                        if utils::api::is_gateway_error(status) {
-                            log::warn!(
-                                "Failed to upload file {} after the gateway retries: {}",
-                                path,
-                                status
-                            );
-                            gateway_gave_up = true;
+                        // A 502 or a rate limit that got this far is the
+                        // platform being unavailable, not something wrong with
+                        // this one file: the 502 because an upload is a write
+                        // and so is never replayed, the 429 because its retries
+                        // are already spent. Walking the remaining paths would
+                        // just collect the same answer once per file, so stop
+                        // uploading source files altogether.
+                        if utils::api::is_transient_error(status) {
+                            log::warn!("Failed to upload file {}: {}", path, status);
+                            platform_declined = true;
                             break 'files;
                         }
                         log::warn!("Failed to upload file {} {}... retrying", status, path);
@@ -419,12 +462,12 @@ pub fn upload_scan(
 
     // Everything the aborted walk never attempted still counts as unsent, or
     // the closing summary would report one failure for a whole skipped tree.
-    if gateway_gave_up {
-        let distinct: HashSet<&String> = paths.iter().collect();
+    if platform_declined {
+        let distinct: HashSet<&String> = uploads.iter().map(|upload| &upload.report_path).collect();
         let unsent = distinct.len() - uploaded_paths.len();
         upload_error_count += unsent;
         log::warn!(
-            "Stopped uploading source files: Corgea was still answering 502 after the retries. {} of {} files were not sent.",
+            "Stopped uploading source files: Corgea is not accepting them right now. {} of {} files were not sent.",
             unsent,
             distinct.len()
         );
@@ -468,15 +511,16 @@ pub fn upload_scan(
                 index + 1,
                 total_chunks
             ));
-            let response = utils::api::send_with_retries("a scan report chunk upload", || {
-                client
-                    .post(&scan_upload_url)
-                    .header(header::CONTENT_TYPE, "application/json")
-                    .header("Upload-Offset", offset.to_string())
-                    .header("Upload-Length", input_size.to_string())
-                    .body(chunk.to_vec())
-                    .send()
-            });
+            let response =
+                utils::api::send_with_retries("a scan report chunk upload", &Method::POST, || {
+                    client
+                        .post(&scan_upload_url)
+                        .header(header::CONTENT_TYPE, "application/json")
+                        .header("Upload-Offset", offset.to_string())
+                        .header("Upload-Length", input_size.to_string())
+                        .body(chunk.to_vec())
+                        .send()
+                });
 
             let should_break = match &response {
                 Ok(res) => {
@@ -519,7 +563,7 @@ pub fn upload_scan(
         last_response.expect("Failed to upload scan.")
     } else {
         debug(&format!("POST: {}", scan_upload_url));
-        utils::api::send_with_retries("the scan report upload", || {
+        utils::api::send_with_retries("the scan report upload", &Method::POST, || {
             client
                 .post(&scan_upload_url)
                 .header(header::CONTENT_TYPE, "application/json")
@@ -600,7 +644,7 @@ pub fn upload_scan(
     if git_config_path.exists() {
         debug("Uploading .git/config");
         debug(&format!("POST: {}", git_config_upload_url));
-        let res = utils::api::send_with_retries("the git config upload", || {
+        let res = utils::api::send_with_retries("the git config upload", &Method::POST, || {
             let form = reqwest::blocking::multipart::Form::new()
                 .file("file", git_config_path)
                 .expect("Failed to read file");
@@ -788,21 +832,109 @@ mod tests {
         );
     }
 
+    fn planned(root: &Path, report_paths: &[&str]) -> Vec<(String, String)> {
+        plan_source_uploads(
+            root,
+            &report_paths
+                .iter()
+                .map(|path| path.to_string())
+                .collect::<Vec<_>>(),
+        )
+        .into_iter()
+        .map(|upload| {
+            (
+                upload.report_path,
+                upload.local_path.to_string_lossy().into_owned(),
+            )
+        })
+        .collect()
+    }
+
+    #[test]
+    fn plan_source_uploads_appends_policy_files_and_skips_duplicates() {
+        let root = tempfile::tempdir().unwrap();
+        write_policy(&root.path().join("corgea.yaml"));
+        let policy = root
+            .path()
+            .join("corgea.yaml")
+            .to_string_lossy()
+            .into_owned();
+
+        assert_eq!(
+            planned(root.path(), &["src/source.py"]),
+            vec![
+                ("src/source.py".to_string(), "src/source.py".to_string()),
+                ("corgea.yaml".to_string(), policy.clone()),
+            ]
+        );
+        assert_eq!(
+            planned(root.path(), &["src/source.py", "corgea.yaml"]),
+            vec![
+                ("src/source.py".to_string(), "src/source.py".to_string()),
+                ("corgea.yaml".to_string(), "corgea.yaml".to_string()),
+            ]
+        );
+    }
+
+    /// Only the file read from disk is rebased. `path=` keeps the report's own
+    /// path, because that is what the engine matches the report against.
+    #[test]
+    fn plan_source_uploads_rebases_the_local_file_but_not_the_uploaded_path() {
+        let root = tempfile::tempdir().unwrap();
+        for path in ["src/a.py", "src/b.py"] {
+            std::fs::create_dir_all(root.path().join(path).parent().unwrap()).unwrap();
+            std::fs::write(root.path().join(path), "x = 1\n").unwrap();
+        }
+
+        assert_eq!(
+            planned(
+                root.path(),
+                &["/builds/acme/repo/src/a.py", "/builds/acme/repo/src/b.py"]
+            ),
+            vec![
+                (
+                    "/builds/acme/repo/src/a.py".to_string(),
+                    root.path().join("src/a.py").to_string_lossy().into_owned(),
+                ),
+                (
+                    "/builds/acme/repo/src/b.py".to_string(),
+                    root.path().join("src/b.py").to_string_lossy().into_owned(),
+                ),
+            ]
+        );
+    }
+
+    /// A policy file found by walking the root always exists, so letting one
+    /// into the prefix search would report the whole report as already
+    /// matching and call the search off.
     #[test]
-    fn merge_corgea_policy_files_appends_missing_and_skips_duplicates() {
+    fn plan_source_uploads_settles_the_prefix_before_adding_policy_files() {
         let root = tempfile::tempdir().unwrap();
         write_policy(&root.path().join("corgea.yaml"));
+        for path in ["src/a.py", "src/b.py"] {
+            std::fs::create_dir_all(root.path().join(path).parent().unwrap()).unwrap();
+            std::fs::write(root.path().join(path), "x = 1\n").unwrap();
+        }
+
+        let plan = planned(root.path(), &["ci/src/a.py", "ci/src/b.py"]);
 
         assert_eq!(
-            merge_corgea_policy_files(vec!["src/source.py".into()], root.path()),
-            vec!["src/source.py".to_string(), "corgea.yaml".to_string()]
+            plan[0],
+            (
+                "ci/src/a.py".to_string(),
+                root.path().join("src/a.py").to_string_lossy().into_owned(),
+            )
         );
         assert_eq!(
-            merge_corgea_policy_files(
-                vec!["src/source.py".into(), "corgea.yaml".into()],
+            plan[2],
+            (
+                "corgea.yaml".to_string(),
                 root.path()
+                    .join("corgea.yaml")
+                    .to_string_lossy()
+                    .into_owned(),
             ),
-            vec!["src/source.py".to_string(), "corgea.yaml".to_string()]
+            "the policy file is read from the root, never through the report's prefix"
         );
     }
 }

{corgea_cli-1.14.0 → corgea_cli-1.14.2}/src/scanners/fortify.rs

@@ -1,8 +1,9 @@
 use crate::scan::{upload_scan, ScanUploadResult};
 use crate::Config;
-use quick_xml::events::Event;
+use quick_xml::events::{BytesStart, Event};
 use quick_xml::reader::Reader;
 use quick_xml::XmlVersion;
+use std::collections::HashSet;
 use std::fs::File;
 use std::io;
 use std::io::{BufReader, Read};
@@ -68,6 +69,15 @@ pub fn parse(
     result
 }
 
+/// The raw FVDL and the source files the engine will look up for it.
+///
+/// A vulnerability can carry several `SourceLocation`s, where the earlier ones
+/// are the enclosing scope and only the last is the finding itself, and the
+/// engine reads that last one only. The earlier ones can sit in an unrelated
+/// tree -- Fortify records its own bundled libraries under
+/// `AppData/Local/Fortify/.../_fortify_libraries_/` -- which is not on the
+/// machine running the CLI, so collecting them aborted the upload of a report
+/// whose real files were all present.
 fn extract_file_path(scan_file: PathBuf) -> (String, Vec<String>) {
     let mut paths: Vec<String> = Vec::new();
 
@@ -84,6 +94,9 @@ fn extract_file_path(scan_file: PathBuf) -> (String, Vec<String>) {
 
     let mut buf = Vec::new();
     let mut in_vulnerability = false;
+    let mut seen = HashSet::new();
+    // The last SourceLocation seen in the current Vulnerability.
+    let mut selected: Option<String> = None;
 
     loop {
         match xml_reader.read_event_into(&mut buf) {
@@ -94,48 +107,15 @@ fn extract_file_path(scan_file: PathBuf) -> (String, Vec<String>) {
                 if tag_name == b"Vulnerability" {
                     in_vulnerability = true;
                 } else if tag_name == b"SourceLocation" && in_vulnerability {
-                    for attr_result in e.attributes() {
-                        match attr_result {
-                            Ok(attr) => {
-                                let attr_key = attr.key.as_ref();
-                                if attr_key == b"path" {
-                                    if let Ok(value) =
-                                        attr.normalized_value(XmlVersion::Implicit1_0)
-                                    {
-                                        let path_str = value.to_string();
-                                        if !paths.contains(&path_str) {
-                                            paths.push(path_str);
-                                        }
-                                    }
-                                }
-                            }
-                            Err(e) => println!("Error processing attribute: {}", e),
-                        }
+                    if let Some(path) = source_location_path(e) {
+                        selected = Some(path);
                     }
                 }
             }
             Ok(Event::Empty(ref e)) => {
-                let e_name = e.name();
-                let tag_name = e_name.as_ref();
-
-                if tag_name == b"SourceLocation" && in_vulnerability {
-                    for attr_result in e.attributes() {
-                        match attr_result {
-                            Ok(attr) => {
-                                let attr_key = attr.key.as_ref();
-                                if attr_key == b"path" {
-                                    if let Ok(value) =
-                                        attr.normalized_value(XmlVersion::Implicit1_0)
-                                    {
-                                        let path_str = value.to_string();
-                                        if !paths.contains(&path_str) {
-                                            paths.push(path_str);
-                                        }
-                                    }
-                                }
-                            }
-                            Err(e) => println!("Error processing attribute: {}", e),
-                        }
+                if e.name().as_ref() == b"SourceLocation" && in_vulnerability {
+                    if let Some(path) = source_location_path(e) {
+                        selected = Some(path);
                     }
                 }
             }
@@ -145,6 +125,12 @@ fn extract_file_path(scan_file: PathBuf) -> (String, Vec<String>) {
 
                 if tag_name == b"Vulnerability" {
                     in_vulnerability = false;
+
+                    if let Some(path) = selected.take() {
+                        if seen.insert(path.clone()) {
+                            paths.push(path);
+                        }
+                    }
                 }
             }
             Ok(Event::Eof) => break,
@@ -157,6 +143,23 @@ fn extract_file_path(scan_file: PathBuf) -> (String, Vec<String>) {
     (contents, paths)
 }
 
+/// The `path` attribute of a `SourceLocation`, with XML entities resolved.
+fn source_location_path(element: &BytesStart) -> Option<String> {
+    for attr_result in element.attributes() {
+        match attr_result {
+            Ok(attr) if attr.key.as_ref() == b"path" => {
+                if let Ok(value) = attr.normalized_value(XmlVersion::Implicit1_0) {
+                    return Some(value.to_string());
+                }
+            }
+            Ok(_) => {}
+            Err(e) => println!("Error processing attribute: {}", e),
+        }
+    }
+
+    None
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -176,6 +179,8 @@ mod tests {
   <Vulnerabilities>
     <Vulnerability>
       <SourceLocation path="src/start/a&amp;b.java"></SourceLocation>
+    </Vulnerability>
+    <Vulnerability>
       <SourceLocation path="src/empty/App.java" line="42"/>
     </Vulnerability>
     <Vulnerability>
@@ -204,4 +209,29 @@ mod tests {
              ignores the out-of-scope SourceLocation, and de-duplicates"
         );
     }
+
+    /// The engine reads the last SourceLocation of a vulnerability and no
+    /// other, so the earlier ones must not be uploaded: they can name Fortify's
+    /// own bundled libraries, which are not on the machine running the CLI and
+    /// used to abort the upload before any file was sent.
+    #[test]
+    fn extract_file_path_keeps_only_the_location_the_finding_is_reported_at() {
+        let fvdl = r#"<?xml version="1.0" encoding="UTF-8"?>
+<FVDL>
+  <Vulnerabilities>
+    <Vulnerability>
+      <SourceLocation path="C:/Users/dev/AppData/Local/Fortify/_fortify_libraries_/node/fs.d.ts" line="1"/>
+      <SourceLocation path="src/App.java" line="42"/>
+    </Vulnerability>
+  </Vulnerabilities>
+</FVDL>"#;
+
+        let mut tmp = tempfile::NamedTempFile::new().expect("create temp fvdl");
+        tmp.write_all(fvdl.as_bytes()).expect("write fvdl");
+        tmp.flush().expect("flush fvdl");
+
+        let (_, paths) = extract_file_path(tmp.path().to_path_buf());
+
+        assert_eq!(paths, vec!["src/App.java".to_string()]);
+    }
 }