corgea-cli 1.13.0__tar.gz → 1.14.0__tar.gz

{corgea_cli-1.13.0 → corgea_cli-1.14.0}/Cargo.lock

@@ -369,7 +369,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "corgea"
-version = "1.13.0"
+version = "1.14.0"
 dependencies = [
  "chrono",
  "clap",

{corgea_cli-1.13.0 → corgea_cli-1.14.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "corgea"
-version = "1.13.0"
+version = "1.14.0"
 edition = "2021"
 readme = "README.md"
 

{corgea_cli-1.13.0 → corgea_cli-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corgea-cli
-Version: 1.13.0
+Version: 1.14.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{corgea_cli-1.13.0 → corgea_cli-1.14.0}/README.md

@@ -39,7 +39,8 @@ third-party report.
 
 All three exit 1 if the scan fails, printing the reason and the scanners that hit
 problems. A scan that completes with a scanner missing exits 0 with a warning.
-Waiting gives up after 10 hours; override with `CORGEA_SCAN_TIMEOUT_SECONDS`.
+While waiting, the scan's status is read every 3 seconds. Waiting gives up after
+10 hours; override with `CORGEA_SCAN_TIMEOUT_SECONDS`.
 `--fail`/`--block-on` then wait up to 15 minutes for blocking rules to be
 evaluated; override with `CORGEA_BLOCKING_RULES_TIMEOUT_SECONDS`.
 
@@ -47,6 +48,21 @@ evaluated; override with `CORGEA_BLOCKING_RULES_TIMEOUT_SECONDS`.
 trips: both are written before `--fail`/`--block-on` are evaluated, so a scan
 that exits 1 on a blocking rule still leaves its report behind to ingest.
 
+### Gateway errors are retried, not surfaced
+
+Every Corgea API call the CLI makes — uploads included — replays itself when the
+platform's proxy answers `502 Bad Gateway`, waiting 10s, then 30s, then 50s. A
+request that is still answered 502 after those three retries fails the command
+in the usual way, so a pipeline exits non-zero on a real outage and rides out the
+blips a busy platform produces under parallel scans. Each retry is logged, and
+the count belongs to a single request: any successful call starts the next one
+with the full three retries again.
+
+The one place this stops early is `corgea upload`'s per-file source upload: a 502
+that outlives the retries there is taken as the platform being unavailable rather
+than one bad file, so the remaining paths are reported as unsent instead of each
+spending another 90 seconds.
+
 ### Skipping a re-scan of the same commit
 
 A pipeline that re-runs on an unchanged commit can reuse the scan it already

{corgea_cli-1.13.0 → corgea_cli-1.14.0}/src/scan.rs

@@ -6,10 +6,67 @@ use reqwest::header;
 use serde_json::Value;
 use std::collections::HashSet;
 use std::io::{self, Read};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use uuid::Uuid;
 
+const CORGEA_POLICY_FILENAMES: &[&str] = &["corgea.yaml", "corgea.yml"];
+
+fn is_regular_file(path: &Path) -> bool {
+    !path.is_symlink() && path.is_file()
+}
+
+/// `corgea.yaml` / `corgea.yml` under `root`, including a gitignored copy at `root`.
+fn find_corgea_policy_files(root: &Path) -> Vec<String> {
+    if !root.is_dir() {
+        return Vec::new();
+    }
+
+    let mut found = Vec::new();
+    for name in CORGEA_POLICY_FILENAMES {
+        if is_regular_file(&root.join(name)) {
+            found.push((*name).to_string());
+        }
+    }
+
+    let walker = ignore::WalkBuilder::new(root)
+        .standard_filters(true)
+        .build();
+    for result in walker {
+        let Ok(entry) = result else {
+            continue;
+        };
+        let path = entry.path();
+        if !is_regular_file(path) {
+            continue;
+        }
+        let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
+            continue;
+        };
+        if !CORGEA_POLICY_FILENAMES.contains(&name) {
+            continue;
+        }
+        if let Ok(rel) = path.strip_prefix(root) {
+            let rel = rel.to_string_lossy().replace('\\', "/");
+            if !found.iter().any(|existing| existing == &rel) {
+                found.push(rel);
+            }
+        }
+    }
+    found.sort();
+    found
+}
+
+fn merge_corgea_policy_files(mut paths: Vec<String>, root: &Path) -> Vec<String> {
+    for yaml in find_corgea_policy_files(root) {
+        if !paths.iter().any(|path| path == &yaml) {
+            debug(&format!("Including repo policy file: {yaml}"));
+            paths.push(yaml);
+        }
+    }
+    paths
+}
+
 pub fn run_command(base_cmd: &String, mut command: Command) -> String {
     match which::which(base_cmd) {
         Ok(_) => {
@@ -222,6 +279,8 @@ pub fn upload_scan(
     save_to_file: bool,
     project_name: Option<String>,
 ) -> Option<ScanUploadResult> {
+    let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
+    let paths = merge_corgea_policy_files(paths, &cwd);
     let in_ci = running_in_ci();
     let ci_platform = which_ci();
     let github_env_vars = get_github_env_vars();
@@ -269,8 +328,9 @@ pub fn upload_scan(
     let mut uploaded_paths = HashSet::new();
     let mut uploaded_count = 0;
     let mut upload_error_count = 0;
+    let mut gateway_gave_up = false;
 
-    for path in &paths {
+    'files: for path in &paths {
         if !Path::new(&path).exists() {
             log::error!(
                 "Required file {} not found which is required for the scan, exiting.",
@@ -295,7 +355,7 @@ pub fn upload_scan(
 
         while attempts < 3 && !success {
             debug(&format!("POST: {}", src_upload_url));
-            let res = utils::api::retry_on_network_error("file upload", || {
+            let res = utils::api::send_with_retries("a source file upload", || {
                 let form = reqwest::blocking::multipart::Form::new()
                     .file("file", fp)
                     .expect("Failed to read file");
@@ -313,6 +373,20 @@ pub fn upload_scan(
                             "Code upload failed with status: {}. Response body: {}",
                             status, body
                         ));
+                        // A 502 that outlived the retry schedule is the platform
+                        // being unavailable, not this one file. Retrying it here
+                        // would spend the schedule twice over, and walking the
+                        // remaining paths would spend a fresh 90 seconds on each
+                        // of them, so stop uploading source files altogether.
+                        if utils::api::is_gateway_error(status) {
+                            log::warn!(
+                                "Failed to upload file {} after the gateway retries: {}",
+                                path,
+                                status
+                            );
+                            gateway_gave_up = true;
+                            break 'files;
+                        }
                         log::warn!("Failed to upload file {} {}... retrying", status, path);
                         std::thread::sleep(std::time::Duration::from_secs(1));
                         attempts += 1;
@@ -343,6 +417,19 @@ pub fn upload_scan(
         }
     }
 
+    // Everything the aborted walk never attempted still counts as unsent, or
+    // the closing summary would report one failure for a whole skipped tree.
+    if gateway_gave_up {
+        let distinct: HashSet<&String> = paths.iter().collect();
+        let unsent = distinct.len() - uploaded_paths.len();
+        upload_error_count += unsent;
+        log::warn!(
+            "Stopped uploading source files: Corgea was still answering 502 after the retries. {} of {} files were not sent.",
+            unsent,
+            distinct.len()
+        );
+    }
+
     if uploaded_count == 0 {
         log::error!("Failed to upload any files for the scan, exiting.");
         std::process::exit(1);
@@ -381,7 +468,7 @@ pub fn upload_scan(
                 index + 1,
                 total_chunks
             ));
-            let response = utils::api::retry_on_network_error("scan chunk upload", || {
+            let response = utils::api::send_with_retries("a scan report chunk upload", || {
                 client
                     .post(&scan_upload_url)
                     .header(header::CONTENT_TYPE, "application/json")
@@ -432,7 +519,7 @@ pub fn upload_scan(
         last_response.expect("Failed to upload scan.")
     } else {
         debug(&format!("POST: {}", scan_upload_url));
-        utils::api::retry_on_network_error("scan upload", || {
+        utils::api::send_with_retries("the scan report upload", || {
             client
                 .post(&scan_upload_url)
                 .header(header::CONTENT_TYPE, "application/json")
@@ -513,7 +600,7 @@ pub fn upload_scan(
     if git_config_path.exists() {
         debug("Uploading .git/config");
         debug(&format!("POST: {}", git_config_upload_url));
-        let res = utils::api::retry_on_network_error("git config upload", || {
+        let res = utils::api::send_with_retries("the git config upload", || {
             let form = reqwest::blocking::multipart::Form::new()
                 .file("file", git_config_path)
                 .expect("Failed to read file");
@@ -597,6 +684,7 @@ pub fn upload_scan(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::path::Path;
 
     #[test]
     fn scan_url_prefers_project_id_when_present() {
@@ -631,4 +719,90 @@ mod tests {
             "https://www.corgea.app/project/corgea_cli-1.0?scan_id=scan-123"
         );
     }
+
+    fn write_policy(path: &Path) {
+        std::fs::write(path, "policies: []\n").unwrap();
+    }
+
+    fn git_repo(root: &Path) {
+        std::fs::create_dir(root.join(".git")).unwrap();
+    }
+
+    #[test]
+    fn find_corgea_policy_files_picks_yaml_and_yml_by_basename() {
+        let root = tempfile::tempdir().unwrap();
+        write_policy(&root.path().join("corgea.yaml"));
+        std::fs::create_dir(root.path().join("src")).unwrap();
+        write_policy(&root.path().join("src/corgea.yml"));
+        write_policy(&root.path().join("with-guidance.yaml"));
+
+        assert_eq!(
+            find_corgea_policy_files(root.path()),
+            vec!["corgea.yaml".to_string(), "src/corgea.yml".to_string()]
+        );
+    }
+
+    #[test]
+    fn find_corgea_policy_files_returns_empty_when_none_exist() {
+        let root = tempfile::tempdir().unwrap();
+        assert!(find_corgea_policy_files(root.path()).is_empty());
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn find_corgea_policy_files_skips_symlink_outside_repo() {
+        use std::os::unix::fs::symlink;
+        let root = tempfile::tempdir().unwrap();
+        let secret = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(secret.path(), "secret").unwrap();
+        symlink(secret.path(), root.path().join("corgea.yaml")).unwrap();
+
+        assert!(find_corgea_policy_files(root.path()).is_empty());
+    }
+
+    #[test]
+    fn find_corgea_policy_files_skips_gitignored_nested_copies() {
+        let root = tempfile::tempdir().unwrap();
+        git_repo(root.path());
+        std::fs::write(root.path().join(".gitignore"), "node_modules/\n").unwrap();
+        write_policy(&root.path().join("corgea.yaml"));
+        std::fs::create_dir_all(root.path().join("node_modules/pkg")).unwrap();
+        write_policy(&root.path().join("node_modules/pkg/corgea.yaml"));
+
+        assert_eq!(
+            find_corgea_policy_files(root.path()),
+            vec!["corgea.yaml".to_string()]
+        );
+    }
+
+    #[test]
+    fn find_corgea_policy_files_keeps_root_file_even_if_gitignored() {
+        let root = tempfile::tempdir().unwrap();
+        git_repo(root.path());
+        std::fs::write(root.path().join(".gitignore"), "corgea.yaml\n").unwrap();
+        write_policy(&root.path().join("corgea.yaml"));
+
+        assert_eq!(
+            find_corgea_policy_files(root.path()),
+            vec!["corgea.yaml".to_string()]
+        );
+    }
+
+    #[test]
+    fn merge_corgea_policy_files_appends_missing_and_skips_duplicates() {
+        let root = tempfile::tempdir().unwrap();
+        write_policy(&root.path().join("corgea.yaml"));
+
+        assert_eq!(
+            merge_corgea_policy_files(vec!["src/source.py".into()], root.path()),
+            vec!["src/source.py".to_string(), "corgea.yaml".to_string()]
+        );
+        assert_eq!(
+            merge_corgea_policy_files(
+                vec!["src/source.py".into(), "corgea.yaml".into()],
+                root.path()
+            ),
+            vec!["src/source.py".to_string(), "corgea.yaml".to_string()]
+        );
+    }
 }

{corgea_cli-1.13.0 → corgea_cli-1.14.0}/src/scanners/blast.rs

@@ -16,6 +16,15 @@ use std::time::{Duration, Instant};
 const SCAN_TIMEOUT_ENV: &str = "CORGEA_SCAN_TIMEOUT_SECONDS";
 const DEFAULT_SCAN_TIMEOUT: Duration = Duration::from_secs(10 * 60 * 60);
 
+/// How long to pause between scan status reads.
+///
+/// Scans run for minutes, so reading more often than this only adds load: at
+/// one read a second, a single wait against the default 10-hour budget can
+/// reach 36,000 requests, and a pipeline scanning in parallel multiplies that
+/// by every concurrent wait. Three seconds costs a pipeline no more than a few
+/// seconds of extra latency on the final status.
+const SCAN_POLL_INTERVAL: Duration = Duration::from_secs(3);
+
 /// Overrides how long the CI gate waits for blocking rules to be evaluated.
 const BLOCKING_RULES_TIMEOUT_ENV: &str = "CORGEA_BLOCKING_RULES_TIMEOUT_SECONDS";
 
@@ -1034,7 +1043,7 @@ pub fn wait_for_scan(config: &Config, scan_id: &str, budget: WaitBudget) {
     let mut last_status = String::from("unknown");
 
     let result = loop {
-        thread::sleep(Duration::from_secs(1));
+        thread::sleep(SCAN_POLL_INTERVAL);
         // Every read is capped to what is left of the budget: on the client's
         // own timeout a stalled read would otherwise keep us going long past
         // the wait the user asked for.
@@ -1781,6 +1790,18 @@ mod tests {
         assert_eq!(format_timeout(Duration::from_secs(90)), "90s");
     }
 
+    #[test]
+    fn poll_interval_is_the_documented_one_and_short_against_the_budget() {
+        assert_eq!(SCAN_POLL_INTERVAL, Duration::from_secs(3));
+        // A pause that ever grew toward the budget would spend the whole wait
+        // asleep and report a finished scan long after it finished.
+        assert!(
+            SCAN_POLL_INTERVAL < DEFAULT_SCAN_TIMEOUT / 100,
+            "poll interval {SCAN_POLL_INTERVAL:?} is not short against the \
+             {DEFAULT_SCAN_TIMEOUT:?} budget it runs inside"
+        );
+    }
+
     #[test]
     fn blocking_rules_timeout_outlasts_the_doghouse_wait_windows() {
         // This side fails closed on its own deadline, so it must never expire