shellsim 0.1.25__tar.gz → 0.1.27__tar.gz

{shellsim-0.1.25 → shellsim-0.1.27}/Cargo.lock

@@ -38,6 +38,15 @@ version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
+[[package]]
+name = "ar_archive_writer"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
+dependencies = [
+ "object 0.39.1",
+]
+
 [[package]]
 name = "arbitrary"
 version = "1.4.2"
@@ -619,6 +628,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "object"
+version = "0.39.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "object"
 version = "0.40.0"
@@ -664,6 +682,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "psm"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622"
+dependencies = [
+ "ar_archive_writer",
+ "cc",
+]
+
 [[package]]
 name = "pulley-interpreter"
 version = "49.0.0"
@@ -861,7 +889,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim"
-version = "0.1.25"
+version = "0.1.27"
 dependencies = [
  "base64",
  "crc32fast",
@@ -879,6 +907,7 @@ dependencies = [
  "serde_json",
  "sha1",
  "sha2",
+ "stacker",
  "toml",
  "wasmtime",
  "wat",
@@ -917,6 +946,19 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
+[[package]]
+name = "stacker"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "psm",
+ "windows-sys",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.118"
@@ -1122,7 +1164,7 @@ dependencies = [
  "log",
  "mach2",
  "memfd",
- "object",
+ "object 0.40.0",
  "once_cell",
  "postcard",
  "pulley-interpreter",
@@ -1159,7 +1201,7 @@ dependencies = [
  "hashbrown 0.17.1",
  "indexmap",
  "log",
- "object",
+ "object 0.40.0",
  "postcard",
  "rustc-demangle",
  "semver",
@@ -1221,7 +1263,7 @@ dependencies = [
  "gimli",
  "itertools",
  "log",
- "object",
+ "object 0.40.0",
  "pulley-interpreter",
  "smallvec",
  "target-lexicon",
@@ -1276,7 +1318,7 @@ checksum = "5978e43aa5e9a3a03ebea8a4d4a015c92144268167abb27d99ab1b53224b28ae"
 dependencies = [
  "cranelift-codegen",
  "log",
- "object",
+ "object 0.40.0",
  "wasmtime-environ",
 ]
 

{shellsim-0.1.25 → shellsim-0.1.27}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shellsim"
-version = "0.1.25"
+version = "0.1.27"
 edition = "2021"
 description = "In-process BusyBox-like environment for RL rollouts and agent tests"
 license = "Apache-2.0"
@@ -44,6 +44,7 @@ wasmtime = { version = "49.0.0", default-features = false, features = ["runtime"
 num-bigint = "0.4"
 num-traits = "0.2"
 libm = "0.2"
+stacker = "0.1"
 
 # Process-level syscall sandbox (Linux only): a startup seccomp filter that denies
 # network/exec/fork so no Python-level trick can reach the real OS for those. shellsim

{shellsim-0.1.25 → shellsim-0.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shellsim
-Version: 0.1.25
+Version: 0.1.27
 Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: Microsoft :: Windows
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust
 Classifier: Topic :: Software Development :: Testing
 Classifier: Typing :: Typed
-Requires-Dist: shellsim-c-toolchain==0.1.25 ; extra == 'c'
+Requires-Dist: shellsim-c-toolchain==0.1.27 ; extra == 'c'
 Provides-Extra: c
 License-File: LICENSE
 Summary: In-process BusyBox-like environment for RL rollouts and agent tests

{shellsim-0.1.25 → shellsim-0.1.27}/pyproject.toml

@@ -25,7 +25,7 @@ classifiers = [
 keywords = ["agents", "containers", "reinforcement-learning", "sandbox", "simulation", "testing"]
 
 [project.optional-dependencies]
-c = ["shellsim-c-toolchain==0.1.25"]
+c = ["shellsim-c-toolchain==0.1.27"]
 
 [project.urls]
 Homepage = "https://github.com/rjpower/shellsim"

{shellsim-0.1.25 → shellsim-0.1.27}/python/native/Cargo.lock

@@ -38,6 +38,15 @@ version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
+[[package]]
+name = "ar_archive_writer"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
+dependencies = [
+ "object 0.39.1",
+]
+
 [[package]]
 name = "arbitrary"
 version = "1.4.2"
@@ -619,6 +628,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "object"
+version = "0.39.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "object"
 version = "0.40.0"
@@ -670,6 +688,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "psm"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622"
+dependencies = [
+ "ar_archive_writer",
+ "cc",
+]
+
 [[package]]
 name = "pulley-interpreter"
 version = "49.0.0"
@@ -925,7 +953,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim"
-version = "0.1.25"
+version = "0.1.27"
 dependencies = [
  "base64",
  "crc32fast",
@@ -943,13 +971,14 @@ dependencies = [
  "serde_json",
  "sha1",
  "sha2",
+ "stacker",
  "toml",
  "wasmtime",
 ]
 
 [[package]]
 name = "shellsim-python-bindings"
-version = "0.1.25"
+version = "0.1.27"
 dependencies = [
  "base64",
  "pyo3",
@@ -991,6 +1020,19 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
+[[package]]
+name = "stacker"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "psm",
+ "windows-sys",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.119"
@@ -1169,7 +1211,7 @@ dependencies = [
  "log",
  "mach2",
  "memfd",
- "object",
+ "object 0.40.0",
  "once_cell",
  "postcard",
  "pulley-interpreter",
@@ -1206,7 +1248,7 @@ dependencies = [
  "hashbrown 0.17.1",
  "indexmap",
  "log",
- "object",
+ "object 0.40.0",
  "postcard",
  "rustc-demangle",
  "semver",
@@ -1268,7 +1310,7 @@ dependencies = [
  "gimli",
  "itertools",
  "log",
- "object",
+ "object 0.40.0",
  "pulley-interpreter",
  "smallvec",
  "target-lexicon",
@@ -1323,7 +1365,7 @@ checksum = "5978e43aa5e9a3a03ebea8a4d4a015c92144268167abb27d99ab1b53224b28ae"
 dependencies = [
  "cranelift-codegen",
  "log",
- "object",
+ "object 0.40.0",
  "wasmtime-environ",
 ]
 

{shellsim-0.1.25 → shellsim-0.1.27}/python/native/Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name = "shellsim-python-bindings"
-version = "0.1.25"
+version = "0.1.27"
 edition = "2021"
 description = "Python bindings for shellsim"
 license = "Apache-2.0"

{shellsim-0.1.25 → shellsim-0.1.27}/src/bin/shellsim-python.rs

@@ -3,6 +3,8 @@
 //! Host filesystem access ends at project ingestion. The selected tree is copied into a fresh
 //! VFS before Python starts, and simulated code receives no path back to the host.
 
+#![forbid(unsafe_code)]
+
 use std::fs;
 use std::io::{IsTerminal, Read, Write};
 use std::path::{Path, PathBuf};

{shellsim-0.1.25 → shellsim-0.1.27}/src/commands/awk/eval.rs

@@ -95,6 +95,11 @@ pub(super) fn validate(program: &Program) -> Result<(), String> {
 }
 
 fn validate_stmt(statement: &Stmt) -> Result<(), String> {
+    // The parser bounds tree depth; growth keeps that depth safe on small host stacks.
+    crate::stack::grow(|| validate_stmt_inner(statement))
+}
+
+fn validate_stmt_inner(statement: &Stmt) -> Result<(), String> {
     match statement {
         Stmt::Block(statements) => {
             for statement in statements {
@@ -167,6 +172,11 @@ fn validate_lvalue(target: &LValue) -> Result<(), String> {
 }
 
 fn validate_expr(expression: &Expr) -> Result<(), String> {
+    // The parser bounds tree depth; growth keeps that depth safe on small host stacks.
+    crate::stack::grow(|| validate_expr_inner(expression))
+}
+
+fn validate_expr_inner(expression: &Expr) -> Result<(), String> {
     match expression {
         Expr::Regex(pattern) => regex::Regex::new(pattern)
             .map(|_| ())
@@ -230,6 +240,8 @@ pub(super) fn execute(
         io,
         state,
         literal_regexes: HashMap::new(),
+        state_reserved: 0,
+        fields_reserved: 0,
     };
 
     let mut requested_exit = match runtime.phase(program, Phase::Begin) {
@@ -273,9 +285,62 @@ struct Runtime<'a, 'io> {
     io: &'a mut Io<'io>,
     state: State,
     literal_regexes: HashMap<String, regex::Regex>,
+    /// Machine memory reserved for variables and array elements assigned by the program.
+    state_reserved: u64,
+    /// The part of `state_reserved` held by the current record's fields.
+    fields_reserved: u64,
+}
+
+impl Drop for Runtime<'_, '_> {
+    fn drop(&mut self) {
+        self.system.release_memory(self.state_reserved);
+    }
+}
+
+/// Modeled bookkeeping cost of one stored variable or array element beyond its text.
+const SLOT_OVERHEAD: u64 = 64;
+/// Modeled bytes of one split field beyond its text.
+const FIELD_OVERHEAD: u64 = 32;
+
+/// Modeled bytes of one variable or array element stored under `key`.
+fn slot_bytes(key: &str, value: &Scalar) -> u64 {
+    (key.len() as u64)
+        .saturating_add(value.text.len() as u64)
+        .saturating_add(SLOT_OVERHEAD)
 }
 
 impl Runtime<'_, '_> {
+    /// Reserve growth, or release shrinkage, of program state before the change is stored.
+    fn resize_state(&mut self, old: u64, new: u64) -> Result<(), Flow> {
+        if new > old {
+            if !self.system.reserve_memory(new - old) {
+                return Err(Flow::Exhausted);
+            }
+            self.state_reserved = self.state_reserved.saturating_add(new - old);
+        } else {
+            self.system.release_memory(old - new);
+            self.state_reserved = self.state_reserved.saturating_sub(old - new);
+        }
+        Ok(())
+    }
+
+    /// Store one array element, charging its size first.
+    fn store_element(&mut self, name: &str, key: String, value: Scalar) -> Result<(), Flow> {
+        let old = self
+            .state
+            .arrays
+            .get(name)
+            .and_then(|array| array.get(&key))
+            .map_or(0, |old| slot_bytes(&key, old));
+        self.resize_state(old, slot_bytes(&key, &value))?;
+        self.state
+            .arrays
+            .entry(name.to_string())
+            .or_default()
+            .insert(key, value);
+        Ok(())
+    }
+
     fn phase(&mut self, program: &Program, phase: Phase) -> Flow {
         for rule in &program.rules {
             let selected = match (&rule.pattern, phase) {
@@ -309,6 +374,10 @@ impl Runtime<'_, '_> {
     }
 
     fn statement(&mut self, statement: &Stmt) -> Flow {
+        crate::stack::grow(|| self.statement_inner(statement))
+    }
+
+    fn statement_inner(&mut self, statement: &Stmt) -> Flow {
         if !self.system.charge_cpu(1) {
             return Flow::Exhausted;
         }
@@ -392,8 +461,13 @@ impl Runtime<'_, '_> {
             Stmt::Continue => Flow::Continue,
             Stmt::Delete(target) => match self.array_target(target) {
                 Ok((name, key)) => {
-                    if let Some(array) = self.state.arrays.get_mut(&name) {
-                        array.remove(&key);
+                    let removed = self
+                        .state
+                        .arrays
+                        .get_mut(&name)
+                        .and_then(|array| array.remove(&key));
+                    if let Some(removed) = removed {
+                        let _ = self.resize_state(slot_bytes(&key, &removed), 0);
                     }
                     Flow::Normal
                 }
@@ -457,6 +531,10 @@ impl Runtime<'_, '_> {
     }
 
     fn expr(&mut self, expression: &Expr) -> Result<Scalar, Flow> {
+        crate::stack::grow(|| self.expr_inner(expression))
+    }
+
+    fn expr_inner(&mut self, expression: &Expr) -> Result<Scalar, Flow> {
         if !self.system.charge_cpu(1) {
             return Err(Flow::Exhausted);
         }
@@ -740,13 +818,27 @@ impl Runtime<'_, '_> {
         } else {
             self.state.fs.clone()
         };
-        let values = split_fields(&input, &separator)?;
-        let array = self.state.arrays.entry(array_name.clone()).or_default();
-        array.clear();
-        for (index, value) in values.iter().enumerate() {
-            array.insert((index + 1).to_string(), Scalar::string(value.clone()));
-        }
-        Ok(Scalar::number(values.len() as f64))
+        let released = self
+            .state
+            .arrays
+            .remove(array_name.as_str())
+            .map_or(0, |array| {
+                array.iter().fold(0_u64, |total, (key, value)| {
+                    total.saturating_add(slot_bytes(key, value))
+                })
+            });
+        self.resize_state(released, 0)?;
+        self.state.arrays.insert(array_name.clone(), HashMap::new());
+        let mut count = 0_usize;
+        for_each_field(&input, &separator, |field| {
+            count += 1;
+            self.store_element(
+                array_name,
+                count.to_string(),
+                Scalar::string(field.to_string()),
+            )
+        })?;
+        Ok(Scalar::number(count as f64))
     }
 
     fn substitute(&mut self, args: &[Expr], global: bool) -> Result<Scalar, Flow> {
@@ -851,6 +943,12 @@ impl Runtime<'_, '_> {
                 )))
             }
             _ => {
+                let old = self
+                    .state
+                    .vars
+                    .get(name)
+                    .map_or(0, |old| slot_bytes(name, old));
+                self.resize_state(old, slot_bytes(name, &value))?;
                 self.state.vars.insert(name.to_string(), value);
             }
         }
@@ -870,8 +968,21 @@ impl Runtime<'_, '_> {
         }
     }
 
+    /// Replace `$0` and its fields, charging each field before it is stored.
     fn set_record(&mut self, record: String) -> Result<(), Flow> {
-        self.state.fields = split_fields(&record, &self.state.fs)?;
+        self.state.fields.clear();
+        self.resize_state(self.fields_reserved, 0)?;
+        self.fields_reserved = 0;
+        let separator = self.state.fs.clone();
+        let mut fields = Vec::new();
+        for_each_field(&record, &separator, |field| {
+            let bytes = FIELD_OVERHEAD.saturating_add(field.len() as u64);
+            self.resize_state(0, bytes)?;
+            self.fields_reserved = self.fields_reserved.saturating_add(bytes);
+            fields.push(field.to_string());
+            Ok(())
+        })?;
+        self.state.fields = fields;
         self.state.record = record;
         Ok(())
     }
@@ -936,11 +1047,7 @@ impl Runtime<'_, '_> {
             }
             LValue::Array { name, indices } => {
                 let key = self.array_key(indices)?;
-                self.state
-                    .arrays
-                    .entry(name.clone())
-                    .or_default()
-                    .insert(key, value);
+                self.store_element(name, key, value)?;
             }
         }
         Ok(())
@@ -991,12 +1098,27 @@ pub(super) fn assignment_is_unsupported(name: &str) -> bool {
 }
 
 fn split_fields(record: &str, separator: &str) -> Result<Vec<String>, Flow> {
+    let mut fields = Vec::new();
+    for_each_field(record, separator, |field| {
+        fields.push(field.to_string());
+        Ok(())
+    })?;
+    Ok(fields)
+}
+
+/// Visit the fields of `record` in order, so callers can charge each one before storing it.
+fn for_each_field(
+    record: &str,
+    separator: &str,
+    mut visit: impl FnMut(&str) -> Result<(), Flow>,
+) -> Result<(), Flow> {
     if separator == " " || separator.is_empty() {
-        return Ok(record.split_whitespace().map(str::to_string).collect());
+        return record.split_whitespace().try_for_each(visit);
     }
-    regex::Regex::new(separator)
-        .map(|regex| regex.split(record).map(str::to_string).collect())
-        .map_err(|error| Flow::Error(format!("invalid field separator: {error}")))
+    let regex = regex::Regex::new(separator)
+        .map_err(|error| Flow::Error(format!("invalid field separator: {error}")))?;
+    let result = regex.split(record).try_for_each(&mut visit);
+    result
 }
 
 /// Check an initial field separator before input is read or output is produced.
@@ -1168,6 +1290,11 @@ fn take_digits(chars: &mut std::iter::Peekable<std::str::Chars<'_>>) -> Option<S
 }
 
 fn number_text(value: f64) -> String {
+    // Integer formatting is much cheaper than exact float formatting, and `-0` keeps its sign
+    // through the float path.
+    if value.fract() == 0.0 && value.abs() < 1e18 && !(value == 0.0 && value.is_sign_negative()) {
+        return (value as i64).to_string();
+    }
     if value.fract() == 0.0 && value.is_finite() {
         format!("{value:.0}")
     } else {