shellsim 0.1.9__tar.gz → 0.1.10__tar.gz

{shellsim-0.1.9 → shellsim-0.1.10}/Cargo.lock

@@ -330,7 +330,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim"
-version = "0.1.9"
+version = "0.1.10"
 dependencies = [
  "base64",
  "crc32fast",

{shellsim-0.1.9 → shellsim-0.1.10}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shellsim"
-version = "0.1.9"
+version = "0.1.10"
 edition = "2021"
 description = "In-process BusyBox-like environment for RL rollouts and agent tests"
 license = "Apache-2.0"

{shellsim-0.1.9 → shellsim-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shellsim
-Version: 0.1.9
+Version: 0.1.10
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: MacOS :: MacOS X

{shellsim-0.1.9 → shellsim-0.1.10}/python/native/Cargo.lock

@@ -406,7 +406,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim"
-version = "0.1.9"
+version = "0.1.10"
 dependencies = [
  "base64",
  "crc32fast",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim-python-bindings"
-version = "0.1.9"
+version = "0.1.10"
 dependencies = [
  "pyo3",
  "serde",

{shellsim-0.1.9 → shellsim-0.1.10}/python/native/Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name = "shellsim-python-bindings"
-version = "0.1.9"
+version = "0.1.10"
 edition = "2021"
 description = "Python bindings for shellsim"
 license = "Apache-2.0"

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/ast.rs

@@ -50,10 +50,17 @@ pub enum StatementKind {
         body: Vec<Statement>,
         otherwise: Vec<Statement>,
     },
+    AsyncFor {
+        target: AssignmentTarget,
+        iterable: Expression,
+        body: Vec<Statement>,
+        otherwise: Vec<Statement>,
+    },
     Function {
         name: String,
         parameters: Vec<Parameter>,
         body: Vec<Statement>,
+        is_async: bool,
     },
     Class {
         name: String,
@@ -91,6 +98,11 @@ pub enum StatementKind {
         target: Option<AssignmentTarget>,
         body: Vec<Statement>,
     },
+    AsyncWith {
+        context: Expression,
+        target: Option<AssignmentTarget>,
+        body: Vec<Statement>,
+    },
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -122,11 +134,13 @@ impl StatementKind {
             Self::If { .. }
                 | Self::While { .. }
                 | Self::For { .. }
+                | Self::AsyncFor { .. }
                 | Self::Function { .. }
                 | Self::Class { .. }
                 | Self::Decorated { .. }
                 | Self::Try { .. }
                 | Self::With { .. }
+                | Self::AsyncWith { .. }
         )
     }
 }
@@ -180,6 +194,7 @@ pub enum ExpressionKind {
     },
     Yield(Option<Box<Expression>>),
     YieldFrom(Box<Expression>),
+    Await(Box<Expression>),
     Attribute {
         value: Box<Expression>,
         name: String,

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/bytecode.rs

@@ -65,6 +65,7 @@ pub struct Code {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CallSignature {
     pub is_generator: bool,
+    pub is_coroutine: bool,
     pub positional_count: usize,
     pub variadic_slot: Option<usize>,
     pub default_slots: Box<[usize]>,
@@ -223,9 +224,12 @@ pub enum Opcode {
     Reraise,
     Raise(bool),
     Yield,
+    AwaitResult,
     WithEnter,
     WithExit,
     WithExitException,
+    AsyncWithExitException,
+    AsyncWithFinishException,
     PopExpression,
     Halt,
 }
@@ -321,9 +325,13 @@ pub enum Operation {
     Raise(bool),
     /// Suspend a generator frame and return the value on top of the stack.
     Yield,
+    /// Unwrap the scheduler outcome sent into a suspended coroutine.
+    AwaitResult,
     WithEnter,
     WithExit,
     WithExitException,
+    AsyncWithExitException,
+    AsyncWithFinishException,
     PopExpression,
     Halt,
 }
@@ -489,9 +497,12 @@ impl CodeBuilder {
             Operation::Reraise => Opcode::Reraise,
             Operation::Raise(cause) => Opcode::Raise(cause),
             Operation::Yield => Opcode::Yield,
+            Operation::AwaitResult => Opcode::AwaitResult,
             Operation::WithEnter => Opcode::WithEnter,
             Operation::WithExit => Opcode::WithExit,
             Operation::WithExitException => Opcode::WithExitException,
+            Operation::AsyncWithExitException => Opcode::AsyncWithExitException,
+            Operation::AsyncWithFinishException => Opcode::AsyncWithFinishException,
             Operation::PopExpression => Opcode::PopExpression,
             Operation::Halt => Opcode::Halt,
         }
@@ -503,6 +514,7 @@ impl CodeBuilder {
         spans: Vec<Span>,
         parameters: Vec<Parameter>,
         local_names: Vec<String>,
+        is_coroutine: bool,
     ) -> CodeRef {
         let positional_count = parameters
             .iter()
@@ -512,6 +524,7 @@ impl CodeBuilder {
             is_generator: instructions
                 .iter()
                 .any(|instruction| matches!(instruction.opcode, Opcode::Yield)),
+            is_coroutine,
             positional_count,
             variadic_slot: parameters.iter().position(|parameter| parameter.variadic),
             default_slots: parameters

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/compiler.rs

@@ -15,6 +15,7 @@ pub fn compile(program: Program) -> CodeRef {
         finalizers: Vec::new(),
         protected_regions: Vec::new(),
         in_function: false,
+        is_coroutine: false,
         globals: HashSet::new(),
         nonlocals: HashSet::new(),
         named_expression: NamedExpressionContext::local(),
@@ -38,6 +39,7 @@ struct Compiler {
     /// at compile time rather than losing that state in a suspended generator.
     protected_regions: Vec<&'static str>,
     in_function: bool,
+    is_coroutine: bool,
     globals: HashSet<String>,
     nonlocals: HashSet<String>,
     named_expression: NamedExpressionContext,
@@ -155,7 +157,7 @@ impl Compiler {
             });
             spans.push(instruction.span);
         }
-        builder.finish(bytecode, spans, parameters, local_names)
+        builder.finish(bytecode, spans, parameters, local_names, self.is_coroutine)
     }
 
     fn statements(&mut self, statements: Vec<Statement>) {
@@ -320,10 +322,70 @@ impl Compiler {
                 let finished = self.instructions.len();
                 self.finish_loop(finished);
             }
+            StatementKind::AsyncFor {
+                target,
+                iterable,
+                body,
+                otherwise,
+            } => {
+                let iterator_name =
+                    format!("$__shellsim_async_iterator_{}", self.instructions.len());
+                self.expression(iterable);
+                self.emit(Operation::LoadAttribute("__aiter__".into()), span);
+                self.emit(
+                    Operation::Call {
+                        positional: 0,
+                        keywords: Vec::new(),
+                        starred: Vec::new(),
+                    },
+                    span,
+                );
+                self.emit(Operation::StoreName(iterator_name.clone()), span);
+                let next = self.instructions.len();
+                let begin = self.emit(Operation::TryBegin(usize::MAX), span);
+                self.emit(Operation::LoadName(iterator_name), span);
+                self.emit(Operation::LoadAttribute("__anext__".into()), span);
+                self.emit(
+                    Operation::Call {
+                        positional: 0,
+                        keywords: Vec::new(),
+                        starred: Vec::new(),
+                    },
+                    span,
+                );
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                self.emit(Operation::TryEnd, span);
+                self.store_target(target, span);
+                self.loops.push(LoopContext {
+                    continue_target: next,
+                    iterator_on_stack: false,
+                    finalizer_depth: self.finalizers.len(),
+                    breaks: Vec::new(),
+                });
+                self.statements(body);
+                self.emit(Operation::Jump(next), span);
+                let handler = self.instructions.len();
+                self.patch_try_begin(begin, handler);
+                self.emit(Operation::LoadName("StopAsyncIteration".into()), span);
+                self.emit(Operation::MatchException { typed: true }, span);
+                let rethrow = self.emit(Operation::PopJumpIfFalse(usize::MAX), span);
+                self.emit(Operation::PopTop, span);
+                self.emit(Operation::ClearException, span);
+                self.statements(otherwise);
+                let done = self.emit(Operation::Jump(usize::MAX), span);
+                self.patch_jump(rethrow, self.instructions.len());
+                self.emit(Operation::PopTop, span);
+                self.emit(Operation::Reraise, span);
+                let finished = self.instructions.len();
+                self.patch_jump(done, finished);
+                self.finish_loop(finished);
+            }
             StatementKind::Function {
                 name,
                 parameters,
                 body,
+                is_async,
             } => {
                 let defaults = parameters
                     .iter()
@@ -338,6 +400,7 @@ impl Compiler {
                     finalizers: Vec::new(),
                     protected_regions: Vec::new(),
                     in_function: true,
+                    is_coroutine: is_async,
                     globals: HashSet::new(),
                     nonlocals: HashSet::new(),
                     named_expression: NamedExpressionContext::local(),
@@ -391,6 +454,7 @@ impl Compiler {
                     finalizers: Vec::new(),
                     protected_regions: Vec::new(),
                     in_function: false,
+                    is_coroutine: false,
                     globals: HashSet::new(),
                     nonlocals: HashSet::new(),
                     named_expression: NamedExpressionContext::local(),
@@ -660,6 +724,60 @@ impl Compiler {
                 self.emit(Operation::WithExitException, span);
                 self.patch_jump(done, self.instructions.len());
             }
+            StatementKind::AsyncWith {
+                context,
+                target,
+                body,
+            } => {
+                let context_name = format!("$__shellsim_async_context_{}", self.instructions.len());
+                self.expression(context);
+                self.emit(Operation::StoreName(context_name.clone()), span);
+                self.emit(Operation::LoadName(context_name.clone()), span);
+                self.emit(Operation::LoadAttribute("__aenter__".into()), span);
+                self.emit(
+                    Operation::Call {
+                        positional: 0,
+                        keywords: Vec::new(),
+                        starred: Vec::new(),
+                    },
+                    span,
+                );
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                if let Some(target) = target {
+                    self.store_target(target, span);
+                } else {
+                    self.emit(Operation::PopTop, span);
+                }
+                let begin = self.emit(Operation::TryBegin(usize::MAX), span);
+                self.statements(body);
+                self.emit(Operation::TryEnd, span);
+                self.emit(Operation::LoadName(context_name.clone()), span);
+                self.emit(Operation::LoadAttribute("__aexit__".into()), span);
+                for _ in 0..3 {
+                    self.emit(Operation::LoadConstant(Constant::None), span);
+                }
+                self.emit(
+                    Operation::Call {
+                        positional: 3,
+                        keywords: Vec::new(),
+                        starred: vec![false; 3],
+                    },
+                    span,
+                );
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                self.emit(Operation::PopTop, span);
+                let done = self.emit(Operation::Jump(usize::MAX), span);
+                let handler = self.instructions.len();
+                self.patch_try_begin(begin, handler);
+                self.emit(Operation::LoadName(context_name), span);
+                self.emit(Operation::AsyncWithExitException, span);
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                self.emit(Operation::AsyncWithFinishException, span);
+                self.patch_jump(done, self.instructions.len());
+            }
         }
     }
 
@@ -971,6 +1089,18 @@ impl Compiler {
                     self.emit(Operation::LoadConstant(Constant::None), span);
                 }
             }
+            ExpressionKind::Await(value) => {
+                if !self.is_coroutine {
+                    self.emit(
+                        Operation::RuntimeError("'await' outside async function".into()),
+                        span,
+                    );
+                } else {
+                    self.expression(*value);
+                    self.emit(Operation::Yield, span);
+                    self.emit(Operation::AwaitResult, span);
+                }
+            }
             ExpressionKind::Attribute { value, name } => {
                 self.expression(*value);
                 self.emit(Operation::LoadAttribute(name), span);
@@ -1042,6 +1172,7 @@ impl Compiler {
                     finalizers: Vec::new(),
                     protected_regions: Vec::new(),
                     in_function: true,
+                    is_coroutine: false,
                     globals: HashSet::new(),
                     nonlocals: HashSet::new(),
                     named_expression: NamedExpressionContext::local(),
@@ -1190,6 +1321,7 @@ impl Compiler {
             finalizers: Vec::new(),
             protected_regions: Vec::new(),
             in_function: true,
+            is_coroutine: false,
             globals: HashSet::new(),
             nonlocals: HashSet::new(),
             named_expression,
@@ -1239,6 +1371,7 @@ impl Compiler {
             finalizers: Vec::new(),
             protected_regions: Vec::new(),
             in_function: true,
+            is_coroutine: false,
             globals: HashSet::new(),
             nonlocals: HashSet::new(),
             named_expression,
@@ -1280,6 +1413,7 @@ impl Compiler {
             finalizers: Vec::new(),
             protected_regions: Vec::new(),
             in_function: true,
+            is_coroutine: false,
             globals: HashSet::new(),
             nonlocals: HashSet::new(),
             named_expression,

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/heap.rs

@@ -212,6 +212,7 @@ pub enum Object {
         stack: Vec<Value>,
         exhausted: bool,
         running: bool,
+        return_value: Value,
     },
     Module {
         name: String,
@@ -1430,9 +1431,15 @@ fn trace_object(
             trace_value(*callable, object_work);
             trace_value(*sentinel, object_work);
         }
-        Object::Generator { scope, stack, .. } => {
+        Object::Generator {
+            scope,
+            stack,
+            return_value,
+            ..
+        } => {
             scope_work.push(*scope);
             trace_values(stack.iter().copied(), object_work);
+            trace_value(*return_value, object_work);
         }
         Object::Module { scope, .. } => scope_work.push(*scope),
         Object::Array { storage, .. } => object_work.push(*storage),

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/lexer.rs

@@ -280,6 +280,8 @@ impl<'a> Lexer<'a> {
             "raise" => TokenKind::Raise,
             "with" => TokenKind::With,
             "yield" => TokenKind::Yield,
+            "async" => TokenKind::Async,
+            "await" => TokenKind::Await,
             name => TokenKind::Name((*name).to_string()),
         }
     }

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/mod.rs

@@ -1302,6 +1302,7 @@ fn collect_pytest_functions(source: &str) -> Result<PytestCollection, String> {
             name,
             parameters,
             body,
+            ..
         } = function
         else {
             unreachable!()

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/native.rs

@@ -452,6 +452,9 @@ pub(super) trait PyRuntime {
         generator: PyIterator,
         value: PyValue,
     ) -> PyResult<Option<PyValue>>;
+    fn generator_return_value(&self, generator: PyIterator) -> PyResult<PyValue>;
+    /// Resume a coroutine and return 0 for yield, 1 for return, or 2 for an exception.
+    fn coroutine_step(&mut self, coroutine: PyIterator, value: PyValue) -> PyResult<(u8, PyValue)>;
     fn generator_close(&mut self, generator: PyIterator) -> PyResult<()>;
     fn generator_throw(&mut self, generator: PyIterator, exception: PyValue) -> PyResult;
     fn new_iterator(&mut self, values: Vec<PyValue>) -> PyResult<PyValue>;

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/parser.rs

@@ -90,6 +90,12 @@ impl Parser {
             )?;
             self.separators();
         }
+        let is_async = self.take(|kind| matches!(kind, TokenKind::Async)).is_some();
+        if is_async
+            && !self.at(|kind| matches!(kind, TokenKind::Def | TokenKind::For | TokenKind::With))
+        {
+            return Err(self.error("expected 'def', 'for', or 'with' after 'async'"));
+        }
         let kind = if self.take(|kind| matches!(kind, TokenKind::If)).is_some() {
             self.if_statement(start)?
         } else if self.take(|kind| matches!(kind, TokenKind::Try)).is_some() {
@@ -145,10 +151,18 @@ impl Parser {
                 None
             };
             let body = self.suite()?;
-            StatementKind::With {
-                context,
-                target,
-                body,
+            if is_async {
+                StatementKind::AsyncWith {
+                    context,
+                    target,
+                    body,
+                }
+            } else {
+                StatementKind::With {
+                    context,
+                    target,
+                    body,
+                }
             }
         } else if self.take(|kind| matches!(kind, TokenKind::While)).is_some() {
             let test = self.expression()?;
@@ -176,11 +190,20 @@ impl Parser {
             } else {
                 Vec::new()
             };
-            StatementKind::For {
-                target,
-                iterable,
-                body,
-                otherwise,
+            if is_async {
+                StatementKind::AsyncFor {
+                    target,
+                    iterable,
+                    body,
+                    otherwise,
+                }
+            } else {
+                StatementKind::For {
+                    target,
+                    iterable,
+                    body,
+                    otherwise,
+                }
             }
         } else if self.take(|kind| matches!(kind, TokenKind::Def)).is_some() {
             let name = self.name("expected a function name after 'def'")?;
@@ -257,6 +280,7 @@ impl Parser {
                 name,
                 parameters,
                 body: self.suite()?,
+                is_async,
             }
         } else if self.take(|kind| matches!(kind, TokenKind::Class)).is_some() {
             let name = self.name("expected a class name after 'class'")?;
@@ -1180,6 +1204,13 @@ impl Parser {
 
     fn unary(&mut self) -> Result<Expression, ParseError> {
         let start = self.peek().span;
+        if self.take(|kind| matches!(kind, TokenKind::Await)).is_some() {
+            let value = self.postfix()?;
+            return Ok(Expression {
+                span: start.through(value.span),
+                kind: ExpressionKind::Await(Box::new(value)),
+            });
+        }
         let operator = if self.take(|kind| matches!(kind, TokenKind::Plus)).is_some() {
             Some(UnaryOperator::Positive)
         } else if self.take(|kind| matches!(kind, TokenKind::Minus)).is_some() {

shellsim-0.1.10/src/python/stdlib/asyncio.rs

@@ -0,0 +1,47 @@
+//! VM primitives used by the capability-free `asyncio` compatibility module.
+//!
+//! Scheduling policy stays in frozen Python. This module exposes only bounded coroutine frame
+//! stepping so the facade cannot acquire host threads, clocks, or I/O capabilities.
+
+use super::super::native::{
+    CallArgs, FunctionDef, ModuleDef, PyError, PyIterator, PyKind, PyResult, PyRuntime, PyValueCast,
+};
+use super::super::Value;
+
+pub(super) static MODULE: ModuleDef = ModuleDef {
+    name: "_asyncio",
+    functions: &[
+        FunctionDef {
+            module: "_asyncio",
+            name: "_step",
+            call: step,
+        },
+        FunctionDef {
+            module: "_asyncio",
+            name: "_is_coroutine",
+            call: is_coroutine,
+        },
+    ],
+    values: &[],
+};
+
+fn step(runtime: &mut dyn PyRuntime, args: CallArgs) -> PyResult {
+    args.expect_positional("_asyncio._step", 2, 2)?;
+    args.reject_keywords("_asyncio._step")?;
+    let coroutine = args.positional()[0];
+    if runtime.kind(&coroutine)? != PyKind::Generator {
+        return Err(PyError::type_error("expected a coroutine"));
+    }
+    let iterator = coroutine.cast::<PyIterator>(runtime)?;
+    let sent = args.positional()[1];
+    let (status, value) = runtime.coroutine_step(iterator, sent)?;
+    runtime.new_tuple(vec![Value::Int(i64::from(status)), value])
+}
+
+fn is_coroutine(runtime: &mut dyn PyRuntime, args: CallArgs) -> PyResult {
+    args.expect_positional("_asyncio._is_coroutine", 1, 1)?;
+    args.reject_keywords("_asyncio._is_coroutine")?;
+    Ok(Value::Bool(
+        runtime.kind(&args.positional()[0])? == PyKind::Generator,
+    ))
+}

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/stdlib/frozen.rs

@@ -7,6 +7,7 @@
 /// cannot accidentally expose build-machine files at runtime.
 pub(super) fn module_source(name: &str) -> Option<&'static str> {
     match name {
+        "asyncio" => Some(include_str!("source/asyncio.py")),
         "abc" => Some(include_str!("source/abc.py")),
         "base64" => Some(include_str!("source/base64.py")),
         "codecs" => Some(include_str!("source/codecs.py")),

{shellsim-0.1.9 → shellsim-0.1.10}/src/python/stdlib/mod.rs

@@ -4,6 +4,7 @@
 //! Keeping this directory self-contained makes each shim straightforward to review and test.
 
 pub mod argparse;
+mod asyncio;
 mod base64;
 pub mod bisect;
 pub mod collections;
@@ -57,6 +58,7 @@ pub(super) fn frozen_builtin(name: &str) -> Option<(&'static str, &'static str)>
 /// Resolve one capability-free module through the declarative native registry.
 pub(super) fn native_module(name: &str) -> Option<&'static ModuleDef> {
     match name {
+        "_asyncio" => Some(&asyncio::MODULE),
         "argparse" => Some(&argparse::MODULE),
         "_base64" => Some(&base64::MODULE),
         "_json" => Some(&json::MODULE),