shellsim 0.1.8__tar.gz → 0.1.10__tar.gz

{shellsim-0.1.8 → shellsim-0.1.10}/Cargo.lock

@@ -330,7 +330,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim"
-version = "0.1.8"
+version = "0.1.10"
 dependencies = [
  "base64",
  "crc32fast",

{shellsim-0.1.8 → shellsim-0.1.10}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shellsim"
-version = "0.1.8"
+version = "0.1.10"
 edition = "2021"
 description = "In-process BusyBox-like environment for RL rollouts and agent tests"
 license = "Apache-2.0"

{shellsim-0.1.8 → shellsim-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shellsim
-Version: 0.1.8
+Version: 0.1.10
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: MacOS :: MacOS X

{shellsim-0.1.8 → shellsim-0.1.10}/python/native/Cargo.lock

@@ -406,7 +406,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim"
-version = "0.1.8"
+version = "0.1.10"
 dependencies = [
  "base64",
  "crc32fast",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "shellsim-python-bindings"
-version = "0.1.8"
+version = "0.1.10"
 dependencies = [
  "pyo3",
  "serde",

{shellsim-0.1.8 → shellsim-0.1.10}/python/native/Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name = "shellsim-python-bindings"
-version = "0.1.8"
+version = "0.1.10"
 edition = "2021"
 description = "Python bindings for shellsim"
 license = "Apache-2.0"

{shellsim-0.1.8 → shellsim-0.1.10}/src/python/ast.rs

@@ -50,10 +50,17 @@ pub enum StatementKind {
         body: Vec<Statement>,
         otherwise: Vec<Statement>,
     },
+    AsyncFor {
+        target: AssignmentTarget,
+        iterable: Expression,
+        body: Vec<Statement>,
+        otherwise: Vec<Statement>,
+    },
     Function {
         name: String,
         parameters: Vec<Parameter>,
         body: Vec<Statement>,
+        is_async: bool,
     },
     Class {
         name: String,
@@ -91,6 +98,11 @@ pub enum StatementKind {
         target: Option<AssignmentTarget>,
         body: Vec<Statement>,
     },
+    AsyncWith {
+        context: Expression,
+        target: Option<AssignmentTarget>,
+        body: Vec<Statement>,
+    },
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -122,11 +134,13 @@ impl StatementKind {
             Self::If { .. }
                 | Self::While { .. }
                 | Self::For { .. }
+                | Self::AsyncFor { .. }
                 | Self::Function { .. }
                 | Self::Class { .. }
                 | Self::Decorated { .. }
                 | Self::Try { .. }
                 | Self::With { .. }
+                | Self::AsyncWith { .. }
         )
     }
 }
@@ -173,14 +187,14 @@ pub enum ExpressionKind {
         value: Box<Expression>,
         clauses: Vec<ComprehensionClause>,
     },
-    /// The VM deliberately materializes this as a list for now.  This gives
-    /// generator expressions an ordinary iterable and isolated comprehension
-    /// scope while keeping the bounded-resource contract simple and explicit.
+    /// A lazy comprehension evaluated in its own generator frame.
     GeneratorExpression {
         element: Box<Expression>,
         clauses: Vec<ComprehensionClause>,
     },
     Yield(Option<Box<Expression>>),
+    YieldFrom(Box<Expression>),
+    Await(Box<Expression>),
     Attribute {
         value: Box<Expression>,
         name: String,

{shellsim-0.1.8 → shellsim-0.1.10}/src/python/bytecode.rs

@@ -65,6 +65,7 @@ pub struct Code {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CallSignature {
     pub is_generator: bool,
+    pub is_coroutine: bool,
     pub positional_count: usize,
     pub variadic_slot: Option<usize>,
     pub default_slots: Box<[usize]>,
@@ -223,9 +224,12 @@ pub enum Opcode {
     Reraise,
     Raise(bool),
     Yield,
+    AwaitResult,
     WithEnter,
     WithExit,
     WithExitException,
+    AsyncWithExitException,
+    AsyncWithFinishException,
     PopExpression,
     Halt,
 }
@@ -321,9 +325,13 @@ pub enum Operation {
     Raise(bool),
     /// Suspend a generator frame and return the value on top of the stack.
     Yield,
+    /// Unwrap the scheduler outcome sent into a suspended coroutine.
+    AwaitResult,
     WithEnter,
     WithExit,
     WithExitException,
+    AsyncWithExitException,
+    AsyncWithFinishException,
     PopExpression,
     Halt,
 }
@@ -489,9 +497,12 @@ impl CodeBuilder {
             Operation::Reraise => Opcode::Reraise,
             Operation::Raise(cause) => Opcode::Raise(cause),
             Operation::Yield => Opcode::Yield,
+            Operation::AwaitResult => Opcode::AwaitResult,
             Operation::WithEnter => Opcode::WithEnter,
             Operation::WithExit => Opcode::WithExit,
             Operation::WithExitException => Opcode::WithExitException,
+            Operation::AsyncWithExitException => Opcode::AsyncWithExitException,
+            Operation::AsyncWithFinishException => Opcode::AsyncWithFinishException,
             Operation::PopExpression => Opcode::PopExpression,
             Operation::Halt => Opcode::Halt,
         }
@@ -503,6 +514,7 @@ impl CodeBuilder {
         spans: Vec<Span>,
         parameters: Vec<Parameter>,
         local_names: Vec<String>,
+        is_coroutine: bool,
     ) -> CodeRef {
         let positional_count = parameters
             .iter()
@@ -512,6 +524,7 @@ impl CodeBuilder {
             is_generator: instructions
                 .iter()
                 .any(|instruction| matches!(instruction.opcode, Opcode::Yield)),
+            is_coroutine,
             positional_count,
             variadic_slot: parameters.iter().position(|parameter| parameter.variadic),
             default_slots: parameters

{shellsim-0.1.8 → shellsim-0.1.10}/src/python/compiler.rs

@@ -15,6 +15,7 @@ pub fn compile(program: Program) -> CodeRef {
         finalizers: Vec::new(),
         protected_regions: Vec::new(),
         in_function: false,
+        is_coroutine: false,
         globals: HashSet::new(),
         nonlocals: HashSet::new(),
         named_expression: NamedExpressionContext::local(),
@@ -38,6 +39,7 @@ struct Compiler {
     /// at compile time rather than losing that state in a suspended generator.
     protected_regions: Vec<&'static str>,
     in_function: bool,
+    is_coroutine: bool,
     globals: HashSet<String>,
     nonlocals: HashSet<String>,
     named_expression: NamedExpressionContext,
@@ -155,7 +157,7 @@ impl Compiler {
             });
             spans.push(instruction.span);
         }
-        builder.finish(bytecode, spans, parameters, local_names)
+        builder.finish(bytecode, spans, parameters, local_names, self.is_coroutine)
     }
 
     fn statements(&mut self, statements: Vec<Statement>) {
@@ -320,10 +322,70 @@ impl Compiler {
                 let finished = self.instructions.len();
                 self.finish_loop(finished);
             }
+            StatementKind::AsyncFor {
+                target,
+                iterable,
+                body,
+                otherwise,
+            } => {
+                let iterator_name =
+                    format!("$__shellsim_async_iterator_{}", self.instructions.len());
+                self.expression(iterable);
+                self.emit(Operation::LoadAttribute("__aiter__".into()), span);
+                self.emit(
+                    Operation::Call {
+                        positional: 0,
+                        keywords: Vec::new(),
+                        starred: Vec::new(),
+                    },
+                    span,
+                );
+                self.emit(Operation::StoreName(iterator_name.clone()), span);
+                let next = self.instructions.len();
+                let begin = self.emit(Operation::TryBegin(usize::MAX), span);
+                self.emit(Operation::LoadName(iterator_name), span);
+                self.emit(Operation::LoadAttribute("__anext__".into()), span);
+                self.emit(
+                    Operation::Call {
+                        positional: 0,
+                        keywords: Vec::new(),
+                        starred: Vec::new(),
+                    },
+                    span,
+                );
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                self.emit(Operation::TryEnd, span);
+                self.store_target(target, span);
+                self.loops.push(LoopContext {
+                    continue_target: next,
+                    iterator_on_stack: false,
+                    finalizer_depth: self.finalizers.len(),
+                    breaks: Vec::new(),
+                });
+                self.statements(body);
+                self.emit(Operation::Jump(next), span);
+                let handler = self.instructions.len();
+                self.patch_try_begin(begin, handler);
+                self.emit(Operation::LoadName("StopAsyncIteration".into()), span);
+                self.emit(Operation::MatchException { typed: true }, span);
+                let rethrow = self.emit(Operation::PopJumpIfFalse(usize::MAX), span);
+                self.emit(Operation::PopTop, span);
+                self.emit(Operation::ClearException, span);
+                self.statements(otherwise);
+                let done = self.emit(Operation::Jump(usize::MAX), span);
+                self.patch_jump(rethrow, self.instructions.len());
+                self.emit(Operation::PopTop, span);
+                self.emit(Operation::Reraise, span);
+                let finished = self.instructions.len();
+                self.patch_jump(done, finished);
+                self.finish_loop(finished);
+            }
             StatementKind::Function {
                 name,
                 parameters,
                 body,
+                is_async,
             } => {
                 let defaults = parameters
                     .iter()
@@ -338,6 +400,7 @@ impl Compiler {
                     finalizers: Vec::new(),
                     protected_regions: Vec::new(),
                     in_function: true,
+                    is_coroutine: is_async,
                     globals: HashSet::new(),
                     nonlocals: HashSet::new(),
                     named_expression: NamedExpressionContext::local(),
@@ -391,6 +454,7 @@ impl Compiler {
                     finalizers: Vec::new(),
                     protected_regions: Vec::new(),
                     in_function: false,
+                    is_coroutine: false,
                     globals: HashSet::new(),
                     nonlocals: HashSet::new(),
                     named_expression: NamedExpressionContext::local(),
@@ -660,6 +724,60 @@ impl Compiler {
                 self.emit(Operation::WithExitException, span);
                 self.patch_jump(done, self.instructions.len());
             }
+            StatementKind::AsyncWith {
+                context,
+                target,
+                body,
+            } => {
+                let context_name = format!("$__shellsim_async_context_{}", self.instructions.len());
+                self.expression(context);
+                self.emit(Operation::StoreName(context_name.clone()), span);
+                self.emit(Operation::LoadName(context_name.clone()), span);
+                self.emit(Operation::LoadAttribute("__aenter__".into()), span);
+                self.emit(
+                    Operation::Call {
+                        positional: 0,
+                        keywords: Vec::new(),
+                        starred: Vec::new(),
+                    },
+                    span,
+                );
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                if let Some(target) = target {
+                    self.store_target(target, span);
+                } else {
+                    self.emit(Operation::PopTop, span);
+                }
+                let begin = self.emit(Operation::TryBegin(usize::MAX), span);
+                self.statements(body);
+                self.emit(Operation::TryEnd, span);
+                self.emit(Operation::LoadName(context_name.clone()), span);
+                self.emit(Operation::LoadAttribute("__aexit__".into()), span);
+                for _ in 0..3 {
+                    self.emit(Operation::LoadConstant(Constant::None), span);
+                }
+                self.emit(
+                    Operation::Call {
+                        positional: 3,
+                        keywords: Vec::new(),
+                        starred: vec![false; 3],
+                    },
+                    span,
+                );
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                self.emit(Operation::PopTop, span);
+                let done = self.emit(Operation::Jump(usize::MAX), span);
+                let handler = self.instructions.len();
+                self.patch_try_begin(begin, handler);
+                self.emit(Operation::LoadName(context_name), span);
+                self.emit(Operation::AsyncWithExitException, span);
+                self.emit(Operation::Yield, span);
+                self.emit(Operation::AwaitResult, span);
+                self.emit(Operation::AsyncWithFinishException, span);
+                self.patch_jump(done, self.instructions.len());
+            }
         }
     }
 
@@ -936,10 +1054,7 @@ impl Compiler {
                 self.emit_dict_comprehension(*key, *value, clauses, span);
             }
             ExpressionKind::GeneratorExpression { element, clauses } => {
-                // The first implementation intentionally materializes generator expressions.
-                // It is still a separate function scope (like CPython's generator frame), and
-                // all iteration remains metered by the ordinary bytecode loop.
-                self.emit_comprehension(ComprehensionKind::List, *element, clauses, span);
+                self.emit_generator_expression(*element, clauses, span);
             }
             ExpressionKind::Yield(value) => {
                 if !self.in_function {
@@ -947,11 +1062,6 @@ impl Compiler {
                         Operation::RuntimeError("'yield' outside function".into()),
                         span,
                     );
-                } else if let Some(region) = self.protected_regions.last() {
-                    self.emit(
-                        Operation::RuntimeError(format!("yield is not supported inside a {region} cleanup region; suspending here would lose exception or context-manager state")),
-                        span,
-                    );
                 } else {
                     if let Some(value) = value {
                         self.expression(*value);
@@ -961,6 +1071,36 @@ impl Compiler {
                     self.emit(Operation::Yield, span);
                 }
             }
+            ExpressionKind::YieldFrom(value) => {
+                if !self.in_function {
+                    self.emit(
+                        Operation::RuntimeError("'yield from' outside function".into()),
+                        span,
+                    );
+                } else {
+                    self.expression(*value);
+                    self.emit(Operation::GetIterator, span);
+                    let next = self.emit(Operation::ForIterator(usize::MAX), span);
+                    self.emit(Operation::Yield, span);
+                    self.emit(Operation::PopTop, span);
+                    self.emit(Operation::Jump(next), span);
+                    let exhausted = self.instructions.len();
+                    self.patch_jump(next, exhausted);
+                    self.emit(Operation::LoadConstant(Constant::None), span);
+                }
+            }
+            ExpressionKind::Await(value) => {
+                if !self.is_coroutine {
+                    self.emit(
+                        Operation::RuntimeError("'await' outside async function".into()),
+                        span,
+                    );
+                } else {
+                    self.expression(*value);
+                    self.emit(Operation::Yield, span);
+                    self.emit(Operation::AwaitResult, span);
+                }
+            }
             ExpressionKind::Attribute { value, name } => {
                 self.expression(*value);
                 self.emit(Operation::LoadAttribute(name), span);
@@ -1032,6 +1172,7 @@ impl Compiler {
                     finalizers: Vec::new(),
                     protected_regions: Vec::new(),
                     in_function: true,
+                    is_coroutine: false,
                     globals: HashSet::new(),
                     nonlocals: HashSet::new(),
                     named_expression: NamedExpressionContext::local(),
@@ -1180,6 +1321,7 @@ impl Compiler {
             finalizers: Vec::new(),
             protected_regions: Vec::new(),
             in_function: true,
+            is_coroutine: false,
             globals: HashSet::new(),
             nonlocals: HashSet::new(),
             named_expression,
@@ -1216,6 +1358,47 @@ impl Compiler {
         );
     }
 
+    fn emit_generator_expression(
+        &mut self,
+        element: Expression,
+        clauses: Vec<ComprehensionClause>,
+        span: Span,
+    ) {
+        let named_expression = self.comprehension_named_expression_context();
+        let mut nested = Compiler {
+            instructions: Vec::new(),
+            loops: Vec::new(),
+            finalizers: Vec::new(),
+            protected_regions: Vec::new(),
+            in_function: true,
+            is_coroutine: false,
+            globals: HashSet::new(),
+            nonlocals: HashSet::new(),
+            named_expression,
+            is_class_scope: false,
+            structural_depth: self.structural_depth,
+        };
+        nested.emit_generator_comprehension_body(&clauses, 0, &element, span);
+        nested.emit(Operation::LoadConstant(Constant::None), span);
+        nested.emit(Operation::Return, span);
+        self.emit(
+            Operation::MakeFunction {
+                name: "<genexpr>".into(),
+                code: nested.finish(Vec::new()),
+                defaults: 0,
+            },
+            span,
+        );
+        self.emit(
+            Operation::Call {
+                positional: 0,
+                keywords: Vec::new(),
+                starred: Vec::new(),
+            },
+            span,
+        );
+    }
+
     fn emit_dict_comprehension(
         &mut self,
         key: Expression,
@@ -1230,6 +1413,7 @@ impl Compiler {
             finalizers: Vec::new(),
             protected_regions: Vec::new(),
             in_function: true,
+            is_coroutine: false,
             globals: HashSet::new(),
             nonlocals: HashSet::new(),
             named_expression,
@@ -1307,6 +1491,35 @@ impl Compiler {
         self.patch_jump(next, exhausted);
     }
 
+    fn emit_generator_comprehension_body(
+        &mut self,
+        clauses: &[ComprehensionClause],
+        index: usize,
+        element: &Expression,
+        span: Span,
+    ) {
+        let clause = &clauses[index];
+        self.expression(clause.iterable.clone());
+        self.emit(Operation::GetIterator, span);
+        let next = self.emit(Operation::ForIterator(usize::MAX), span);
+        self.store_target(clause.target.clone(), span);
+        for condition in &clause.conditions {
+            self.expression(condition.clone());
+            let skip = self.emit(Operation::PopJumpIfFalse(usize::MAX), span);
+            self.patch_jump(skip, next);
+        }
+        if index + 1 < clauses.len() {
+            self.emit_generator_comprehension_body(clauses, index + 1, element, span);
+        } else {
+            self.expression(element.clone());
+            self.emit(Operation::Yield, span);
+            self.emit(Operation::PopTop, span);
+        }
+        self.emit(Operation::Jump(next), span);
+        let exhausted = self.instructions.len();
+        self.patch_jump(next, exhausted);
+    }
+
     fn emit_dict_comprehension_body(
         &mut self,
         clauses: &[ComprehensionClause],

{shellsim-0.1.8 → shellsim-0.1.10}/src/python/heap.rs

@@ -212,6 +212,7 @@ pub enum Object {
         stack: Vec<Value>,
         exhausted: bool,
         running: bool,
+        return_value: Value,
     },
     Module {
         name: String,
@@ -1430,9 +1431,15 @@ fn trace_object(
             trace_value(*callable, object_work);
             trace_value(*sentinel, object_work);
         }
-        Object::Generator { scope, stack, .. } => {
+        Object::Generator {
+            scope,
+            stack,
+            return_value,
+            ..
+        } => {
             scope_work.push(*scope);
             trace_values(stack.iter().copied(), object_work);
+            trace_value(*return_value, object_work);
         }
         Object::Module { scope, .. } => scope_work.push(*scope),
         Object::Array { storage, .. } => object_work.push(*storage),

{shellsim-0.1.8 → shellsim-0.1.10}/src/python/lexer.rs

@@ -280,6 +280,8 @@ impl<'a> Lexer<'a> {
             "raise" => TokenKind::Raise,
             "with" => TokenKind::With,
             "yield" => TokenKind::Yield,
+            "async" => TokenKind::Async,
+            "await" => TokenKind::Await,
             name => TokenKind::Name((*name).to_string()),
         }
     }