retracesoftware-proxy 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

retracesoftware/proxy/record.py

@@ -1,133 +1,221 @@
 import retracesoftware.functional as functional
-import retracesoftware_utils as utils
+import retracesoftware.utils as utils
+import retracesoftware.stream as stream
 
-from retracesoftware.proxy.proxytype import dynamic_proxytype, superdict, is_descriptor, extending_proxytype, dynamic_int_proxytype, make_extensible
-from retracesoftware.proxy.gateway import gateway_pair
+from retracesoftware.proxy.proxytype import *
+# from retracesoftware.proxy.gateway import gateway_pair
 from retracesoftware.proxy.proxysystem import ProxySystem
+from retracesoftware.proxy.thread import write_thread_switch
+from retracesoftware.install.tracer import Tracer
+from retracesoftware.proxy.stubfactory import StubRef, ExtendedRef
 
 import sys
+import os
+import types
+import gc
 
-class ExtendedProxy:
-    __slots__ = []
+class Placeholder:
+    __slots__ = ['id', '__weakref__']
 
-class ProxyRef:
-    def __init__(self, module, name):
-        self.module = module
-        self.name = name
-
-    def resolve(self):
-        return getattr(sys.modules[self.module], self.name)
-
-class ProxySpec(ProxyRef):
-    def __init__(self, module, name, methods, members):
-        super().__init__(module, name)
-        self.methods = methods
-        self.members = members
-
-    def __str__(self):
-        return f'ProxySpec(module = {self.module}, name = {self.name}, methods = {self.methods}, members = {self.members})'
+    def __init__(self, id):
+        self.id = id
     
 def keys_where_value(pred, dict):
     for key,value in dict.items():
         if pred(value): yield key
 
+types_lookup = {v:k for k,v in types.__dict__.items() if isinstance(v, type)}
+
+def resolve(obj):
+    try:
+        return getattr(sys.modules[obj.__module__], obj.__name__)
+    except:
+        return None
+    
 def resolveable(obj):
     try:
         return getattr(sys.modules[obj.__module__], obj.__name__) is obj
     except:
         return False
+
+def resolveable_name(obj):
+    if obj in types_lookup:
+        return ('types', types_lookup[obj])
+    elif resolve(obj) is obj:
+        return (obj.__module__, obj.__name__)
+    else:
+        return None
+
+# when 
+class RecordProxySystem(ProxySystem):
     
-class ExtProxytype:
+    def bind(self, obj):
+        self.bindings[obj] = self.writer.handle(Placeholder(self.next_placeholder_id))
+        self.writer(self.bindings[obj])
+        self.next_placeholder_id += 1
+
+    def before_fork(self):
+        self.writer.close()
+        super().before_fork()
+        # self.writer.path = self.dynamic_path
+
+    def after_fork_in_child(self):
+        new_path = self.new_child_path(self.writer.path)
+        new_path.parent.mkdir()
+        self.writer.path = new_path
+        super().after_fork_in_child()
+
+    def after_fork_in_parent(self):
+        super().after_fork_in_parent()
+        self.thread_state.value = self.saved_thread_state
+        self.writer.reopen()
+
+    def __init__(self, thread_state, 
+                 immutable_types, 
+                 tracing_config,
+                 path):
+        
+        self.fork_counter = 0
+
+        self.getpid = thread_state.wrap(
+            desired_state = 'disabled', function = os.getpid)
+        
+        self.pid = self.getpid()
 
-    def __init__(self, gateway, extended_types, writer):
-        self.gateway = gateway
-        self.writer = writer
-        self.extended_types = extended_types
+        self.writer = stream.writer(path)
+        
+        w = self.writer.handle('TRACE')
+        def trace_writer(*args):
+            print(f'Trace: {args}')
+            w(*args)
 
-    def __call__(self, cls):
-        assert isinstance(cls, type)
+        self.extended_types = {}
+        self.bindings = utils.id_dict()
+        self.next_placeholder_id = 0
 
-        if cls in self.extended_types:
-            return functional.side_effect(lambda obj: utils.set_type(obj, self.extended_types[cls]))
-        else:
-            if resolveable(cls):
-                ref = self.writer.handle(ProxyRef(name = cls.__name__, module = cls.__module__))
-            else:
-                blacklist = ['__getattribute__']
-                descriptors = {k: v for k,v in superdict(cls).items() if k not in blacklist and is_descriptor(v) }
-
-                methods = [k for k, v in descriptors.items() if utils.is_method_descriptor(v)]
-                members = [k for k, v in descriptors.items() if not utils.is_method_descriptor(v)]
-
-                ref = self.writer.handle(ProxySpec(name = cls.__name__, 
-                                                module = cls.__module__,
-                                                methods = methods,
-                                                members = members))
+        serialize = functional.walker(self.bindings.get_else_key)
+        
+        thread_switch_monitor = \
+            utils.thread_switch_monitor(
+                functional.repeatedly(functional.sequence(utils.thread_id, self.writer)))
+
+        self.sync = lambda function: functional.firstof(thread_switch_monitor, functional.always(self.writer.handle('SYNC')), function)
             
-            proxytype = dynamic_proxytype(handler = self.gateway, cls = cls)
+        error = self.writer.handle('ERROR')
 
-            self.writer.type_serializer[proxytype] = functional.constantly(ref)
-            return proxytype
+        def write_error(cls, val, traceback):
+            error(cls, val)
         
+        self.set_thread_id = functional.partial(utils.set_thread_id, self.writer)
 
-# when 
-class RecordProxySystem(ProxySystem):
-    
-    def __init__(self, thread_state, immutable_types, tracer, writer):
-        super().__init__(thread_state = thread_state)
+        def watch(f): return functional.either(thread_switch_monitor, f)
 
-        self.immutable_types = immutable_types
+        tracer = Tracer(tracing_config, writer = trace_writer)
 
-        self.extended_types = {}
-        self.writer = writer
-        self.bind = writer.placeholder
+        self.wrap_int_to_ext = watch
+        
+        self.on_int_call = functional.mapargs(transform = serialize, function = self.writer.handle('CALL'))
+        
+        self.on_ext_result = functional.if_then_else(
+                                functional.isinstanceof(str), 
+                                self.writer.handle('RESULT'), 
+                                functional.sequence(serialize, self.writer))
+        
+        self.on_ext_error = write_error
 
-        # on_ext_result = writer.handle('RESULT')
-        on_ext_result = functional.if_then_else(
-            functional.isinstanceof(str), writer.handle('RESULT'), writer)
+        self.ext_apply = self.int_apply = functional.apply
 
-        def int_proxytype(gateway):
-            return lambda cls: dynamic_int_proxytype(handler = gateway, cls = cls, bind = self.bind)
+        super().__init__(thread_state = thread_state, tracer = tracer, immutable_types = immutable_types)
+
+    def dynamic_ext_proxytype(self, cls):
         
-        def ext_proxytype(gateway):
-            return ExtProxytype(gateway = gateway, writer = writer, extended_types = self.extended_types)
-    
-        error = writer.handle('ERROR')
+        proxytype = super().dynamic_ext_proxytype(cls)
 
-        def write_error(cls, val, traceback):
-            error(cls, val)
+        blacklist = ['__class__', '__dict__', '__module__', '__doc__']
 
-        self.ext_handler, self.int_handler = gateway_pair(
-            thread_state,
-            tracer,
-            immutable_types = immutable_types,
-            int_proxytype = int_proxytype,
-            ext_proxytype = ext_proxytype,
-            on_int_call = writer.handle('CALL'),
-            on_ext_result = functional.side_effect(on_ext_result),
-            on_ext_error = write_error)
-
-    def extend_type(self, base):
-
-        if base in self.extended_types:
-            return self.extended_types[base]
-
-        # self.tracer.log('proxy.ext.new.extended', f'{base.__module__}.{base.__name__}')
-
-        extended = extending_proxytype(
-            cls = base,
-            thread_state = self.thread_state, 
-            ext_handler = self.ext_handler,
-            int_handler = self.int_handler,
-            on_subclass_new = self.bind,
-            is_stub = False)
+        methods = []
+        members = []
+
+        for key,value in proxytype.__dict__.items():
+            if key not in blacklist:
+                if utils.is_method_descriptor(value):
+                    methods.append(key)
+                elif not key.startswith("__retrace"):
+                    members.append(key)
+
+        ref = self.writer.handle(StubRef(
+            name = cls.__name__,
+            module = cls.__module__,
+            methods = methods,
+            members = members))
         
-        ref = self.writer.handle(ProxyRef(name = base.__name__, module = base.__module__,))
-        self.writer.type_serializer[extended] = functional.constantly(ref)
+        # list(proxytype.__dict__.keys())
+
+        # if resolveable_name(cls):
+        #     module, name = resolveable_name(cls)
+        #     ref = self.writer.handle(StubRef(type = 'dynamic', name = name, module = module))
+        # else:
+        #     blacklist = ['__getattribute__']
+        #     descriptors = {k: v for k,v in superdict(cls).items() if k not in blacklist and is_descriptor(v) }
+
+        #     methods = [k for k, v in descriptors.items() if utils.is_method_descriptor(v)]
+        #     members = [k for k, v in descriptors.items() if not utils.is_method_descriptor(v)]
+
+        #     ref = self.writer.handle(ProxySpec(name = cls.__name__, 
+        #                                     module = cls.__module__,
+        #                                     methods = methods,
+        #                                     members = members))
+  
+        self.writer.type_serializer[proxytype] = functional.constantly(ref)
+
+        return proxytype
+
+    def ext_proxytype(self, cls):
+        assert isinstance(cls, type), f"record.proxytype requires a type but was passed: {cls}"
+
+        # resolved = resolve(cls)
+
+        proxytype = super().ext_proxytype(cls)
+
+        assert resolve(cls) is cls
+
+        # ref = self.writer.handle(StubRef(name = cls.__name__,
+        #                                  module = cls.__module__))
+
+        # if resolveable_name(cls):
+        #     module, name = resolveable_name(cls)
+        #     ref = self.writer.handle(StubRef(name = cls.__name__, module = cls.__module__))
+        # else:
+        #     blacklist = ['__getattribute__']
+        #     descriptors = {k: v for k,v in superdict(cls).items() if k not in blacklist and is_descriptor(v) }
+
+        #     methods = [k for k, v in descriptors.items() if utils.is_method_descriptor(v)]
+        #     members = [k for k, v in descriptors.items() if not utils.is_method_descriptor(v)]
+
+        #     ref = self.writer.handle(ProxySpec(name = cls.__name__, 
+        #                                     module = cls.__module__,
+        #                                     methods = methods,
+        #                                     members = members))
+  
+        # self.writer.type_serializer[proxytype] = functional.constantly(ref)
+        return proxytype
+
+
+    def extend_type(self, cls):
+
+        if cls in self.extended_types:
+            return self.extended_types[cls]
+
+        extended = super().extend_type(cls)
         
-        # make_extensible(cls = extended, handler = self.int_handler, on_new = self.writer.placeholder)
+        self.extended_types[cls] = extended 
 
-        self.immutable_types.add(extended)
-        self.extended_types[base] = extended 
+        ref = self.writer.handle(ExtendedRef(name = cls.__name__,
+                                             module = cls.__module__))
+
+        # for binding?
+        # self.writer(ref)
+
+        self.writer.type_serializer[extended] = functional.constantly(ref)
 
         return extended

retracesoftware/proxy/replay.py

@@ -1,43 +1,55 @@
 import retracesoftware.functional as functional
 import retracesoftware_utils as utils
+import retracesoftware.stream as stream
 
-from retracesoftware.proxy.proxytype import dynamic_proxytype, dynamic_int_proxytype, extending_proxytype, make_extensible, dynamic_stubtype, stubtype_from_spec, Stub
-from retracesoftware.proxy.gateway import gateway_pair
-
-from retracesoftware.proxy.record import ProxyRef, ProxySpec
-from retracesoftware.proxy.proxysystem import ProxySystem
+from retracesoftware.install.tracer import Tracer
+from retracesoftware.proxy.thread import per_thread_messages
+from retracesoftware.proxy.proxytype import *
+# from retracesoftware.proxy.gateway import gateway_pair
+from retracesoftware.proxy.record import StubRef, Placeholder
+from retracesoftware.proxy.proxysystem import ProxySystem, RetraceError
+from retracesoftware.proxy.stubfactory import StubFactory, StubFunction
 
 import os
-
+import weakref
+import traceback
 # we can have a dummy method descriptor, its has a __name__ and when called, returns the next element
 
 # for types, we can patch the __new__ method
 # do it from C and immutable types can be patched too
 # patch the tp_new pointer?
 
+class ReplayError(RetraceError):
+    pass
+
 class ReplayProxySystem(ProxySystem):
     
     def stubtype(self, cls):
+        print(f'In stubtype, class: {cls}')
+        assert not issubclass(cls, Proxy)
+
         return dynamic_proxytype(handler = self.ext_handler, cls = cls)
 
-    def stubtype_from_spec(self, spec):
-        print (f'FOOO!!! {spec}')
-        return stubtype_from_spec(
-            handler = self.ext_handler,
-            module = spec.module, 
-            name = spec.name,
-            methods = spec.methods,
-            members = spec.members)
+    def create_stub(self): return True
+
+    # def stubtype_from_spec(self, spec):
+    #     print (f'FOOO!!! {spec}')
+    #     return stubtype_from_spec(
+    #         handler = self.ext_handler,
+    #         module = spec.module, 
+    #         name = spec.name,
+    #         methods = spec.methods,
+    #         members = spec.members)
 
     @utils.striptraceback
     def next_result(self):
         while True:
-            next = self.reader()
+            next = self.messages()
 
             if next == 'CALL':
-                func = self.reader()
-                args = self.reader()
-                kwargs = self.reader()
+                func = self.messages()
+                args = self.messages()
+                kwargs = self.messages()
 
                 try:
                     func(*args, **kwargs)
@@ -45,66 +57,178 @@ class ReplayProxySystem(ProxySystem):
                     pass
 
             elif next == 'RESULT':
-                return self.reader()
+                return self.messages()
             elif next == 'ERROR':
-                err_type = self.reader()
-                err_value = self.reader()
+                err_type = self.messages()
+                err_value = self.messages()
                 utils.raise_exception(err_type, err_value)
             else:
-                assert not isinstance(next, str)
+                if type(next) is str: print('FOOBAR!!!!')
+                assert type(next) is not str
                 return next
 
-    def __init__(self, thread_state, immutable_types, tracer, reader):
+    def bind(self, obj):
+        read = self.messages()
+        
+        assert isinstance(read, Placeholder)
+
+        self.bindings[read] = obj
+
+    # def dynamic_path(self):
+    #     if self.getpid() != self.pid:
+    #         self.pid = self.getpid()
+    #         # ok we are in child, calculate new path
+    #         self.path = self.path / f'fork-{self.fork_counter}'
+    #         self.fork_counter = 0
+        
+    #     return self.path
+
+    def after_fork_in_child(self):
+        self.reader.path = self.new_child_path(self.reader.path)
+        super().after_fork_in_child()
+
+    # def dynamic_ext_proxytype(self, cls):
+    #     raise Exception('dynamic_ext_proxytype should not be called in replay')
+
+    def proxy_function(self, obj):
+        func = functional.repeatedly(self.next_result)
+        func.__name__ = obj.__name__
+
+        return super().proxy_function(func)
+
+    def __init__(self, 
+                 thread_state,
+                 immutable_types,
+                 tracing_config,
+                 path,
+                 fork_path = []):
+        
         # self.writer = writer
-        super().__init__(thread_state = thread_state)
-        self.immutable_types = immutable_types
+        # super().__init__(thread_state = thread_state)
+        reader = stream.reader(path)
+
+        self.bindings = utils.id_dict()
+        self.set_thread_id = utils.set_thread_id
+        self.fork_path = fork_path
+        deserialize = functional.walker(self.bindings.get_else_key)
+
+        self.messages = functional.sequence(per_thread_messages(reader), deserialize)
+
+        self.stub_factory = StubFactory(thread_state = thread_state, next_result = self.next_result)
+
+        # messages = reader
+
+        def readnext():
+            with thread_state.select('disabled'):
+                try:
+                    return self.messages()
+                except Exception as error:
+                    # print(f'Error reading stream: {error}')
+                    traceback.print_exc()
+
+                    os._exit(1)
+
+            # print(f'read: {obj}')
+            # return obj
+
+
+        # lookup = weakref.WeakKeyDictionary()
+        
+        # debug = debug_level(config)
+
+        # int_refs = {}
+            
+        def read_required(required):
+            obj = readnext()
+            if obj != required:
+                print(f'Replay: {required} Record: {obj}')
+                for i in range(5):
+                    print(readnext())
+
+                utils.sigtrap(None)
+                os._exit(1)
+                raise Exception(f'Expected: {required} but got: {obj}')
+
+        def trace_writer(name, *args):
+            with thread_state.select('disabled'):
+                print(f'Trace: {name} {args}')
+                
+                read_required('TRACE')
+                read_required(name)
+
+                for arg in args:
+                    read_required(arg)
+
+        # self.tracer = Tracer(tracing_config, writer = trace_writer)
+        # self.immutable_types = immutable_types
+
         self.reader = reader
-        self.bind = self.reader.supply
 
-        add_stubtype = functional.side_effect(immutable_types.add)
+        # def foo(cls):
+        #     print(cls)
+        #     assert isinstance(cls, type)
+        #     immutable_types.add(cls)
+
+        # add_stubtype = functional.side_effect(foo)
+        # add_stubtype = functional.side_effect(immutable_types.add)
+
+        # reader.type_deserializer[ProxyRef] = functional.sequence(lambda ref: ref.resolve(), self.stubtype, add_stubtype)
 
-        reader.type_deserializer[ProxyRef] = functional.sequence(lambda ref: ref.resolve(), self.stubtype, add_stubtype)
-        reader.type_deserializer[ProxySpec] = functional.sequence(self.stubtype_from_spec, add_stubtype)
+        reader.type_deserializer[StubRef] = self.stub_factory
+        # reader.type_deserializer[ProxySpec] = functional.sequence(self.stubtype_from_spec, add_stubtype)
 
         # on_ext_result = functional.if_then_else(
         #     functional.is_instanceof(str), writer.handle('RESULT'), writer)
 
-        def int_proxytype(gateway):
-            return lambda cls: dynamic_int_proxytype(handler = gateway, cls = cls, bind = self.bind)
+        # def int_proxytype(gateway):
+        #     return lambda cls: dynamic_int_proxytype(handler = gateway, cls = cls, bind = self.bind)
 
-        def is_stub_type(obj):
-            return functional.typeof(obj) is type and issubclass(obj, Stub)
+        # create_stubs = functional.walker(functional.when(is_stub_ref, lambda stub: stub.create()))
+        # create_stubs = functional.walker(functional.when(is_stub_type, lambda cls: cls()))
 
-        create_stubs = functional.walker(functional.when(is_stub_type, lambda cls: cls()))
+        # self.ext_apply = functional.repeatedly(functional.sequence(self.next_result, create_stubs))
+        # self.ext_apply = functional.repeatedly(self.next_result)
+        
+        def read_sync(): read_required('SYNC')
 
-        ext_apply = functional.repeatedly(functional.sequence(self.next_result, create_stubs))
-                
-        self.ext_handler, self.int_handler = gateway_pair(
-            thread_state,
-            tracer,
-            immutable_types = immutable_types,
-            ext_apply = ext_apply,
-            int_proxytype = int_proxytype,
-            ext_proxytype = functional.identity)
-
-    def extend_type(self, base):
+        self.sync = lambda function: utils.observer(on_call = functional.always(read_sync), function = function)
+    
+        super().__init__(thread_state = thread_state, 
+                         tracer = Tracer(tracing_config, writer = trace_writer), 
+                         immutable_types = immutable_types)
+
+        # super().__init__(
+        #     thread_state=thread_state, 
+        #     immutable_types= immutable_types,
+        #     tracer=self.tracer,
+        #     ext_apply = ext_apply)
+        
+        # self.ext_handler, self.int_handler = gateway_pair(
+        #     thread_state,
+        #     self.tracer,
+        #     immutable_types = immutable_types,
+        #     ext_apply = ext_apply,
+        #     int_proxytype = int_proxytype,
+        #     ext_proxytype = functional.identity)
+
+    # def extend_type(self, base):
         
-        # ok, how to provide __getattr__ style access, 
+    #     # ok, how to provide __getattr__ style access, 
 
-        extended = extending_proxytype(
-            cls = base,
-            thread_state = self.thread_state, 
-            int_handler = self.int_handler,
-            ext_handler = self.ext_handler,
-            on_subclass_new = self.bind,
-            is_stub = True)
+    #     extended = extending_proxytype(
+    #         cls = base,
+    #         thread_state = self.thread_state, 
+    #         int_handler = self.int_handler,
+    #         ext_handler = self.ext_handler,
+    #         on_subclass_new = self.bind,
+    #         is_stub = True)
 
-        self.immutable_types.add(extended)
-        # proxytype = extending_proxytype(base)
+    #     self.immutable_types.add(extended)
+    #     # proxytype = extending_proxytype(base)
 
-        # make_extensible(cls = extended, 
-        #                 int_handler = self.int_handler, 
-        #                 ext_handler = self.ext_handler,
-        #                 on_new = self.reader.supply)
+    #     # make_extensible(cls = extended, 
+    #     #                 int_handler = self.int_handler, 
+    #     #                 ext_handler = self.ext_handler,
+    #     #                 on_new = self.reader.supply)
 
-        return extended
+    #     return extended