retracesoftware-proxy 0.1.5__py3-none-any.whl → 0.2.4__py3-none-any.whl

retracesoftware/proxy/replay.py

@@ -1,86 +1,20 @@
 import retracesoftware.functional as functional
 import retracesoftware_utils as utils
-import retracesoftware.stream as stream
 
 from retracesoftware.install.tracer import Tracer
-from retracesoftware.proxy.thread import per_thread_messages
+from retracesoftware.proxy.thread import per_thread_messages, thread_id
+from retracesoftware.proxy.messagestream import *
 from retracesoftware.proxy.proxytype import *
 # from retracesoftware.proxy.gateway import gateway_pair
-from retracesoftware.proxy.record import StubRef, Placeholder
-from retracesoftware.proxy.proxysystem import ProxySystem, RetraceError
-from retracesoftware.proxy.stubfactory import StubFactory, StubFunction
+from retracesoftware.proxy.record import StubRef
+from retracesoftware.proxy.proxysystem import ProxySystem
+from retracesoftware.proxy.stubfactory import StubFactory
+from retracesoftware.proxy.globalref import GlobalRef
 
 import os
-import weakref
-import traceback
-# we can have a dummy method descriptor, its has a __name__ and when called, returns the next element
-
-# for types, we can patch the __new__ method
-# do it from C and immutable types can be patched too
-# patch the tp_new pointer?
-
-class ReplayError(RetraceError):
-    pass
 
 class ReplayProxySystem(ProxySystem):
     
-    def stubtype(self, cls):
-        assert not issubclass(cls, Proxy)
-
-        return dynamic_proxytype(handler = self.ext_handler, cls = cls)
-
-    def create_stub(self): return True
-
-    # def stubtype_from_spec(self, spec):
-    #     print (f'FOOO!!! {spec}')
-    #     return stubtype_from_spec(
-    #         handler = self.ext_handler,
-    #         module = spec.module, 
-    #         name = spec.name,
-    #         methods = spec.methods,
-    #         members = spec.members)
-
-    @utils.striptraceback
-    def next_result(self):
-        while True:
-            next = self.messages()
-
-            if next == 'CALL':
-                func = self.messages()
-                args = self.messages()
-                kwargs = self.messages()
-
-                try:
-                    func(*args, **kwargs)
-                except:
-                    pass
-
-            elif next == 'RESULT':
-                return self.messages()
-            elif next == 'ERROR':
-                err_type = self.messages()
-                err_value = self.messages()
-                utils.raise_exception(err_type, err_value)
-            else:
-                assert type(next) is not str
-                return next
-
-    def bind(self, obj):
-        read = self.messages()
-        
-        assert isinstance(read, Placeholder)
-
-        self.bindings[read] = obj
-
-    # def dynamic_path(self):
-    #     if self.getpid() != self.pid:
-    #         self.pid = self.getpid()
-    #         # ok we are in child, calculate new path
-    #         self.path = self.path / f'fork-{self.fork_counter}'
-    #         self.fork_counter = 0
-        
-    #     return self.path
-
     def after_fork_in_child(self):
         self.reader.path = self.new_child_path(self.reader.path)
         super().after_fork_in_child()
@@ -88,145 +22,117 @@ class ReplayProxySystem(ProxySystem):
     # def dynamic_ext_proxytype(self, cls):
     #     raise Exception('dynamic_ext_proxytype should not be called in replay')
 
-    def proxy_function(self, obj):
-        func = functional.repeatedly(self.next_result)
-        func.__name__ = obj.__name__
-
-        return super().proxy_function(func)
+    @property
+    def ext_apply(self): 
+        return functional.repeatedly(self.next_result)
 
-    def __init__(self, 
-                 thread_state,
-                 immutable_types,
-                 tracing_config,
-                 path,
-                 fork_path = []):
-        
-        # self.writer = writer
-        # super().__init__(thread_state = thread_state)
-        reader = stream.reader(path)
-
-        self.bindings = utils.id_dict()
-        self.set_thread_id = utils.set_thread_id
-        self.fork_path = fork_path
-        deserialize = functional.walker(self.bindings.get_else_key)
-
-        self.messages = functional.sequence(per_thread_messages(reader), deserialize)
+    def proxy__new__(self, __new__, *args, **kwargs):
+        func = functional.repeatedly(self.next_result)
+        func.__name__ = '__new__'
+        return super().proxy__new__(func, *args, **kwargs)
 
-        self.stub_factory = StubFactory(thread_state = thread_state, next_result = self.next_result)
+    def basetype(self, cls):
+        return self.stub_factory.create_stubtype(StubRef(cls))
 
-        # messages = reader
+    def trace_writer(self, name, *args):
+        with self.thread_state.select('disabled'):
+            # read = self.messages_read
 
-        def readnext():
-            with thread_state.select('disabled'):
-                try:
-                    return self.messages()
-                except Exception as error:
-                    # print(f'Error reading stream: {error}')
-                    traceback.print_exc()
+            self.read_required('TRACE')
+            # self.read_required(read)
+            self.read_required(name)
 
+            if name == 'stacktrace':
+                print('FOOO!!!')
+                os._exit(1)
+                record = self.readnext()
+                if args[0] == record:
+                    self.last_matching_stack = args[0]
+                else:
+                    on_stack_mismatch(
+                        last_matching = self.last_matching_stack,
+                        record = record,
+                        replay = args[0])
                     os._exit(1)
+            else:
+                # print(f'Trace: {self.reader.messages_read} {name} {args}')
+                for arg in args:
+                    self.read_required(arg)
 
-            # print(f'read: {obj}')
-            # return obj
-
+    def on_thread_exit(self, thread_id):
+        # print(f'on_thread_exit!!!!')
+        self.reader.wake_pending()
 
-        # lookup = weakref.WeakKeyDictionary()
-        
-        # debug = debug_level(config)
-
-        # int_refs = {}
-            
-        def read_required(required):
-            obj = readnext()
-            if obj != required:
-                print(f'Replay: {required} Record: {obj}')
-                for i in range(5):
-                    print(readnext())
-
-                utils.sigtrap(None)
-                os._exit(1)
-                raise Exception(f'Expected: {required} but got: {obj}')
+    def __init__(self, 
+                 reader,
+                 thread_state,
+                 immutable_types,
+                 tracing_config,
+                 traceargs,
+                 verbose = False,
+                 fork_path = [],
+                 skip_weakref_callbacks = False):
 
-        def trace_writer(name, *args):
-            with thread_state.select('disabled'):
-                print(f'Trace: {name} {args}')
-                
-                read_required('TRACE')
-                read_required(name)
+        self.reader = reader
+        # self.skip_weakref_callbacks = skip_weakref_callbacks
 
-                for arg in args:
-                    read_required(arg)
+        self.fork_path = fork_path
 
-        # self.tracer = Tracer(tracing_config, writer = trace_writer)
-        # self.immutable_types = immutable_types
+        self.messages = MessageStream(
+            thread_state = thread_state,
+            source = reader, 
+            skip_weakref_callbacks = skip_weakref_callbacks,
+            verbose = verbose)
 
-        self.reader = reader
+        self.checkpoint = self.messages.checkpoint
+        self.bind = self.messages.bind
+        self.next_result = self.messages.result
+        self.exclude_from_stacktrace = self.messages.excludes.add
 
-        # def foo(cls):
-        #     print(cls)
-        #     assert isinstance(cls, type)
-        #     immutable_types.add(cls)
+        self.stub_factory = StubFactory(thread_state = thread_state, next_result = self.next_result)
 
-        # add_stubtype = functional.side_effect(foo)
-        # add_stubtype = functional.side_effect(immutable_types.add)
+        self.last_matching_stack = None
 
-        # reader.type_deserializer[ProxyRef] = functional.sequence(lambda ref: ref.resolve(), self.stubtype, add_stubtype)
+        def run_ref(ref):
+            print(f'run_ref!!!! {ref}')
+            return ref()
 
-        reader.type_deserializer[StubRef] = self.stub_factory
-        # reader.type_deserializer[ProxySpec] = functional.sequence(self.stubtype_from_spec, add_stubtype)
+        self.reader.type_deserializer[StubRef] = self.stub_factory
+        # self.reader.type_deserializer[GlobalRef] = lambda ref: ref()
+        self.reader.type_deserializer[GlobalRef] = run_ref
 
-        # on_ext_result = functional.if_then_else(
-        #     functional.is_instanceof(str), writer.handle('RESULT'), writer)
+        excludes = [ReplayProxySystem.trace_writer]
+        
+        for exclude in excludes:
+            self.messages.excludes.add(exclude)
 
-        # def int_proxytype(gateway):
-        #     return lambda cls: dynamic_int_proxytype(handler = gateway, cls = cls, bind = self.bind)
+        sync = functional.lazy(self.messages.read_required, 'SYNC')
 
-        # create_stubs = functional.walker(functional.when(is_stub_ref, lambda stub: stub.create()))
-        # create_stubs = functional.walker(functional.when(is_stub_type, lambda cls: cls()))
+        read_sync = thread_state.dispatch(utils.noop, internal = sync)
 
-        # self.ext_apply = functional.repeatedly(functional.sequence(self.next_result, create_stubs))
-        # self.ext_apply = functional.repeatedly(self.next_result)
-        
-        def read_sync(): read_required('SYNC')
+        self.on_ext_call = sync
 
-        self.sync = lambda function: utils.observer(on_call = functional.always(read_sync), function = function)
-    
-        super().__init__(thread_state = thread_state, 
-                         tracer = Tracer(tracing_config, writer = trace_writer), 
-                         immutable_types = immutable_types)
-
-        # super().__init__(
-        #     thread_state=thread_state, 
-        #     immutable_types= immutable_types,
-        #     tracer=self.tracer,
-        #     ext_apply = ext_apply)
+        self.sync = lambda function: utils.observer(on_call = read_sync, function = function)
         
-        # self.ext_handler, self.int_handler = gateway_pair(
-        #     thread_state,
-        #     self.tracer,
-        #     immutable_types = immutable_types,
-        #     ext_apply = ext_apply,
-        #     int_proxytype = int_proxytype,
-        #     ext_proxytype = functional.identity)
-
-    # def extend_type(self, base):
-        
-    #     # ok, how to provide __getattr__ style access, 
+        self.create_from_external = utils.noop
 
-    #     extended = extending_proxytype(
-    #         cls = base,
-    #         thread_state = self.thread_state, 
-    #         int_handler = self.int_handler,
-    #         ext_handler = self.ext_handler,
-    #         on_subclass_new = self.bind,
-    #         is_stub = True)
+        if skip_weakref_callbacks:
+            self.wrap_weakref_callback = \
+                lambda callback: \
+                    thread_state.dispatch(
+                        callback, internal = self.disable_for(callback))
+        else:
+            self.on_weakref_callback_start = functional.lazy(self.messages.read_required, 'ON_WEAKREF_CALLBACK_START')
+            self.on_weakref_callback_end = functional.lazy(self.messages.read_required, 'ON_WEAKREF_CALLBACK_END')
 
-    #     self.immutable_types.add(extended)
-    #     # proxytype = extending_proxytype(base)
+        super().__init__(thread_state = thread_state, 
+                         tracer = Tracer(tracing_config, writer = self.trace_writer),
+                         immutable_types = immutable_types,
+                         traceargs = traceargs)
 
-    #     # make_extensible(cls = extended, 
-    #     #                 int_handler = self.int_handler, 
-    #     #                 ext_handler = self.ext_handler,
-    #     #                 on_new = self.reader.supply)
+    def write_trace(self, obj):
+        if 'TRACER' != self.messages():
+            utils.sigtrap(obj)
 
-    #     return extended
+        # self.read_required       ('TRACER')
+        self.read_required(obj)

retracesoftware/proxy/serializer.py

@@ -0,0 +1,28 @@
+def serializer(obj):
+    cls = type(obj)
+
+    try:
+        if issubclass(cls, Enum):
+            return str(obj)
+
+        if issubclass(cls, (tuple, list)):
+            return list(map(serializer, obj))
+        elif issubclass(cls, dict):
+            return {str(serializer(k)):serializer(v) for k,v in obj.items()}
+        elif issubclass(cls, int):
+            if obj > 1000000 or obj < -1000000:
+                return "XXXX"
+            else:
+                return int(obj)
+        elif issubclass(cls, (bool, str, types.NoneType)):
+            return obj
+        elif issubclass(cls, types.FunctionType):
+            return obj.__qualname__
+        elif issubclass(cls, types.CellType):
+            return getattr(obj, '__qualname__', 'CellType')
+        elif issubclass(cls, types.GeneratorType):
+            return obj.__qualname__
+        else:
+            return cleanse(str(obj))
+    except:
+        return f'Unserializable object of type: {cls}'

retracesoftware/proxy/startthread.py

@@ -0,0 +1,40 @@
+from retracesoftware import functional
+from retracesoftware import utils
+
+import _thread
+
+# _thread.start_new_thread(function, args[, kwargs])
+
+# push a per thread executor function, as a context manager
+
+counters = _thread._local()
+counters.id = ()
+counters.counter = 0
+
+def with_thread_id(thread_id, function, *args, **kwargs):
+    counters.id = thread_id
+    counters.counter = 0
+    return function(*args, **kwargs)
+
+thread_id = functional.lazy(getattr, counters, 'id')
+
+def start_new_thread(original, wrapper, function, *args):
+    return original(wrapper(function), *args)
+
+def wrap_thread_function(thread_state, function):
+    if thread_state.value == 'internal':
+        next_id = counters.id + (counters.counter,)
+        counters.counter += 1
+        return functional.partial(with_thread_id, next_id, thread_state.wrap('internal', function))
+    else:
+        return function
+
+def patch_thread_start(thread_state):
+
+    wrapper = functional.partial(wrap_thread_function, thread_state)
+
+    _thread.start_new = functional.partial(start_new_thread, _thread.start_new, wrapper)
+    _thread.start_new_thread = functional.partial(start_new_thread, _thread.start_new_thread, wrapper)
+
+    import threading
+    threading._start_new_thread = _thread.start_new_thread

retracesoftware/proxy/stubfactory.py

@@ -13,14 +13,33 @@ class ExtendedRef:
         self.module = module
 
 class StubRef:
-    def __init__(self, module, name, methods, members):
-        self.name = name
-        self.module = module
-        self.methods = methods
-        self.members = members
+    def __init__(self, cls):
+        blacklist = ['__class__', '__dict__', '__module__', '__doc__', '__new__']
+
+        self.methods = []
+        self.static_methods = []
+        self.class_methods = []
+
+        for key,value in cls.__dict__.items():
+            if key not in blacklist:
+                if isinstance(value, classmethod):
+                    self.class_methods.append(key)
+                elif isinstance(value, staticmethod):
+                    self.class_methods.append(key)
+                elif utils.is_method_descriptor(value):
+                    self.methods.append(key)
+
+        self.name = cls.__name__ 
+        self.module = cls.__module__
+
+    # def __init__(self, module, name, methods, members):
+    #     self.name = name
+    #     self.module = module
+    #     self.methods = methods
+    #     self.members = members
 
     def __str__(self):
-        return f'StubRef(module = {self.module}, name = {self.name}, methods = {self.methods}, members = {self.members})'
+        return f'StubRef(module = {self.module}, name = {self.name}, methods = {self.methods})'
 
 def resolve(module, name):
     try:
@@ -33,6 +52,11 @@ class StubMethodDescriptor(functional.repeatedly):
         super().__init__(next_result)
         self.__name__ = name
 
+    # @utils.striptraceback
+    # def __call__(self, *args, **kwargs):
+    #     super().__call__(*args, **kwargs)
+
+    #@utils.striptraceback
     def __str__(self):
         return f"stub - {__name__}"
     
@@ -41,29 +65,25 @@ class StubMemberDescriptor:
         self.next_result = next_result
         self.__name__ = name
 
+    #@utils.striptraceback
     def __get__(self, instance, owner):
         if instance is None:
             return self
         
         return self.next_result()
 
+    #@utils.striptraceback
     def __set__(self, instance, value):
         return self.next_result()
 
+    #@utils.striptraceback
     def __delete__(self, instance):
         return self.next_result()
-
+ 
+    #@utils.striptraceback
     def __str__(self):
         return f"stub member - {__name__}"
 
-class StubFunction(functional.repeatedly):
-    def __init__(self, name, next_result):
-        super().__init__(next_result)
-        self.__name__ = name
-
-    def __str__(self):
-        return f"stub function - {__name__}"
-
 class StubFactory:
 
     __slots__ = ['next_result', 'thread_state', 'cache']
@@ -85,16 +105,21 @@ class StubFactory:
         next_result = self.thread_state.dispatch(disabled, external = self.next_result)
 
         return StubMemberDescriptor(name = name, next_result = next_result)
-
+ 
     def create_method(self, name):
 
         def disabled(*args, **kwargs):
             if self.thread_state.value == 'disabled' and name == '__repr__':
                 return f"stub - {name}"
             else:
-                print(f'Error trying to call descriptor: {name} {args} {kwargs}, retrace mode: {self.thread_state.value}')
-                utils.sigtrap(None)
-                os._exit(1)
+                return None
+                # print(f'Error trying to call descriptor: {name} {args} {kwargs}, retrace mode: {self.thread_state.value}')
+
+                # import traceback
+                # traceback.print_stack()
+                
+                # utils.sigtrap(None)
+                # os._exit(1)
             
         next_result = self.thread_state.dispatch(disabled, external = self.next_result)
 
@@ -105,36 +130,66 @@ class StubFactory:
 
     def create_stubtype(self, spec):
 
+        assert self.thread_state.value == 'disabled'
+
         slots = {
             '__module__': spec.module,
             '__qualname__': spec.name,
+            '__name__': spec.name,
         }
 
         for method in spec.methods:
             slots[method] = self.create_method(method)
             assert utils.is_method_descriptor(slots[method])
 
-        for member in spec.members:
-            slots[member] = self.create_member(member)
+        def on_disabled(name):
+            print(f'Error trying to get/set attribute: {name}, when retrace mode: {self.thread_state.value} was not external')
+            utils.sigtrap(None)
+            os._exit(1)
+
+        #@utils.striptraceback
+        def getattr(instance, name):
+            print('In stub getattr!!!')
+            if self.thread_state.value == 'external':
+                return self.next_result()
+            else:
+                print(f'Error trying to get attribute: {name}, when retrace mode: {self.thread_state.value} was not external')
+                utils.sigtrap(None)
+                os._exit(1)
+
+        #@utils.striptraceback
+        def setattr(instance, name, value):
+            if self.thread_state.value == 'external':
+                return self.next_result()
+            else:
+                print(f'Error trying to set attribute: {name}, to: {value} when retrace mode: {self.thread_state.value} was not external')
+                utils.sigtrap(None)
+                os._exit(1)
+
+        slots['__getattr__'] = self.thread_state.method_dispatch(on_disabled, external = self.next_result)
+        # slots['__getattr__'] = getattr
+        slots['__setattr__'] = self.thread_state.method_dispatch(on_disabled, external = self.next_result)
 
         resolved = resolve(spec.module, spec.name)
 
-        if isinstance(resolved, type):
-            slots['__class__'] = property(functional.repeatedly(resolved))
+        # if isinstance(resolved, type):
+        #     slots['__class__'] = property(functional.repeatedly(resolved))
+
+        # else:
+        #     utils.sigtrap(f'{spec.module}.{spec.name}')
 
         stubtype = type(spec.name, (Stub, ), slots)
 
         for method in spec.methods:
             slots[method].__objclass__ = stubtype
 
+        stubtype.__retrace_target_type__ = resolved
+
         return stubtype
 
     def __call__(self, spec):
-        print(f'In stubFactory.__call__ {spec}')
         if spec not in self.cache:
             self.cache[spec] = self.create_stubtype(spec)
         
         stubtype = self.cache[spec]
-        stub = stubtype.__new__(stubtype)
-        print(f'In stubFactory created new')
-        return stub
+        return stubtype.__new__(stubtype)