retracesoftware-proxy 0.1.21__py3-none-any.whl → 0.2.0__py3-none-any.whl

retracesoftware/proxy/record.py

@@ -7,6 +7,7 @@ from retracesoftware.proxy.proxytype import *
 from retracesoftware.proxy.proxysystem import ProxySystem
 from retracesoftware.proxy.thread import write_thread_switch, ThreadSwitch, thread_id
 from retracesoftware.install.tracer import Tracer
+from retracesoftware.install.patchfindspec import patch_find_spec
 from retracesoftware.proxy.stubfactory import StubRef, ExtendedRef
 from retracesoftware.proxy.globalref import GlobalRef
 
@@ -15,11 +16,11 @@ import os
 import types
 import gc
 
-class Placeholder:
-    __slots__ = ['id', '__weakref__']
+# class Placeholder:
+#     __slots__ = ['id', '__weakref__']
 
-    def __init__(self, id):
-        self.id = id
+#     def __init__(self, id):
+#         self.id = id
     
 def keys_where_value(pred, dict):
     for key,value in dict.items():
@@ -50,13 +51,14 @@ def resolveable_name(obj):
 # when 
 class RecordProxySystem(ProxySystem):
     
-    def bind(self, obj):
-        self.bindings[obj] = self.writer.handle(Placeholder(self.next_placeholder_id))
-        self.writer(self.bindings[obj])
-        self.next_placeholder_id += 1
+    # def bind(self, obj):
+    #     self.bindings[obj] = self.writer.handle(Placeholder(self.next_placeholder_id))
+    #     self.writer(self.bindings[obj])
+    #     self.next_placeholder_id += 1
 
     def before_fork(self):
-        self.writer.close()
+        self.writer.keep_open = False
+        # self.writer.close()
         super().before_fork()
         # self.writer.path = self.dynamic_path
 
@@ -64,107 +66,136 @@ class RecordProxySystem(ProxySystem):
         new_path = self.new_child_path(self.writer.path)
         new_path.parent.mkdir()
         self.writer.path = new_path
+        self.writer.keep_open = True
         super().after_fork_in_child()
 
     def after_fork_in_parent(self):
         super().after_fork_in_parent()
         self.thread_state.value = self.saved_thread_state
-        self.writer.reopen()
+        self.writer.keep_open = True
+        # self.writer.reopen()
     
     def set_thread_id(self, id):
         utils.set_thread_id(self.writer.handle(ThreadSwitch(id)))
         # utils.set_thread_id(id)
 
-    def is_entry_frame(self, frame):
-        if super().is_entry_frame(frame):
-            self.write_main_path(frame.function.__code__.co_filename)
-            return True
-        return False
+    # def is_entry_frame(self, frame):
+    #     if super().is_entry_frame(frame):
+    #         self.write_main_path(frame.function.__code__.co_filename)
+    #         return True
+    #     return False
+
+    def create_from_external(self, obj):
+        # class of obj is bound
+        # can now write type(obj)
+
+        self.bind(obj)
+
+        breakpoint()
+
+    def patch_type(self, cls):
+
+        patched = super().patch_type(cls)
+
+        self.writer.type_serializer[patched] = functional.side_effect(self.writer.ext_bind)
+
+        return patched
+
+    def exclude_from_stacktrace(self, func):
+        self.writer.exclude_from_stacktrace(func)
 
-    def __init__(self, thread_state, 
+    def on_gc_event(self, phase, info):
+        if phase == 'start':
+            self.writer.stacktraces = False
+            self.on_start_collect(info['generation'])
+            
+        elif phase == 'stop':
+            self.on_end_collect()
+            self.writer.stacktraces = self.stacktraces
+
+    def __init__(self, thread_state,
+                 writer,
                  immutable_types, 
                  tracing_config,
-                 write_main_path,
-                 path):
+                 maybe_collect,
+                 traceargs):
         
         self.fork_counter = 0
-        self.write_main_path = write_main_path
-
+        # self.write_main_path = write_main_path
+    
         self.getpid = thread_state.wrap(
             desired_state = 'disabled', function = os.getpid)
         
         self.pid = self.getpid()
 
-        self.writer = stream.writer(path = path, thread = thread_id)
-        
-        # def on_switch():
-        #     print(f"On thread switch!!!")
-        #     # print(f"On thread switch!!!: {utils.thread_id().id}")
-        #     # utils.sigtrap(utils.thread_id())
-        #     self.writer(utils.thread_id())
-        #     # utils.sigtrap(utils.thread_id())
-
-        # write = utils.observer(on_call = utils.thread_switch_monitor(on_switch), function = self.writer)
-
-        # w = self.writer.handle('TRACE')
-        # def trace_writer(*args):
-        #     print(f'Trace: {args}')
-        #     w(*args)
-
-        self.extended_types = {}
-        self.bindings = utils.id_dict()
-        self.next_placeholder_id = 0
+        self.writer = writer
 
-        serialize = functional.walker(self.bindings.get_else_key)
-        
-        # def on_switch():
-        #     print("On thread switch!!!")
-        #     # utils.sigtrap(utils.thread_id())
-        #     utils.thread_id()()
-        #     # utils.sigtrap(utils.thread_id())
+        self.stacktraces = self.writer.stacktraces
 
-        # self.thread_switch_monitor = utils.thread_switch_monitor(on_switch)
+        if self.stacktraces:
+            def set(status):
+                self.writer.stacktraces = status
 
-        # self.thread_switch_monitor = utils.thread_switch_monitor(functional.repeatedly(utils.thread_id))
+            self.wrap_weakref_callback = functional.sequence(
+                self.wrap_weakref_callback,
+                lambda callback:           
+                    utils.observer(
+                        on_call = functional.lazy(set, False),
+                        on_result = functional.lazy(set, True),
+                        on_error = functional.lazy(set, True),
+                        function = callback))
+                
+        self.exclude_from_stacktrace(RecordProxySystem.patch_type)
+        self.exclude_from_stacktrace(patch_find_spec.__call__)
 
-        # self.sync = lambda function: functional.firstof(self.thread_switch_monitor, functional.always(self.writer.handle('SYNC')), function)
+        self.extended_types = {}
 
         sync_handle = self.writer.handle('SYNC')
 
+        self.on_ext_call = functional.lazy(utils.runall(maybe_collect, sync_handle) if maybe_collect else sync_handle) 
+
         write_sync = thread_state.dispatch(utils.noop, internal = functional.lazy(sync_handle))
 
         self.sync = lambda function: \
             utils.observer(on_call = write_sync, function = function)
-
         error = self.writer.handle('ERROR')
 
         def write_error(cls, val, traceback):
+            assert isinstance(val, BaseException)
             error(cls, val)
         
-        # self.set_thread_id = functional.partial(utils.set_thread_id, self.writer)
-
-        # w = self.writer.handle('TRACE')
-        # def foo(name, *args):
-        #     print(f'writing: {self.writer.messages_written} {name} {args}')
-        #     w(self.writer.messages_written, name, *args)
-
-        # tracer = Tracer(tracing_config, writer = foo)
         tracer = Tracer(tracing_config, writer = self.writer.handle('TRACE'))
 
-        # self.wrap_int_to_ext = self.sync
+        self.writer.exclude_from_stacktrace(write_error)
+        # self.writer.exclude_from_stacktrace(Tracer.write_call)
         
-        self.on_int_call = functional.mapargs(transform = serialize, function = self.writer.handle('CALL'))
+        self.on_int_call = self.writer.handle('CALL')
         
-        self.on_ext_result = functional.sequence(serialize, self.writer.handle('RESULT'))
+        # write_new_ref = self.writer.handle('RESULT')
+
+        self.on_ext_result = self.writer.handle('RESULT')
         
         self.on_ext_error = write_error
 
-        self.ext_apply = self.int_apply = functional.apply
-
         self.writer.type_serializer[types.ModuleType] = GlobalRef
 
+        self.bind = self.writer.bind
+
+        self.create_from_external = self.writer.ext_bind
+
+        self.write_trace = self.writer.handle('TRACER')
+
+        self.checkpoint = self.writer.handle('CHECKPOINT')
+
+        self.on_weakref_callback_start = functional.lazy(self.writer.handle('ON_WEAKREF_CALLBACK_START'))
+        self.on_weakref_callback_end = functional.lazy(self.writer.handle('ON_WEAKREF_CALLBACK_END'))
+
+        self.on_start_collect = self.writer.handle('ON_START_COLLECT')
+        self.on_end_collect = self.writer.handle('ON_END_COLLECT')
+
         super().__init__(thread_state = thread_state, 
                          tracer = tracer, 
+                         traceargs = traceargs,
                          immutable_types = immutable_types)
 
     def ext_proxytype(self, cls):
@@ -172,7 +203,9 @@ class RecordProxySystem(ProxySystem):
         proxytype = super().ext_proxytype(cls)
 
         ref = self.writer.handle(StubRef(proxytype))
-  
+
         self.writer.type_serializer[proxytype] = functional.constantly(ref)
 
         return proxytype
+
+

retracesoftware/proxy/replay.py

@@ -1,146 +1,20 @@
 import retracesoftware.functional as functional
 import retracesoftware_utils as utils
-import retracesoftware.stream as stream
 
 from retracesoftware.install.tracer import Tracer
 from retracesoftware.proxy.thread import per_thread_messages, thread_id
+from retracesoftware.proxy.messagestream import *
 from retracesoftware.proxy.proxytype import *
 # from retracesoftware.proxy.gateway import gateway_pair
-from retracesoftware.proxy.record import StubRef, Placeholder
-from retracesoftware.proxy.proxysystem import ProxySystem, RetraceError
-from retracesoftware.proxy.stubfactory import StubFactory, Stub, StubFunction
+from retracesoftware.proxy.record import StubRef
+from retracesoftware.proxy.proxysystem import ProxySystem
+from retracesoftware.proxy.stubfactory import StubFactory
 from retracesoftware.proxy.globalref import GlobalRef
 
 import os
-import weakref
-import traceback
-import pprint
-
-from itertools import count, islice
-
-# we can have a dummy method descriptor, its has a __name__ and when called, returns the next element
-
-# for types, we can patch the __new__ method
-# do it from C and immutable types can be patched too
-# patch the tp_new pointer?
-
-class ReplayError(RetraceError):
-    pass
-
-def count_matching(*lists):
-    count = 0
-    for slice in zip(*lists):
-        if len(set(slice)) == 1:
-            count += 1
-        else:
-            break
-
-    return count
-
-def on_stack_mismatch(last_matching, record, replay):
-    # print('Common:')
-    # for index, common, replay, record in zip(count(), last_matching_stack, args[0], record):
-    #     if common == replay == record:
-    #         print(common)
-    if last_matching:
-        matching = count_matching(reversed(last_matching),
-                                reversed(record),
-                                reversed(replay))
-
-        print('Common stacktrace:')
-        for line in reversed(list(islice(reversed(last_matching), matching))):
-            print(line)
-        
-        print('last matching stacktrace:')
-        for line in islice(last_matching, 0, len(last_matching) - matching):
-            print(line)
-
-        print('Replay stacktrace:')
-        for line in islice(replay, 0, len(replay) - matching):
-            print(line)
-
-        print('Record stacktrace:')
-        for line in islice(record, 0, len(record) - matching):
-            print(line)
-        
-        print(f'-----------')
-    else:
-        matching = count_matching(reversed(record), reversed(replay))
-
-        print('Common stacktrace:')
-        for line in reversed(list(islice(reversed(record), matching))):
-            print(line)
-        
-        print('Replay stacktrace:')
-        for line in islice(replay, 0, len(replay) - matching):
-            print(line)
-
-        print('Record stacktrace:')
-        for line in islice(record, 0, len(record) - matching):
-            print(line)
-        
-
 
 class ReplayProxySystem(ProxySystem):
     
-    @utils.striptraceback
-    def next_result(self):
-        try:
-            while True:
-                next = self.messages()
-
-                if next == 'CALL':
-                    func = self.messages()
-                    args = self.messages()
-                    kwargs = self.messages()
-
-                    try:
-                        func(*args, **kwargs)
-                    except:
-                        pass
-
-                elif next == 'RESULT':
-                    return self.messages()
-                
-                elif next == 'ERROR':
-                    # breakpoint()
-                    err_type = self.messages()
-                    err_value = self.messages()
-                    utils.raise_exception(err_type, err_value)
-                else:
-                    assert type(next) is not str
-                    return next
-        except TimeoutError:
-            print(f'timeout for reader, active thread: {self.reader.active_thread}, next_control: {self.reader.next_control}')
-
-            for thread,stack in self.reader.stacktraces.items():
-                print(f'thread: {thread}')
-
-                formatted_frames = [
-                    (frame.filename, frame.lineno, frame.function, frame.code_context[0].strip() if frame.code_context else None)
-                    for frame in stack
-                ]
-                formatted_trace = traceback.format_list(formatted_frames)
-                print("Traceback (most recent call last):\n" + "".join(formatted_trace))
-
-            utils.sigtrap(None)
-
-    def bind(self, obj):
-        read = self.messages()
-        
-        assert isinstance(read, Placeholder)
-
-        self.bindings[read] = obj
-
-    # def dynamic_path(self):
-    #     if self.getpid() != self.pid:
-    #         self.pid = self.getpid()
-    #         # ok we are in child, calculate new path
-    #         self.path = self.path / f'fork-{self.fork_counter}'
-    #         self.fork_counter = 0
-        
-    #     return self.path
-
     def after_fork_in_child(self):
         self.reader.path = self.new_child_path(self.reader.path)
         super().after_fork_in_child()
@@ -148,11 +22,9 @@ class ReplayProxySystem(ProxySystem):
     # def dynamic_ext_proxytype(self, cls):
     #     raise Exception('dynamic_ext_proxytype should not be called in replay')
 
-    def proxy_function(self, obj):
-        func = functional.repeatedly(self.next_result)
-        func.__name__ = obj.__name__
-
-        return super().proxy_function(func)
+    @property
+    def ext_apply(self): 
+        return functional.repeatedly(self.next_result)
 
     def proxy__new__(self, __new__, *args, **kwargs):
         func = functional.repeatedly(self.next_result)
@@ -162,45 +34,6 @@ class ReplayProxySystem(ProxySystem):
     def basetype(self, cls):
         return self.stub_factory.create_stubtype(StubRef(cls))
 
-    def readnext(self):
-        with self.thread_state.select('disabled'):
-            try:
-                # obj = self.messages()
-                # print(f'read: {obj}')
-                # return obj
-                return self.messages()
-            except Exception as error:
-                # print(f'Error reading stream: {error}')
-                traceback.print_exc()
-                os._exit(1)
-
-    def read_required(self, required):
-        obj = self.readnext()
-        if obj != required:
-            print('---------------------------------')
-            print('last matching stack')
-            print('---------------------------------')
-            if self.last_matching_stack:
-                for line in self.last_matching_stack:
-                    print(line)
-
-            print('---------------------------------')
-            print(f'Replay: {required}')
-            print('---------------------------------')
-            for line in utils.stacktrace():
-                print(line)
-            print('---------------------------------')
-            print(f'Record: {obj}')
-            print('---------------------------------')
-            for i in range(15):
-                print(self.readnext())
-
-            breakpoint()
-            os._exit(1)
-            raise Exception(f'Expected: {required} but got: {obj}')
-        
-        # self.last_matching_stack = utils.stacktrace()
-
     def trace_writer(self, name, *args):
         with self.thread_state.select('disabled'):
             # read = self.messages_read
@@ -219,7 +52,7 @@ class ReplayProxySystem(ProxySystem):
                     on_stack_mismatch(
                         last_matching = self.last_matching_stack,
                         record = record,
-                        replay = args[0])                        
+                        replay = args[0])
                     os._exit(1)
             else:
                 # print(f'Trace: {self.reader.messages_read} {name} {args}')
@@ -227,69 +60,79 @@ class ReplayProxySystem(ProxySystem):
                     self.read_required(arg)
 
     def on_thread_exit(self, thread_id):
-        print(f'on_thread_exit!!!!')
+        # print(f'on_thread_exit!!!!')
         self.reader.wake_pending()
 
-    def is_entry_frame(self, frame):
-        if super().is_entry_frame(frame) and frame.function.__code__.co_filename == self.mainscript:
-            return True
-        return False
-
     def __init__(self, 
+                 reader,
                  thread_state,
                  immutable_types,
                  tracing_config,
-                 mainscript,
-                 path,
-                 fork_path = []):
-        
-        # self.messages_read = 0
+                 traceargs,
+                 verbose = False,
+                 fork_path = [],
+                 skip_weakref_callbacks = False):
 
-        self.mainscript = mainscript
+        self.reader = reader
+        # self.skip_weakref_callbacks = skip_weakref_callbacks
 
-        self.reader = stream.reader(path, 
-                                    thread = thread_id,
-                                    timeout_seconds = 60)
-    
-        self.bindings = utils.id_dict()
-        # self.set_thread_id = utils.set_thread_id
         self.fork_path = fork_path
-        deserialize = functional.walker(self.bindings.get_else_key)
-
-        # def count(res):
-        #     self.messages_read += 1
-        #     return res
 
-        read_res = functional.vector(functional.lazy(getattr, self.reader, 'messages_read'), self.reader)
+        self.messages = MessageStream(
+            thread_state = thread_state,
+            source = reader, 
+            skip_weakref_callbacks = skip_weakref_callbacks,
+            verbose = verbose)
 
-        def foo():
-            try:
-                messages_read, res = read_res()
-                if issubclass(type(res), Stub):
-                    print(f'res: {utils.thread_id()} {messages_read} stub: {type(res)}')
-                else:
-                    print(f'res: {utils.thread_id()} {messages_read} {res}')
-
-                return res
-            except Exception as error:
-                print(f'Error reading next result: {error}')
-                raise(error)
-        
-        # self.messages = functional.sequence(per_thread_messages(foo), deserialize)
-
-        self.messages = functional.sequence(self.reader, deserialize)
+        self.checkpoint = self.messages.checkpoint
+        self.bind = self.messages.bind
+        self.next_result = self.messages.result
+        self.exclude_from_stacktrace = self.messages.excludes.add
 
         self.stub_factory = StubFactory(thread_state = thread_state, next_result = self.next_result)
 
         self.last_matching_stack = None
 
+        def run_ref(ref):
+            print(f'run_ref!!!! {ref}')
+            return ref()
+
         self.reader.type_deserializer[StubRef] = self.stub_factory
-        self.reader.type_deserializer[GlobalRef] = lambda ref: ref()
-                
-        read_sync = thread_state.dispatch(utils.noop, internal = functional.lazy(thread_state.wrap('disabled', self.read_required), 'SYNC'))
+        # self.reader.type_deserializer[GlobalRef] = lambda ref: ref()
+        self.reader.type_deserializer[GlobalRef] = run_ref
+
+        excludes = [ReplayProxySystem.trace_writer]
+        
+        for exclude in excludes:
+            self.messages.excludes.add(exclude)
+
+        sync = functional.lazy(self.messages.read_required, 'SYNC')
+
+        read_sync = thread_state.dispatch(utils.noop, internal = sync)
+
+        self.on_ext_call = sync
 
         self.sync = lambda function: utils.observer(on_call = read_sync, function = function)
+        
+        self.create_from_external = utils.noop
+
+        if skip_weakref_callbacks:
+            self.wrap_weakref_callback = \
+                lambda callback: \
+                    thread_state.dispatch(
+                        callback, internal = self.disable_for(callback))
+        else:
+            self.on_weakref_callback_start = functional.lazy(self.messages.read_required, 'ON_WEAKREF_CALLBACK_START')
+            self.on_weakref_callback_end = functional.lazy(self.messages.read_required, 'ON_WEAKREF_CALLBACK_END')
 
         super().__init__(thread_state = thread_state, 
                          tracer = Tracer(tracing_config, writer = self.trace_writer),
-                         immutable_types = immutable_types)
+                         immutable_types = immutable_types,
+                         traceargs = traceargs)
+
+    def write_trace(self, obj):
+        if 'TRACER' != self.messages():
+            utils.sigtrap(obj)
+
+        # self.read_required       ('TRACER')
+        self.read_required(obj)

retracesoftware/proxy/serializer.py

@@ -0,0 +1,28 @@
+def serializer(obj):
+    cls = type(obj)
+
+    try:
+        if issubclass(cls, Enum):
+            return str(obj)
+
+        if issubclass(cls, (tuple, list)):
+            return list(map(serializer, obj))
+        elif issubclass(cls, dict):
+            return {str(serializer(k)):serializer(v) for k,v in obj.items()}
+        elif issubclass(cls, int):
+            if obj > 1000000 or obj < -1000000:
+                return "XXXX"
+            else:
+                return int(obj)
+        elif issubclass(cls, (bool, str, types.NoneType)):
+            return obj
+        elif issubclass(cls, types.FunctionType):
+            return obj.__qualname__
+        elif issubclass(cls, types.CellType):
+            return getattr(obj, '__qualname__', 'CellType')
+        elif issubclass(cls, types.GeneratorType):
+            return obj.__qualname__
+        else:
+            return cleanse(str(obj))
+    except:
+        return f'Unserializable object of type: {cls}'

retracesoftware/proxy/startthread.py

@@ -0,0 +1,40 @@
+from retracesoftware import functional
+from retracesoftware import utils
+
+import _thread
+
+# _thread.start_new_thread(function, args[, kwargs])
+
+# push a per thread executor function, as a context manager
+
+counters = _thread._local()
+counters.id = ()
+counters.counter = 0
+
+def with_thread_id(thread_id, function, *args, **kwargs):
+    counters.id = thread_id
+    counters.counter = 0
+    return function(*args, **kwargs)
+
+thread_id = functional.lazy(getattr, counters, 'id')
+
+def start_new_thread(original, wrapper, function, *args):
+    return original(wrapper(function), *args)
+
+def wrap_thread_function(thread_state, function):
+    if thread_state.value == 'internal':
+        next_id = counters.id + (counters.counter,)
+        counters.counter += 1
+        return functional.partial(with_thread_id, next_id, thread_state.wrap('internal', function))
+    else:
+        return function
+
+def patch_thread_start(thread_state):
+
+    wrapper = functional.partial(wrap_thread_function, thread_state)
+
+    _thread.start_new = functional.partial(start_new_thread, _thread.start_new, wrapper)
+    _thread.start_new_thread = functional.partial(start_new_thread, _thread.start_new_thread, wrapper)
+
+    import threading
+    threading._start_new_thread = _thread.start_new_thread