retracesoftware-proxy 0.0.0__py3-none-any.whl

retracesoftware/proxy/replay.py

@@ -0,0 +1,138 @@
+import retracesoftware.functional as functional
+import retracesoftware_utils as utils
+
+from retracesoftware.install.tracer import Tracer
+from retracesoftware.proxy.thread import per_thread_messages, thread_id
+from retracesoftware.proxy.messagestream import *
+from retracesoftware.proxy.proxytype import *
+# from retracesoftware.proxy.gateway import gateway_pair
+from retracesoftware.proxy.record import StubRef
+from retracesoftware.proxy.proxysystem import ProxySystem
+from retracesoftware.proxy.stubfactory import StubFactory
+from retracesoftware.proxy.globalref import GlobalRef
+
+import os
+
+class ReplayProxySystem(ProxySystem):
+    
+    def after_fork_in_child(self):
+        self.reader.path = self.new_child_path(self.reader.path)
+        super().after_fork_in_child()
+
+    # def dynamic_ext_proxytype(self, cls):
+    #     raise Exception('dynamic_ext_proxytype should not be called in replay')
+
+    @property
+    def ext_apply(self): 
+        return functional.repeatedly(self.next_result)
+
+    def proxy__new__(self, __new__, *args, **kwargs):
+        func = functional.repeatedly(self.next_result)
+        func.__name__ = '__new__'
+        return super().proxy__new__(func, *args, **kwargs)
+
+    def basetype(self, cls):
+        return self.stub_factory.create_stubtype(StubRef(cls))
+
+    def trace_writer(self, name, *args):
+        with self.thread_state.select('disabled'):
+            # read = self.messages_read
+
+            self.read_required('TRACE')
+            # self.read_required(read)
+            self.read_required(name)
+
+            if name == 'stacktrace':
+                print('FOOO!!!')
+                os._exit(1)
+                record = self.readnext()
+                if args[0] == record:
+                    self.last_matching_stack = args[0]
+                else:
+                    on_stack_mismatch(
+                        last_matching = self.last_matching_stack,
+                        record = record,
+                        replay = args[0])
+                    os._exit(1)
+            else:
+                # print(f'Trace: {self.reader.messages_read} {name} {args}')
+                for arg in args:
+                    self.read_required(arg)
+
+    def on_thread_exit(self, thread_id):
+        # print(f'on_thread_exit!!!!')
+        self.reader.wake_pending()
+
+    def __init__(self, 
+                 reader,
+                 thread_state,
+                 immutable_types,
+                 tracing_config,
+                 traceargs,
+                 verbose = False,
+                 fork_path = [],
+                 skip_weakref_callbacks = False):
+
+        self.reader = reader
+        # self.skip_weakref_callbacks = skip_weakref_callbacks
+
+        self.fork_path = fork_path
+
+        self.messages = MessageStream(
+            thread_state = thread_state,
+            source = reader, 
+            skip_weakref_callbacks = skip_weakref_callbacks,
+            verbose = verbose)
+
+        self.checkpoint = self.messages.checkpoint
+        self.bind = self.messages.bind
+        self.next_result = self.messages.result
+        self.exclude_from_stacktrace = self.messages.excludes.add
+
+        self.stub_factory = StubFactory(thread_state = thread_state, next_result = self.next_result)
+
+        self.last_matching_stack = None
+
+        def run_ref(ref):
+            print(f'run_ref!!!! {ref}')
+            return ref()
+
+        self.reader.type_deserializer[StubRef] = self.stub_factory
+        # self.reader.type_deserializer[GlobalRef] = lambda ref: ref()
+        self.reader.type_deserializer[GlobalRef] = run_ref
+
+        excludes = [ReplayProxySystem.trace_writer]
+        
+        for exclude in excludes:
+            self.messages.excludes.add(exclude)
+
+        sync = functional.lazy(self.messages.read_required, 'SYNC')
+
+        read_sync = thread_state.dispatch(utils.noop, internal = sync)
+
+        self.on_ext_call = sync
+
+        self.sync = lambda function: utils.observer(on_call = read_sync, function = function)
+        
+        self.create_from_external = utils.noop
+
+        if skip_weakref_callbacks:
+            self.wrap_weakref_callback = \
+                lambda callback: \
+                    thread_state.dispatch(
+                        callback, internal = self.disable_for(callback))
+        else:
+            self.on_weakref_callback_start = functional.lazy(self.messages.read_required, 'ON_WEAKREF_CALLBACK_START')
+            self.on_weakref_callback_end = functional.lazy(self.messages.read_required, 'ON_WEAKREF_CALLBACK_END')
+
+        super().__init__(thread_state = thread_state, 
+                         tracer = Tracer(tracing_config, writer = self.trace_writer),
+                         immutable_types = immutable_types,
+                         traceargs = traceargs)
+
+    def write_trace(self, obj):
+        if 'TRACER' != self.messages():
+            utils.sigtrap(obj)
+
+        # self.read_required       ('TRACER')
+        self.read_required(obj)

retracesoftware/proxy/serializer.py

@@ -0,0 +1,28 @@
+def serializer(obj):
+    cls = type(obj)
+
+    try:
+        if issubclass(cls, Enum):
+            return str(obj)
+
+        if issubclass(cls, (tuple, list)):
+            return list(map(serializer, obj))
+        elif issubclass(cls, dict):
+            return {str(serializer(k)):serializer(v) for k,v in obj.items()}
+        elif issubclass(cls, int):
+            if obj > 1000000 or obj < -1000000:
+                return "XXXX"
+            else:
+                return int(obj)
+        elif issubclass(cls, (bool, str, types.NoneType)):
+            return obj
+        elif issubclass(cls, types.FunctionType):
+            return obj.__qualname__
+        elif issubclass(cls, types.CellType):
+            return getattr(obj, '__qualname__', 'CellType')
+        elif issubclass(cls, types.GeneratorType):
+            return obj.__qualname__
+        else:
+            return cleanse(str(obj))
+    except:
+        return f'Unserializable object of type: {cls}'

retracesoftware/proxy/startthread.py

@@ -0,0 +1,40 @@
+from retracesoftware import functional
+from retracesoftware import utils
+
+import _thread
+
+# _thread.start_new_thread(function, args[, kwargs])
+
+# push a per thread executor function, as a context manager
+
+counters = _thread._local()
+counters.id = ()
+counters.counter = 0
+
+def with_thread_id(thread_id, function, *args, **kwargs):
+    counters.id = thread_id
+    counters.counter = 0
+    return function(*args, **kwargs)
+
+thread_id = functional.lazy(getattr, counters, 'id')
+
+def start_new_thread(original, wrapper, function, *args):
+    return original(wrapper(function), *args)
+
+def wrap_thread_function(thread_state, function):
+    if thread_state.value == 'internal':
+        next_id = counters.id + (counters.counter,)
+        counters.counter += 1
+        return functional.partial(with_thread_id, next_id, thread_state.wrap('internal', function))
+    else:
+        return function
+
+def patch_thread_start(thread_state):
+
+    wrapper = functional.partial(wrap_thread_function, thread_state)
+
+    _thread.start_new = functional.partial(start_new_thread, _thread.start_new, wrapper)
+    _thread.start_new_thread = functional.partial(start_new_thread, _thread.start_new_thread, wrapper)
+
+    import threading
+    threading._start_new_thread = _thread.start_new_thread

retracesoftware/proxy/stubfactory.py

@@ -0,0 +1,195 @@
+import sys
+import os
+
+import retracesoftware.functional as functional
+import retracesoftware.utils as utils
+
+class Stub:
+    __slots__ = []
+
+class ExtendedRef:
+    def __init__(self, module, name):
+        self.type = type
+        self.module = module
+
+class StubRef:
+    def __init__(self, cls):
+        blacklist = ['__class__', '__dict__', '__module__', '__doc__', '__new__']
+
+        self.methods = []
+        self.static_methods = []
+        self.class_methods = []
+
+        for key,value in cls.__dict__.items():
+            if key not in blacklist:
+                if isinstance(value, classmethod):
+                    self.class_methods.append(key)
+                elif isinstance(value, staticmethod):
+                    self.class_methods.append(key)
+                elif utils.is_method_descriptor(value):
+                    self.methods.append(key)
+
+        self.name = cls.__name__ 
+        self.module = cls.__module__
+
+    # def __init__(self, module, name, methods, members):
+    #     self.name = name
+    #     self.module = module
+    #     self.methods = methods
+    #     self.members = members
+
+    def __str__(self):
+        return f'StubRef(module = {self.module}, name = {self.name}, methods = {self.methods})'
+
+def resolve(module, name):
+    try:
+        return getattr(sys.modules[module], name)
+    except:
+        return None
+
+class StubMethodDescriptor(functional.repeatedly):
+    def __init__(self, name, next_result):
+        super().__init__(next_result)
+        self.__name__ = name
+
+    # @utils.striptraceback
+    # def __call__(self, *args, **kwargs):
+    #     super().__call__(*args, **kwargs)
+
+    #@utils.striptraceback
+    def __str__(self):
+        return f"stub - {__name__}"
+    
+class StubMemberDescriptor:
+    def __init__(self, name, next_result):
+        self.next_result = next_result
+        self.__name__ = name
+
+    #@utils.striptraceback
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        
+        return self.next_result()
+
+    #@utils.striptraceback
+    def __set__(self, instance, value):
+        return self.next_result()
+
+    #@utils.striptraceback
+    def __delete__(self, instance):
+        return self.next_result()
+ 
+    #@utils.striptraceback
+    def __str__(self):
+        return f"stub member - {__name__}"
+
+class StubFactory:
+
+    __slots__ = ['next_result', 'thread_state', 'cache']
+
+    def __init__(self, thread_state, next_result):
+        self.next_result = next_result
+        self.thread_state = thread_state
+        self.cache = {}
+
+    def create_member(self, name):
+        def disabled(*args, **kwargs):
+            if self.thread_state.value == 'disabled' and name == '__repr__':
+                return f"stub member - {name}"
+            else:
+                print(f'Error trying to call member descriptor: {name} {args} {kwargs}, retrace mode: {self.thread_state.value}')
+                utils.sigtrap(None)
+                os._exit(1)
+            
+        next_result = self.thread_state.dispatch(disabled, external = self.next_result)
+
+        return StubMemberDescriptor(name = name, next_result = next_result)
+ 
+    def create_method(self, name):
+
+        def disabled(*args, **kwargs):
+            if self.thread_state.value == 'disabled' and name == '__repr__':
+                return f"stub - {name}"
+            else:
+                return None
+                # print(f'Error trying to call descriptor: {name} {args} {kwargs}, retrace mode: {self.thread_state.value}')
+
+                # import traceback
+                # traceback.print_stack()
+                
+                # utils.sigtrap(None)
+                # os._exit(1)
+            
+        next_result = self.thread_state.dispatch(disabled, external = self.next_result)
+
+        func = functional.repeatedly(next_result)
+        func.__name__ = name
+
+        return func
+
+    def create_stubtype(self, spec):
+
+        assert self.thread_state.value == 'disabled'
+
+        slots = {
+            '__module__': spec.module,
+            '__qualname__': spec.name,
+            '__name__': spec.name,
+        }
+
+        for method in spec.methods:
+            slots[method] = self.create_method(method)
+            assert utils.is_method_descriptor(slots[method])
+
+        def on_disabled(name):
+            print(f'Error trying to get/set attribute: {name}, when retrace mode: {self.thread_state.value} was not external')
+            utils.sigtrap(None)
+            os._exit(1)
+
+        #@utils.striptraceback
+        def getattr(instance, name):
+            print('In stub getattr!!!')
+            if self.thread_state.value == 'external':
+                return self.next_result()
+            else:
+                print(f'Error trying to get attribute: {name}, when retrace mode: {self.thread_state.value} was not external')
+                utils.sigtrap(None)
+                os._exit(1)
+
+        #@utils.striptraceback
+        def setattr(instance, name, value):
+            if self.thread_state.value == 'external':
+                return self.next_result()
+            else:
+                print(f'Error trying to set attribute: {name}, to: {value} when retrace mode: {self.thread_state.value} was not external')
+                utils.sigtrap(None)
+                os._exit(1)
+
+        slots['__getattr__'] = self.thread_state.method_dispatch(on_disabled, external = self.next_result)
+        # slots['__getattr__'] = getattr
+        slots['__setattr__'] = self.thread_state.method_dispatch(on_disabled, external = self.next_result)
+
+        resolved = resolve(spec.module, spec.name)
+
+        # if isinstance(resolved, type):
+        #     slots['__class__'] = property(functional.repeatedly(resolved))
+
+        # else:
+        #     utils.sigtrap(f'{spec.module}.{spec.name}')
+
+        stubtype = type(spec.name, (Stub, ), slots)
+
+        for method in spec.methods:
+            slots[method].__objclass__ = stubtype
+
+        stubtype.__retrace_target_type__ = resolved
+
+        return stubtype
+
+    def __call__(self, spec):
+        if spec not in self.cache:
+            self.cache[spec] = self.create_stubtype(spec)
+        
+        stubtype = self.cache[spec]
+        return stubtype.__new__(stubtype)

retracesoftware/proxy/thread.py

@@ -0,0 +1,106 @@
+import retracesoftware.functional as functional
+import retracesoftware_utils as utils
+
+import os
+import _thread
+
+# def thread_aware_writer(writer):
+#     on_thread_switch = functional.sequence(utils.thread_id(), writer.handle('THREAD_SWITCH'))
+#     return utils.threadawareproxy(on_thread_switch = on_thread_switch, target = writer)
+
+class ThreadSwitch:
+    __slots__ = ['id']
+
+    def __init__(self, id):
+        self.id = id
+
+    def __repr__(self):
+        return f'ThreadSwitch<{self.id}>'
+
+    def __str__(self):
+        return f'ThreadSwitch<{self.id}>'
+
+# def set_thread_id(writer, id):
+#     utils.sigtrap(id)
+#     utils.set_thread_id(writer.handle(ThreadSwitch(id)))
+
+def write_thread_switch(writer):
+    on_thread_switch = functional.repeatedly(functional.sequence(utils.thread_id, writer))
+
+    return lambda f: utils.thread_aware_proxy(target = f, on_thread_switch = on_thread_switch, sticky = False)
+
+def prefix_with_thread_id(f, thread_id):
+    current = None
+
+    def next():
+        nonlocal current, f
+        if current is None: current = thread_id()
+
+        obj = f()
+
+        while issubclass(type(obj), ThreadSwitch):
+            current = obj.id
+            obj = f()
+
+        # print(f'prefix_with_thread_id: {(current, obj)}')
+        return (current, obj)
+
+    return next
+
+def per_thread_messages(messages):
+    thread_id = utils.thread_id
+    # thread_id = lambda: 'FOOOOO!!!'
+
+    def on_timeout(demux, key):
+        print(f'ON TIMEOUT!!!! {key} pending: {demux.pending} {demux.pending_keys}')
+        utils.sigtrap(demux)
+        os._exit(1)
+
+    demux = utils.demux(source = prefix_with_thread_id(messages, thread_id),
+                        key_function = lambda obj: obj[0],
+                        timeout_seconds = 60,
+                        on_timeout = on_timeout)
+
+    # def next():
+    #     thread,message = demux(thread_id())
+    #     return message
+    
+    # return next
+    return functional.repeatedly(lambda: demux(thread_id())[1])
+
+
+# _thread.start_new_thread(function, args[, kwargs])
+counters = _thread._local()
+counters.id = ()
+counters.counter = 0
+
+def with_thread_id(thread_id, on_exit, function):
+    def on_call(*args, **kwargs):
+        counters.id = thread_id
+        counters.counter = 0
+
+    def on_result(res):
+        on_exit(thread_id)
+    
+    def on_error(*args):
+        on_exit(thread_id)
+
+    return utils.observer(on_call = on_call, on_result = on_result, on_error = on_error, function = function)
+
+thread_id = functional.lazy(getattr, counters, 'id')
+
+def start_new_thread_wrapper(thread_state, on_exit, start_new_thread):
+
+    def wrapper(function, *args):
+        
+        next_id = counters.id + (counters.counter,)
+        counters.counter += 1
+
+        wrapped_function = with_thread_id(thread_id = next_id,
+                                          on_exit = on_exit, 
+                                          function = thread_state.wrap('internal', function))
+
+        return start_new_thread(wrapped_function, *args)
+
+    return thread_state.dispatch(start_new_thread, internal = wrapper)
+

retracesoftware/replay.py

@@ -0,0 +1,104 @@
+import retracesoftware.stream as stream
+from retracesoftware.run import install, run_with_retrace, ImmutableTypes, thread_states
+import argparse
+import sys
+from pathlib import Path
+from retracesoftware.proxy.thread import thread_id
+from retracesoftware.proxy.replay import ReplayProxySystem
+import retracesoftware.utils as utils
+import json
+from retracesoftware.stackdifference import on_stack_difference
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog=f"python -m {sys.argv[0]}",
+        description="Run a Python module with debugging, logging, etc."
+    )
+
+    parser.add_argument(
+        '--verbose', 
+        action='store_true', 
+        help='Enable verbose output'
+    )
+
+    parser.add_argument(
+        '--timeout',   # or '-r'
+        type = int,      # ensures it's a string (optional, but safe)
+        default = 60,    # default value if not provided
+        help = 'the directory to place the recording files'
+    )
+
+    parser.add_argument(
+        '--recording',   # or '-r'
+        type = str,      # ensures it's a string (optional, but safe)
+        default = '.',    # default value if not provided
+        help = 'the directory to place the recording files'
+    )
+
+    return parser.parse_args()
+
+def load_json(file):
+    with open(file, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+def main():
+    # import debugpy
+
+    # port = 1977
+    # debugpy.listen(("127.0.0.1", port))
+    # debugpy.wait_for_client()
+
+        # self.reader = stream.reader(path, 
+        #                             thread = thread_id,
+        #                             timeout_seconds = 60,
+        #                             verbose = verbose,
+        #                             on_stack_difference = thread_state.wrap('disabled', on_stack_difference),
+        #                             magic_markers = magic_markers)
+
+    # return ReplayProxySystem(thread_state = thread_state, 
+    #                             immutable_types = immutable_types,
+    #                             tracing_config = tracing_config,
+    #                             mainscript = mainscript,
+    #                             path = recording_path / 'trace.bin',
+    #                             tracecalls = env_truthy('RETRACE_ALL', False),
+    #                             verbose = verbose,
+    #                             magic_markers = env_truthy('RETRACE_MAGIC_MARKERS', False))
+
+    args = parse_args()    
+    path = Path(args.recording)
+
+    if not path.exists():
+        raise Exception(f"Recording path: {path} does not exist")
+
+    settings = load_json(path / "settings.json")
+
+    thread_state = utils.ThreadState(*thread_states)
+
+    with stream.reader(path = path / 'trace.bin',
+                       thread = thread_id, 
+                       timeout_seconds = args.timeout,
+                       verbose = args.verbose,
+                       on_stack_difference = thread_state.wrap('disabled', on_stack_difference),
+                       magic_markers = settings['magic_markers']) as reader:
+
+        tracing_config = {}
+
+        system = ReplayProxySystem(
+            reader = reader,
+            thread_state = thread_state,
+            immutable_types = ImmutableTypes(), 
+            tracing_config = tracing_config,
+            tracecalls = settings['trace_inputs'])
+
+        install(system)
+
+        run_with_retrace(system, settings['argv'])
+
+        # install(system)
+
+        # run_with_retrace(system, args.rest[1:])
+
+        # runpy.run_module('foo', run_name="__main__", alter_sys=False)
+
+if __name__ == "__main__":
+    main()