retracesoftware-proxy 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

retracesoftware/config.json

@@ -158,7 +158,12 @@
 
   "modules": {
     "_imp": {
-      "patch_extension_exec": ["exec_dynamic", "exec_builtin"]
+      "patch_extension_exec": ["exec_dynamic", "exec_builtin"],
+      "comment": {
+        "with_state": {
+          "internal": ["create_dynamic"]
+        }
+      }
     },
     "bdb": {
       "methods_with_state": {
@@ -183,7 +188,11 @@
     
     "importlib._bootstrap": {
       "with_state": {
-        "bootstrap": ["_load_unlocked", "_find_spec"]
+        "bootstrap": [
+          "_load_unlocked",
+          "_find_spec",
+          "_lock_unlock_module",
+          "_get_module_lock"]
       }
     },
 
@@ -215,14 +224,20 @@
       ]
     },
     "types": {
+      "patch_hash": ["FunctionType"],
       "immutable_types": [
-        "TracebackType"
+        "TracebackType",
+        "ModuleType"
       ]
     },
     "builtins": {
-      "replace": {
-        "set": "retracesoftware_utils.set",
-        "frozenset": "retracesoftware_utils.frozenset"
+      "patch_hash": ["object"],
+
+      "comment": {
+        "replace": {
+          "set": "retracesoftware_utils.set",
+          "frozenset": "retracesoftware_utils.frozenset"
+        }
       },
       "immutable_types": [
         "BaseException",

retracesoftware/install/patcher.py

@@ -121,9 +121,22 @@ def patch(func):
         if isinstance(spec, str):
             return wrapper(self, [spec], mod_dict)
         elif isinstance(spec, list):
-            return {name: func(self, mod_dict[name]) for name in spec if name in mod_dict}
+            res = {}
+            for name in spec:
+                if name in mod_dict:
+                    value = func(self, mod_dict[name])
+                    if value is not None:
+                        res[name] = value
+            return res
         elif isinstance(spec, dict):
-            return {name: func(self, mod_dict[name], value) for name, value in spec.items() if name in mod_dict}
+            # return {name: func(self, mod_dict[name], value) for name, value in spec.items() if name in mod_dict}
+            res = {}
+            for name,value in spec.items():
+                if name in mod_dict:
+                    value = func(self, mod_dict[name], value)
+                    if value is not None:
+                        res[name] = value
+            return res
         else:
             raise Exception('TODO')
 
@@ -155,6 +168,11 @@ def wrap_method_descriptors(wrapper, prefix, base):
 
     return extended
 
+class PerThread(threading.local):
+    def __init__(self):
+        self.internal = utils.counter()
+        self.externak = utils.counter()
+
 class Patcher:
 
     def __init__(self, thread_state, config, system, 
@@ -164,7 +182,7 @@ class Patcher:
                  post_commit = None):
         
         # validate(config)
-        # system.set_thread_id(0)
+        system.set_thread_id(0)
         # utils.set_thread_id(0)
         self.thread_counter = system.sync(utils.counter(1))
         # self.set_thread_number = set_thread_number
@@ -181,6 +199,13 @@ class Patcher:
         self.exclude_paths = [re.compile(s) for s in config.get('exclude_paths', [])]
         self.typepatcher = {}
 
+        per_thread = PerThread()
+        
+        self.hashfunc = self.thread_state.dispatch(
+            functional.constantly(None),
+            internal = functional.repeatedly(functional.partial(getattr, per_thread, 'internal')),
+            external = functional.repeatedly(functional.partial(getattr, per_thread, 'external')))
+
         def is_phase(name): return getattr(getattr(self, name, None), "is_phase", False)
         
         self.phases = [(name, getattr(self, name)) for name in Patcher.__dict__.keys() if is_phase(name)]
@@ -303,19 +328,35 @@ class Patcher:
                     source.co_name == '<module>' and inspect.getmodule(source)
 
         def exec_wrapper(source, *args,**kwargs):
-            if self.thread_state.value != 'boostrap' and is_module_exec(source):
-                with self.thread_state.select('disabled'):
+            module = None
+
+            with self.thread_state.select('disabled'):
+                if is_module_exec(source):
                     module = inspect.getmodule(source)
-                with self.thread_state.select('internal'):
-                    result = exec(source, *args, **kwargs)
-                
+            
+            if module:
+                result = exec(source, *args, **kwargs)
                 self(module)
-
                 return result
             else:
                 return exec(source, *args,**kwargs)
         
-        return exec_wrapper
+            # if self.thread_state.value != 'bootstrap':
+
+            #     and is_module_exec(source):
+            #     with self.thread_state.select('disabled'):
+            #         module = inspect.getmodule(source)
+            #     with self.thread_state.select('internal'):
+            #         result = exec(source, *args, **kwargs)
+                
+            #     self(module)
+
+            #     return result
+            # else:
+            #     return exec(source, *args,**kwargs)
+        
+        return self.thread_state.dispatch(exec, internal = exec_wrapper)
+    # self.thread_state.wrap(desired_state = 'disabled', function = exec_wrapper)
 
     @patch
     def sync_types(self, value):
@@ -372,6 +413,11 @@ class Patcher:
         
         return updates
 
+    @patch
+    def patch_hash(self, cls):
+        print(f'patching hash for: {cls}')
+        utils.patch_hash(cls = cls, hashfunc = self.hashfunc)
+
     @patch
     def patch_extension_exec(self, exec):
         

retracesoftware/install/record.py

@@ -13,9 +13,15 @@ from datetime import datetime
 import json
 from pathlib import Path
 
-class ThreadSwitch:
-    def __init__(self, id):
-        self.id = id
+# class ThreadSwitch:
+#     def __init__(self, id):
+#         self.id = id
+    
+#     def __repr__(self):
+#         return f'ThreadSwitch<{self.id}>'
+
+#     def __str__(self):
+#         return f'ThreadSwitch<{self.id}>'
 
 def write_files(recording_path):
     with open(recording_path / 'env', 'w') as f:

retracesoftware/proxy/__init__.py

@@ -1,3 +1,3 @@
 from retracesoftware.proxy.record import RecordProxySystem
 from retracesoftware.proxy.replay import ReplayProxySystem
-from retracesoftware.proxy.thread import set_thread_id, per_thread_messages
+from retracesoftware.proxy.thread import per_thread_messages

retracesoftware/proxy/globalref.py

@@ -0,0 +1,31 @@
+import types
+import sys
+
+def find_module_name(mod):
+    for name,value in sys.modules.items():
+        if value is mod:
+            return name
+    
+    raise Exception(f"Cannot create GlobalRef as module {mod} not found in sys.modules")
+
+class GlobalRef:
+    __slots__ = ['parts']
+
+    def __init__(self, obj):
+        if isinstance(obj, types.ModuleType):
+            self.parts = (find_module_name(obj),)
+        else:
+            raise Exception(f"Cannot create GlobalRef from {obj}")
+
+    def __call__(self):
+        module = self.parts[0]
+
+        if len(self.parts) == 0:
+            return module
+        else:
+            obj = getattr(module, self.parts[1])
+            
+            if len(self.parts) == 1:
+                return obj
+            else:
+                raise Exception(f"TODO")

retracesoftware/proxy/proxytype.py

@@ -214,10 +214,16 @@ def dynamic_proxytype(handler, cls):
     
     for name in superdict(cls).keys():
         if name not in blacklist:
-            value = getattr(cls, name)
-            if is_descriptor(value):
-                if utils.is_method_descriptor(value):
-                    spec[name] = wrap(value)
+            try:
+                value = getattr(cls, name)
+                if is_descriptor(value):
+                    if utils.is_method_descriptor(value):
+                        spec[name] = wrap(value)
+            except Exception as error:
+                print(f'FOO! {cls} {name} {error}')
+                breakpoint()
+                raise
+    
                 # else:
                 #     spec[name] = DescriptorProxy(handler = handler, target = value)
 

retracesoftware/proxy/record.py

@@ -5,9 +5,10 @@ import retracesoftware.stream as stream
 from retracesoftware.proxy.proxytype import *
 # from retracesoftware.proxy.gateway import gateway_pair
 from retracesoftware.proxy.proxysystem import ProxySystem
-from retracesoftware.proxy.thread import write_thread_switch
+from retracesoftware.proxy.thread import write_thread_switch, ThreadSwitch
 from retracesoftware.install.tracer import Tracer
 from retracesoftware.proxy.stubfactory import StubRef, ExtendedRef
+from retracesoftware.proxy.globalref import GlobalRef
 
 import sys
 import os
@@ -69,14 +70,11 @@ class RecordProxySystem(ProxySystem):
         super().after_fork_in_parent()
         self.thread_state.value = self.saved_thread_state
         self.writer.reopen()
-
-    def sync(self, function):
-        write_sync = functional.always(self.writer.handle('SYNC'))
-
-        return utils.observer(
-            on_call = functional.firstof(self.thread_switch_monitor, write_sync),
-            function = function)
     
+    def set_thread_id(self, id):
+        utils.set_thread_id(self.writer.handle(ThreadSwitch(id)))
+        # utils.set_thread_id(id)
+
     def __init__(self, thread_state, 
                  immutable_types, 
                  tracing_config,
@@ -91,6 +89,15 @@ class RecordProxySystem(ProxySystem):
 
         self.writer = stream.writer(path)
         
+        def on_switch():
+            print(f"On thread switch!!!")
+            # print(f"On thread switch!!!: {utils.thread_id().id}")
+            # utils.sigtrap(utils.thread_id())
+            self.writer(utils.thread_id())
+            # utils.sigtrap(utils.thread_id())
+
+        write = utils.observer(on_call = utils.thread_switch_monitor(on_switch), function = self.writer)
+
         # w = self.writer.handle('TRACE')
         # def trace_writer(*args):
         #     print(f'Trace: {args}')
@@ -102,39 +109,50 @@ class RecordProxySystem(ProxySystem):
 
         serialize = functional.walker(self.bindings.get_else_key)
         
-        self.thread_switch_monitor = \
-            utils.thread_switch_monitor(
-                functional.repeatedly(functional.sequence(utils.thread_id, self.writer)))
+        # def on_switch():
+        #     print("On thread switch!!!")
+        #     # utils.sigtrap(utils.thread_id())
+        #     utils.thread_id()()
+        #     # utils.sigtrap(utils.thread_id())
+
+        # self.thread_switch_monitor = utils.thread_switch_monitor(on_switch)
+
+        # self.thread_switch_monitor = utils.thread_switch_monitor(functional.repeatedly(utils.thread_id))
 
         # self.sync = lambda function: functional.firstof(self.thread_switch_monitor, functional.always(self.writer.handle('SYNC')), function)
-            
+
+        self.sync = lambda function: \
+            utils.observer(
+                on_call = functional.lazy(write, self.writer.handle('SYNC')), 
+                function = function)
+
         error = self.writer.handle('ERROR')
 
         def write_error(cls, val, traceback):
-            error(cls, val)
+            write(error, cls, val)
         
         # self.set_thread_id = functional.partial(utils.set_thread_id, self.writer)
 
-        def watch(f): return functional.either(self.thread_switch_monitor, f)
-
         # w = self.writer.handle('TRACE')
         # def foo(name, *args):
         #     print(f'writing: {self.writer.messages_written} {name} {args}')
         #     w(self.writer.messages_written, name, *args)
 
         # tracer = Tracer(tracing_config, writer = foo)
-        tracer = Tracer(tracing_config, writer = self.writer.handle('TRACE'))
+        tracer = Tracer(tracing_config, writer = functional.partial(write, self.writer.handle('TRACE')))
 
-        self.wrap_int_to_ext = watch
+        # self.wrap_int_to_ext = self.sync
         
-        self.on_int_call = functional.mapargs(transform = serialize, function = self.writer.handle('CALL'))
+        self.on_int_call = functional.mapargs(transform = serialize, function = functional.partial(write, self.writer.handle('CALL')))
         
-        self.on_ext_result = functional.sequence(serialize, self.writer.handle('RESULT'))
+        self.on_ext_result = functional.sequence(serialize, functional.partial(write, self.writer.handle('RESULT')))
         
         self.on_ext_error = write_error
 
         self.ext_apply = self.int_apply = functional.apply
 
+        self.writer.type_serializer[types.ModuleType] = GlobalRef
+
         super().__init__(thread_state = thread_state, 
                          tracer = tracer, 
                          immutable_types = immutable_types)

retracesoftware/proxy/replay.py

@@ -8,7 +8,8 @@ from retracesoftware.proxy.proxytype import *
 # from retracesoftware.proxy.gateway import gateway_pair
 from retracesoftware.proxy.record import StubRef, Placeholder
 from retracesoftware.proxy.proxysystem import ProxySystem, RetraceError
-from retracesoftware.proxy.stubfactory import StubFactory, StubFunction
+from retracesoftware.proxy.stubfactory import StubFactory, Stub, StubFunction
+from retracesoftware.proxy.globalref import GlobalRef
 
 import os
 import weakref
@@ -153,6 +154,9 @@ class ReplayProxySystem(ProxySystem):
     def read_required(self, required):
         obj = self.readnext()
         if obj != required:
+            print('---------------------------------')
+            print('last matching stack')
+            print('---------------------------------')
             if self.last_matching_stack:
                 for line in self.last_matching_stack:
                     print(line)
@@ -160,6 +164,9 @@ class ReplayProxySystem(ProxySystem):
             print('---------------------------------')
             print(f'Replay: {required}')
             print('---------------------------------')
+            for line in utils.stacktrace():
+                print(line)
+            print('---------------------------------')
             print(f'Record: {obj}')
             print('---------------------------------')
             for i in range(15):
@@ -194,9 +201,6 @@ class ReplayProxySystem(ProxySystem):
                 for arg in args:
                     self.read_required(arg)
 
-    def sync(self, function):
-        return utils.observer(on_call = functional.always(self.read_sync), function = function)
-
     def __init__(self, 
                  thread_state,
                  immutable_types,
@@ -208,25 +212,44 @@ class ReplayProxySystem(ProxySystem):
         self.reader = stream.reader(path)
 
         self.bindings = utils.id_dict()
-        self.set_thread_id = utils.set_thread_id
+        # self.set_thread_id = utils.set_thread_id
         self.fork_path = fork_path
         deserialize = functional.walker(self.bindings.get_else_key)
 
         # def count(res):
         #     self.messages_read += 1
         #     return res
+
+        read_res = functional.vector(functional.lazy(getattr, self.reader, 'messages_read'), self.reader)
+
+        def foo():
+            try:
+                messages_read, res = read_res()
+                if issubclass(type(res), Stub):
+                    print(f'res: {utils.thread_id()} {messages_read} stub: {type(res)}')
+                else:
+                    print(f'res: {utils.thread_id()} {messages_read} {res}')
+
+                return res
+            except Exception as error:
+                print(f'Error reading next result: {error}')
+                raise(error)
         
-        self.messages = functional.sequence(per_thread_messages(self.reader), 
-                                            deserialize)                                        
+        self.messages = functional.sequence(per_thread_messages(foo), deserialize)
+
+        # self.messages = functional.sequence(per_thread_messages(self.reader), deserialize)
 
         self.stub_factory = StubFactory(thread_state = thread_state, next_result = self.next_result)
 
         self.last_matching_stack = None
 
         self.reader.type_deserializer[StubRef] = self.stub_factory
+        self.reader.type_deserializer[GlobalRef] = lambda ref: ref()
                 
-        self.read_sync = functional.always(lambda: self.read_required('SYNC'))
-    
+        read_sync = functional.lazy(self.read_required, 'SYNC')
+
+        self.sync = lambda function: utils.observer(on_call = read_sync, function = function)
+
         super().__init__(thread_state = thread_state, 
                          tracer = Tracer(tracing_config, writer = self.trace_writer),
                          immutable_types = immutable_types)

retracesoftware/proxy/thread.py

@@ -11,17 +11,28 @@ class ThreadSwitch:
     def __init__(self, id):
         self.id = id
 
-def set_thread_id(writer, id):
-    utils.set_thread_id(writer.handle(ThreadSwitch(id)))
+    def __repr__(self):
+        return f'ThreadSwitch<{self.id}>'
+
+    def __str__(self):
+        return f'ThreadSwitch<{self.id}>'
+
+# def set_thread_id(writer, id):
+#     utils.sigtrap(id)
+#     utils.set_thread_id(writer.handle(ThreadSwitch(id)))
 
 def write_thread_switch(writer):
     on_thread_switch = functional.repeatedly(functional.sequence(utils.thread_id, writer))
 
     return lambda f: utils.thread_aware_proxy(target = f, on_thread_switch = on_thread_switch, sticky = False)
 
-def prefix_with_thread_id(f, current):
+def prefix_with_thread_id(f, thread_id):
+    current = None
+
     def next():
         nonlocal current, f
+        if current is None: current = thread_id()
+
         obj = f()
 
         while issubclass(type(obj), ThreadSwitch):
@@ -36,7 +47,8 @@ def per_thread_messages(messages):
     thread_id = utils.thread_id
     # thread_id = lambda: 'FOOOOO!!!'
 
-    demux = utils.demux(source = prefix_with_thread_id(messages, thread_id()), key_function = lambda obj: obj[0])
+    demux = utils.demux(source = prefix_with_thread_id(messages, thread_id),
+                        key_function = lambda obj: obj[0])
 
     # def next():
     #     thread,message = demux(thread_id())

{retracesoftware_proxy-0.1.12.dist-info → retracesoftware_proxy-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: retracesoftware_proxy
-Version: 0.1.12
+Version: 0.1.14
 License: Apache-2.0
 Requires-Dist: retracesoftware_utils
 Requires-Dist: retracesoftware_functional

{retracesoftware_proxy-0.1.12.dist-info → retracesoftware_proxy-0.1.14.dist-info}/RECORD

@@ -1,27 +1,29 @@
 retracesoftware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-retracesoftware/config.json,sha256=UugFK-HB2zW0Lud8RZ8w0EiP0LSJgB4ob_T7JaU1slI,8709
+retracesoftware/config.json,sha256=JU3LYWZbqBLqiHXDc9fBYVh0owMUcKGw3W-vaVm-roo,9024
+retracesoftware/config.yaml,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 retracesoftware/install/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 retracesoftware/install/config.py,sha256=EzE5ifQF2lo--hu2njI4T0FJ-zlnWDJV6i7x0DMkVTw,1364
 retracesoftware/install/edgecases.py,sha256=NR3lyvad9sRsyeDv_Ya8V4xMgPsMPOi9rMcnFOJGOEA,6330
 retracesoftware/install/globals.py,sha256=F8XvIoZQQ10gSRalk30dvdKllxlwxkaggYY6FogLDxY,510
 retracesoftware/install/install.py,sha256=HCD_ji8XCr96b5fNzNdL_8qcEp0Jf05Em7T6GA6u8HU,4969
-retracesoftware/install/patcher.py,sha256=IUkQx5kr4AnniKZ2gWnnx2is1j_y_eb7nz2UKyHyqSA,18894
+retracesoftware/install/patcher.py,sha256=ryX73_KjdcUCC0oy0Uqel_L3bwfZZldS3CIYQ-XAl_k,20520
 retracesoftware/install/predicate.py,sha256=tX7NQc0rGkyyHYO3mduYHcJHbw1wczT53m_Dpkzo6do,2679
-retracesoftware/install/record.py,sha256=tseF_jV4k4HLPTgBPJdjcahl4EQqagoiisMAdGNC52Q,3257
+retracesoftware/install/record.py,sha256=R2GOIA_WAggrNmVwZJh9r1xp-GVu43iKq-ykQ1VKEHE,3408
 retracesoftware/install/references.py,sha256=A-G651IDOfuo00MkbAdpbIQh_15ChvJ7uAVTSmE6zd4,1721
 retracesoftware/install/replay.py,sha256=VUiHvQK3mgAJEGmtE2TFs9kXzxdWtsjibEcGkhZVCVE,1830
 retracesoftware/install/tracer.py,sha256=cHEjiVxIp2iVTJEWndwSbMuiXVyGJQxJYZSGrfpSbCw,5723
 retracesoftware/install/typeutils.py,sha256=_a1PuwdCsYjG1Nkd77V-flqYtwbD4RkJVKn6Z-xABL4,1813
-retracesoftware/proxy/__init__.py,sha256=ZlDZIuUmKFsE9Tvfd2EKGabTepqv8nrbr5pQhCM3IKc,193
+retracesoftware/proxy/__init__.py,sha256=ntIyqKhBRkKEkcW_oOPodikh-mxYl8OXRnSaj-9-Xwc,178
 retracesoftware/proxy/gateway.py,sha256=xESohWXkiNm4ZutU0RgWUwxjxcBWRQ4rQyxIGQXv_F4,1590
+retracesoftware/proxy/globalref.py,sha256=yXtJsOeBHN9xoEgJWA3MJco-jD2SQUef_fDatA4A6rg,803
 retracesoftware/proxy/proxyfactory.py,sha256=qhOqDfMJnLDNkQs26JqDB431MwjjRhGQi8xupJ45asg,12272
 retracesoftware/proxy/proxysystem.py,sha256=ZX6eNT8go-YDbLWD2pq5j_V8DZAHteu-iNfkMO8jLWg,7567
-retracesoftware/proxy/proxytype.py,sha256=H6SJKX0izor1yLOK_QxRlfKLcJC08eMuIYJ1amnoZvQ,12812
-retracesoftware/proxy/record.py,sha256=vtNsoTHP5gsBVJ6dTGHR7QXHGbKm0eONfGcXOb9fhFk,4696
-retracesoftware/proxy/replay.py,sha256=m4Buxlix09aUJX9JYWG1z4jW8Yvbo2zBV353hsZE5qY,7670
+retracesoftware/proxy/proxytype.py,sha256=hP-1vJmWBa9KgZFB3CnZy0UbC4u8Wr2e76lGI0JM2SA,12992
+retracesoftware/proxy/record.py,sha256=QBx3G8mXjUiB9vCXJZ0tZdt3Mi1WEtRhuZ7HOUroNNQ,5505
+retracesoftware/proxy/replay.py,sha256=vVFkhYOOZSprLD_mjx-p6SpDB8fy81SK3x2A5H0odUs,8614
 retracesoftware/proxy/stubfactory.py,sha256=37UX1r8HCAbASTwPdz8QKCrg72NQmM5PsiCho7N2nzg,5129
-retracesoftware/proxy/thread.py,sha256=-SvnyVbANkmX2lLRpOvFtkpdpAoF6DhnnYdOOs7Q8vo,1379
-retracesoftware_proxy-0.1.12.dist-info/METADATA,sha256=HH7c48R8UNiCuIlDo-aS85vN6bbdMVl2WqGIjGmU08o,203
-retracesoftware_proxy-0.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-retracesoftware_proxy-0.1.12.dist-info/top_level.txt,sha256=hYHsR6txLidmqvjBMITpIHvmJJbmoCAgr76-IpZPRz8,16
-retracesoftware_proxy-0.1.12.dist-info/RECORD,,
+retracesoftware/proxy/thread.py,sha256=idkJwJ8rR1kpRHfo81uCi6Y0LEIChAdfc0GqK_bFqTA,1635
+retracesoftware_proxy-0.1.14.dist-info/METADATA,sha256=Y6QfA5aDmUke1rEvUWSOgRIwbEAk7YbbcpW-7m00kQM,203
+retracesoftware_proxy-0.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+retracesoftware_proxy-0.1.14.dist-info/top_level.txt,sha256=hYHsR6txLidmqvjBMITpIHvmJJbmoCAgr76-IpZPRz8,16
+retracesoftware_proxy-0.1.14.dist-info/RECORD,,