retracesoftware-proxy 0.0.0__py3-none-any.whl

retracesoftware/install/edgecases.py

@@ -0,0 +1,242 @@
+# from .proxytype import *
+
+import functools
+import os
+
+from retracesoftware.install import globals
+
+def recvfrom_into(target):
+    @functools.wraps(target)
+    def wrapper(self, buffer, nbytes = 0, flags = 0):
+        data, address = self.recvfrom(len(buffer) if nbytes == 0 else nbytes, flags)
+        buffer[0:len(data)] = data
+        return len(data), address
+    return wrapper
+
+def recv_into(target):
+    @functools.wraps(target)
+    def wrapper(self, buffer, nbytes = 0, flags = 0):
+        data = self.recv(len(buffer) if nbytes == 0 else nbytes, flags)
+        buffer[0:len(data)] = data
+        return len(data)
+    return wrapper
+
+def recvmsg_into(target):
+    @functools.wraps(target)
+    def wrapper(self, buffers, ancbufsize = 0, flags = 0):
+        raise NotImplementedError('TODO')
+    return wrapper
+
+def read(target):
+    @functools.wraps(target)
+    def wrapper(self, *args):
+        # super_type = super(type(self), self)
+
+        if len(args) == 0:
+            return target(self)
+        else:
+            buflen = args[0]
+
+            # pdb.set_trace()
+
+            data = target(self, buflen)
+
+            if len(args) == 1:
+                return data
+            else:
+                buffer = args[1]
+
+                buffer[0:len(data)] = data
+
+                return len(data)
+    return wrapper
+
+def write(target):
+    @functools.wraps(target)
+    def wrapper(self, byteslike):
+        return target(byteslike.tobytes())
+
+    return wrapper
+
+def readinto(target):
+    @functools.wraps(target)
+    def wrapper(self, buffer):
+        bytes = self.read(buffer.nbytes)
+        buffer[:len(bytes)] = bytes
+        return len(bytes)
+    return wrapper
+
+typewrappers = {
+    '_socket': {
+        'socket': {
+            'recvfrom_into': recvfrom_into,
+            'recv_into': recv_into,
+            'recvmsg_into': recvmsg_into
+        }
+    },
+    '_ssl': {
+        '_SSLSocket': {
+            'read': read,
+            # 'write': write
+        }
+    },
+    'io': {
+        'FileIO': {
+            'readinto': readinto 
+        },
+        'BufferedReader': {
+            'readinto': readinto
+        },
+        'BufferedRandom': {
+            'readinto': readinto
+        }
+    }
+}
+
+def patchtype(module, name, cls : type):
+    if module in typewrappers:
+        if name in typewrappers[module]:
+            for method,patcher in typewrappers[module][name].items():
+                setattr(cls, method, patcher(getattr(cls, method)))
+
+def transform_argument(target : callable, position : int, name : str, transform : callable, default = None):
+    assert callable(transform)
+    assert callable(target)
+
+    @functools.wraps(target)
+    def wrapper(*args, **kwargs):
+        if name in kwargs:
+            kwargs[name] = transform(kwargs[name])
+        elif len(args) > position:
+            args = list(args)
+            args[position] = transform(args[position])
+        else:
+            kwargs[name] = transform(default)
+
+        # print(f'Running: {args} {kwargs}')
+        return target(*args, **kwargs)
+
+    return wrapper 
+
+def retrace_env():
+    import threading
+
+    env = {
+        'RETRACE_RECORDING_PATH': globals.recording_path.next_subprocess_path(),
+        # 'RETRACE_PARENT_THREAD_ID': threading.current_thread().__retrace_thread_id__,
+        # 'RETRACE_PARENT_EXEC_COUNTER': str(next_exec_counter()
+        'RETRACE_MODE': 'record'
+        # 'RETRACE_MODE': 'proxy' if os.getenv('RETRACE_MODE') == 'proxy' else 'record'
+    }
+    # a timestamped directory off current directory
+    # controlled by config
+    # 1. if recording_dir is set use that
+    # 2. if recording_dir is not set look for 
+
+    # env['RETRACE_EXECUTION_ID'] = hashlib.md5(json.dumps(env).encode('UTF8')).hexdigest()
+
+    return env
+
+def fork_exec(target):
+
+    def transform(env):
+        r = [f'{k}={v}'.encode('utf-8') for k,v in retrace_env().items()]
+        return env + r if env else r
+
+    return transform_argument(target = target, 
+                              position = 5,
+                              name = 'env',
+                              transform = transform, default = os.environ)
+
+    # import threading
+    # thread_state = threading.current_thread().__retrace__.thread_state
+
+    # patched = transform_argument(target = target, 
+    #                           position = 5,
+    #                           name = 'env',
+    #                           transform = transform, default = os.environ)
+
+    # return thread_state.dispatch(target, internal = patched)
+
+def posix_spawn(target):
+
+    def transform(env):
+        r = retrace_env()
+        return {**env, **r}
+
+    return transform_argument(target = target, 
+                              position = 2,
+                              name = 'env',
+                              transform = transform, default = os.environ)
+
+    # import threading
+    # thread_state = threading.current_thread().__retrace__.thread_state
+
+    # patched = transform_argument(target = target, 
+    #                           position = 2,
+    #                           name = 'env',
+    #                           transform = transform, default = os.environ)
+
+    # return thread_state.dispatch(target, internal = patched)
+
+function_patchers = {
+    '_posixsubprocess': {
+        'fork_exec': fork_exec
+    },
+    'posix': {
+        'posix_spawn': posix_spawn
+    }
+}
+
+# import traceback
+
+def typepatcher(cls : type):
+
+    # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! type: {cls} created')
+    # traceback.print_stack()
+    
+    return typewrappers.get(cls.__module__, {}).get(cls.__name__, {})
+
+    # if cls.__module__ in typewrappers:
+    #     # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TYPEWRAPPER for {cls}')
+
+    #     mod = typewrappers[cls.__module__]
+
+    #     return mod.get(cls.__name__, {})
+    
+    #     if cls.__name__ in mod:
+    #         # if cls.__name__ == '_SSLSocket':
+    #         #     breakpoint()
+
+    #         log.info("Applying specialized typewrapper to %s, updated slots: %s", cls, list(mod[cls.__name__].keys()))
+
+    #         for name,value in mod[cls.__name__].items():
+    #             setattr(cls, name, value(getattr(cls, name)))
+
+    #         # slots = {'__module__': cls.__module__, '__slots__': ()}
+    #         # slots.update(mod[cls.__name__])
+    #         # return type(cls.__name__, (cls, ), slots)
+
+    # return cls
+
+def typewrapper(cls : type):
+
+    # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! type: {cls} created')
+    # traceback.print_stack()
+    
+    if cls.__module__ in typewrappers:
+        # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TYPEWRAPPER for {cls}')
+
+        mod = typewrappers[cls.__module__]
+
+        if cls.__name__ in mod:
+            # if cls.__name__ == '_SSLSocket':
+            #     breakpoint()
+
+            log.info("Applying specialized typewrapper to %s, updated slots: %s", classname(cls), list(mod[cls.__name__].keys()))
+
+            slots = {'__module__': cls.__module__, '__slots__': ()}
+            slots.update(mod[cls.__name__])
+            return type(cls.__name__, (cls, ), slots)
+
+    return cls

retracesoftware/install/globals.py

@@ -0,0 +1,17 @@
+class RecordingPath:
+    def __init__(self, path):
+        self.path = path
+        self.subprocess_counter = 0
+        self.fork_counter = 0
+    
+    def next_subprocess_path(self):
+        path = self.path / f'subprocess-{self.subprocess_counter}'
+        self.subprocess_counter = self.subprocess_counter + 1
+        return path
+
+    def next_fork_path(self):
+        path = self.path / f'fork-{self.fork_counter}'
+        self.fork_counter = self.fork_counter + 1
+        return path
+
+recording_path = None

retracesoftware/install/install.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import retracesoftware.functional as functional
+import retracesoftware_utils as utils
+
+# from retracesoftware_functional import mapargs, walker, first_arg, if_then_else, compose, observer, anyargs, memoize_one_arg, side_effect, partial, threadwatcher, firstof, notinstance_test, instance_test,typeof, isinstanceof, always
+# from retracesoftware.proxy.proxy import ProxyFactory, InternalProxy, ProxySpec, WrappingProxySpec, ExtendingProxySpec, ExtendingProxy
+from retracesoftware_proxy import thread_id
+# from retracesoftware_stream import ObjectWriter, ObjectReader
+import retracesoftware.stream as stream
+
+# from retracesoftware_utils import visitor
+from retracesoftware.install.tracer import Tracer
+from retracesoftware.proxy.record import RecordProxySystem
+from retracesoftware.proxy.replay import ReplayProxySystem
+from datetime import datetime
+
+import os
+import sys
+import json
+import enum
+import _thread
+import pickle
+import weakref
+import types
+from pathlib import Path
+import glob, os
+import re
+
+# from retracesoftware_proxy import *
+# from retracesoftware_utils import *
+# from retracesoftware.proxy import references
+from retracesoftware.install import patcher
+from retracesoftware.install.config import load_config
+# from retracesoftware.proxy.immutabletypes import ImmutableTypes
+# from retracesoftware.proxy import edgecases
+from retracesoftware.install import globals
+# from retracesoftware.proxy.record import RecordProxyFactory
+
+
+class DebugWriter:
+    __slot__ = ['checkpoint']
+
+    def __init__(self, checkpoint):
+        self.checkpoint = checkpoint
+ 
+    def write_call(self, func, *args, **kwargs):
+        self.checkpoint({'type': 'call', 'func': ...})
+
+    def write_result(self, res):
+        self.checkpoint({'type': 'result', 'result': res})
+
+    def write_error(self, *args):
+        self.checkpoint({'type': 'error', 'error': tuple(args)})
+
+class MethodDescriptor:
+    def __init__(self, descriptor):
+        self.cls = descriptor.__objclass__
+        self.name = descriptor.name
+
+def once(*args): return functional.memoize_one_arg(functional.sequence(*args))
+
+any = functional.firstof
+
+
+def compose(*args):
+    new_args = [item for item in args if item is not None]
+    if len(new_args) == 0:
+        raise Exception('TODO')
+    elif len(new_args) == 1:
+        return new_args[0]
+    else:
+        return functional.compose(*new_args)
+
+class SerializedWrappedFunction:
+    def __init__(self, func):
+        if hasattr(func, '__objclass__'):
+            self.cls = func.__objclass__
+        elif hasattr(func, '__module__'):
+            self.module = func.__module__
+    
+        if hasattr(func, '__name__'):
+            self.name = func.__name__
+
+
+def replaying_proxy_factory(thread_state, is_immutable_type, tracer, next, bind, checkpoint):
+
+    # def on_new_ext_proxytype(proxytype):
+    #     assert not issubclass(proxytype, DynamicProxy)
+    #     bind(proxytype)
+        # writer.add_type_serializer(cls = proxytype, serializer = functional.typeof)
+
+    # bind_new_int_proxy = functional.if_then_else(functional.isinstanceof(InternalProxy), functional.memoize_one_arg(bind), None)
+        
+    # on_ext_call = utils.visitor(from_arg = 1, function = bind_new_int_proxy)
+
+    def wrap_int_call(handler):
+        return functional.observer(
+            on_call = tracer('proxy.int.call'),
+            on_result = tracer('proxy.int.result'),
+            on_error = tracer('proxy.int.error'),
+            function = handler)
+    
+    # def is_stub_type(obj):
+    #     return type(obj) == type and issubclass(obj, (WrappingProxy, ExtendingProxy))
+
+    def is_stub_type(obj):
+        return type(obj) == type
+
+    create_stubs = functional.walker(functional.when(is_stub_type, utils.create_stub_object))
+    # create_stubs = functional.walker(functional.when(is_stub_type, utils.create_stub_object))
+
+    def wrap_ext_call(handler):
+        return functional.observer(
+            on_call = tracer('proxy.ext.call'),
+            on_result = tracer('proxy.ext.result'),
+            on_error = tracer('proxy.ext.error'),
+            function = functional.sequence(functional.always(next), create_stubs))
+
+    return ProxyFactory(thread_state = thread_state,
+                        is_immutable_type = is_immutable_type,
+                        tracer = tracer, 
+                        on_new_int_proxy = bind,
+                        # on_new_ext_proxytype = on_new_ext_proxytype,
+                        wrap_int_call = wrap_int_call,
+                        wrap_ext_call = wrap_ext_call)
+
+
+# class Reader:
+
+#     def __init__(self, objectreader):
+#         self.objectreader = objectreader
+
+#     self.objectreader()
+
+def tracing_config(config):
+    level = os.environ.get('RETRACE_DEBUG', config['default_tracing_level'])
+    return config['tracing_levels'].get(level, {})
+
+
+def install(create_system):
+    return patcher.install(config = load_config('config.json'), create_system = create_system)

retracesoftware/install/patcher.py

@@ -0,0 +1,122 @@
+import enum
+from retracesoftware.install.typeutils import modify
+from retracesoftware.install.replace import update
+import threading
+import sys
+import retracesoftware.utils as utils
+import retracesoftware.functional as functional
+import importlib
+
+def resolve(path):
+    module, sep, name = path.rpartition('.')
+
+    if module is None:
+        module = 'builtins'
+    
+    return getattr(importlib.import_module(module), name)
+
+def replace(replacements, coll):
+    return map(lambda x: replacements.get(x, x), coll)
+        
+def patch_class(transforms, cls):
+    with modify(cls):
+        for attr,transform in transforms.items():
+            utils.update(cls, attr, resolve(transform))
+
+    return cls
+
+class PerThread(threading.local):
+    def __init__(self):
+        self.internal = utils.counter()
+        self.external = utils.counter()
+
+def create_patcher(system):
+
+    patcher = {}
+
+    def foreach(func): return lambda config: {name: func for name in config}
+    def selector(func): return lambda config: {name: functional.partial(func, value) for name, value in config.items()}
+
+    def simple_patcher(func): return foreach(functional.side_effect(func))
+
+    def type_attributes(transforms, cls):
+        with modify(cls):
+            for action, attrs in transforms.items():
+                for attr,func in patcher[action](attrs).items():
+                    utils.update(cls, attr, func)
+        return cls
+
+    def bind(obj):
+        if issubclass(obj, enum.Enum):
+            for member in obj:
+                system.bind(member)
+        else:
+            system.bind(obj)
+
+    def add_immutable_type(obj):
+        if not isinstance(obj, type):
+            raise Exception("TODO")
+        system.immutable_types.add(obj)
+        return obj
+
+    per_thread = PerThread()
+        
+    hashfunc = system.thread_state.dispatch(
+        functional.constantly(None),
+        internal = functional.repeatedly(functional.partial(getattr, per_thread, 'internal')),
+        external = functional.repeatedly(functional.partial(getattr, per_thread, 'external')))
+
+    def patch_hash(obj):
+        if not isinstance(obj, type):
+            raise Exception("TODO")
+
+        utils.patch_hash(cls = obj, hashfunc = hashfunc)
+        return obj
+    
+    patcher.update({
+        'type_attributes': selector(type_attributes),
+        'patch_class': selector(patch_class),
+        'disable': foreach(system.disable_for),
+        'patch_types': simple_patcher(system.patch_type),
+        'proxy': foreach(system),
+        'bind': simple_patcher(bind),
+        'wrap': lambda config: {name: resolve(action) for name,action in config.items() },
+        'immutable': simple_patcher(add_immutable_type),
+        'patch_hash': simple_patcher(patch_hash),
+    })
+    return patcher
+
+def patch_namespace(patcher, config, namespace, update_refs):
+    for phase_name, phase_config in config.items():
+        if phase_name in patcher:
+            for name,func in patcher[phase_name](phase_config).items():
+                if name in namespace:
+                    # print(f"patching: {name}")
+
+                    value = namespace[name]
+                    new_value = func(value)
+
+                    if value is not new_value:
+                        namespace[name] = new_value
+
+                        if update_refs:
+                            update(value, new_value)
+        else:
+            print(phase_name)
+            utils.sigtrap('FOO1')
+    
+def patch_module(patcher, config, namespace, update_refs):
+    if '__name__' in namespace:
+        name = namespace['__name__']
+        if name in config:
+            patch_namespace(patcher, config = config[name], namespace=namespace, update_refs=update_refs)
+
+def patch_imported_module(patcher, checkpoint, config, namespace, update_refs):
+    if '__name__' in namespace:
+        name = namespace['__name__']
+        checkpoint(f"importing module: {name}")
+
+        if name in config:
+            # print(f"Patching imported module: {name}")
+            # checkpoint(f"Patching imported module: {name}")
+            patch_namespace(patcher, config = config[name], namespace=namespace, update_refs=update_refs)

retracesoftware/install/patchfindspec.py

@@ -0,0 +1,117 @@
+from functools import wraps
+import os
+from pathlib import Path
+import shutil
+import sys
+
+# Set to track copied modules
+copied_modules = set()
+
+def get_relative_path(module_path: Path, sys_path: list) -> Path:
+    """Compute the relative path of module_path relative to sys.path entries."""
+    module_path = module_path.resolve()
+    for base in sys_path:
+        base_path = Path(base).resolve()
+        try:
+            if module_path.is_relative_to(base_path):
+                return module_path.relative_to(base_path)
+        except ValueError:
+            continue
+    return Path(module_path.name)
+
+class patch_find_spec:
+    def __init__(self, cwd, run_path, python_path):
+        self.cwd = cwd
+        self.run_path = run_path
+        self.python_path = python_path
+
+    def __call__(self, spec):
+        if spec is not None and spec.origin and spec.origin != "built-in" and os.path.isfile(spec.origin):
+            module_name = spec.name
+            if module_name not in copied_modules:
+                module_path = Path(spec.origin)
+
+                dest_path = None
+
+                if module_path.is_relative_to(self.cwd):
+                    dest_path = self.run_path / module_path.relative_to(self.cwd)
+                elif self.python_path:
+                    relative_path = get_relative_path(module_path, sys.path)
+                    dest_path = self.python_path / relative_path
+
+                if dest_path:
+                    dest_path.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(module_path, dest_path)
+                    copied_modules.add(module_name)
+
+    # def __call__(self, fullname, path, target=None):
+    #     spec = self.original_find_spec(fullname, path, target)
+
+    #     if spec is not None and spec.origin and spec.origin != "built-in" and os.path.isfile(spec.origin):
+    #         module_name = spec.name
+    #         if module_name not in copied_modules:
+    #             module_path = Path(spec.origin)
+
+    #             dest_path = None
+
+    #             if module_path.is_relative_to(self.cwd):
+    #                 dest_path = self.run_path / module_path.relative_to(self.cwd)
+    #             elif self.python_path:
+    #                 relative_path = get_relative_path(module_path, sys.path)
+    #                 dest_path = self.python_path / relative_path
+
+    #             if dest_path:
+    #                 dest_path.parent.mkdir(parents=True, exist_ok=True)
+    #                 shutil.copy2(module_path, dest_path)
+    #                 copied_modules.add(module_name)
+
+    #     return spec
+
+# def patch_find_spec(cwd, run_path, python_path, original_find_spec):
+#     """Create a patched version of find_spec that copies .py files."""
+#     @wraps(original_find_spec)
+#     def patched_find_spec(fullname, path, target=None):
+#         spec = original_find_spec(fullname, path, target)
+
+#         if spec is not None and spec.origin and spec.origin != "built-in" and os.path.isfile(spec.origin):
+#             module_name = spec.name
+#             if module_name not in copied_modules:
+#                 module_path = Path(spec.origin)
+
+#                 dest_path = None
+
+#                 if module_path.is_relative_to(cwd):
+#                     dest_path = run_path / module_path.relative_to(cwd)
+#                 elif python_path:
+#                     relative_path = get_relative_path(module_path, sys.path)
+#                     dest_path = python_path / relative_path
+
+#                 if dest_path:
+#                     dest_path.parent.mkdir(parents=True, exist_ok=True)
+#                     shutil.copy2(module_path, dest_path)
+#                     copied_modules.add(module_name)
+
+#                 # elif python_path:
+#                 #     relative_path = get_relative_path(module_path, sys.path)
+#                 #     dest_path = python_path / relative_path
+#                 #     dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+#                 #     shutil.copy2(module_path, dest_path)
+#                 #     copied_modules.add(module_name)
+
+#                     # if module_path.suffix == '.py':
+#                 # elif module_path.suffix == '.py':
+#                 #     relative_path = get_relative_path(module_path, sys.path)
+#                 #     dest_path = path / relative_path
+#                 #     dest_path.parent.mkdir(parents=True, exist_ok=True)
+#                 #     try:
+#                 #         shutil.copy2(module_path, dest_path)
+#                 #         print(f"Copied {module_path} to {dest_path} (relative: {relative_path})")
+#                 #         copied_modules.add(module_name)
+#                 #     except Exception as e:
+#                 #         print(f"Failed to copy {module_path}: {e}")
+#                 # else:
+#                 #     print(f"Skipped copying {module_path} (not a .py file)")
+
+#         return spec
+#     return patched_find_spec