retracesoftware-proxy 0.1.0__py3-none-any.whl

retracesoftware/config.json

@@ -0,0 +1,382 @@
+{
+  "states": [
+    "disabled", {"comment": "Default state when retrace is disabled for a thread"},
+    "internal", {"comment": "Default state when retrace is disabled for a thread"},
+    "external", {"comment": "When target thread is running outside the python system to be recorded"},
+    "retrace", {"comment": "When target thread is running outside the retrace system"},
+    "bootstrap", {"comment": "When the target thread is running in the low-level module load machinery"},
+    "gc", {"comment": "When the target thread is running inside the pyton garbage collector"}
+  ],
+
+  "recording_path": "recordings/%Y%m%d_%H%M%S_%f",
+
+  "tracing_levels": {
+    "none": [],
+    "all": [
+      "proxy.ext.disabled.call",
+      "proxy.int.disabled.call",
+      "proxy.wrapping.new",
+      "proxy.wrapping.new.method",
+      "proxy.int.handler",
+      "proxy.ext.handler",
+      "proxy.int.newref",
+      "proxy.ext.call",
+      "proxy.ext.result",
+      "proxy.ext.error",
+      "proxy.int.call",
+      "proxy.int.result",
+      "proxy.int.error",
+      "proxy.ext.new.extended",
+      "proxy.ext.new.extended.method",
+      "proxy.ext.new.extended.member",
+      "proxy.int.new.extended",
+      "proxy.int.new.extended.method",
+      "install.module",
+      "install.module.phase",
+      "install.module.phase.results"
+    ],
+    "debug": [
+      "proxy.wrapping.new",
+      "proxy.wrapping.new.method",
+      "proxy.int_to_ext.stack",
+      "proxy.ext_to_int.wrap",
+      "proxy.int.handler",
+      "proxy.int.newref",
+      "proxy.ext.call",
+      "proxy.ext.result",
+      "proxy.ext.error",
+      "proxy.int.result",
+      "proxy.int.error",
+      "proxy.ext.new.extended",
+      "proxy.ext.new.extended.method",
+      "proxy.ext.new.extended.member",
+      "proxy.int.new.extended",
+      "proxy.int.new.extended.method",
+      "install.module",
+      "install.module.phase",
+      "install.module.phase.results"
+    ]
+  },
+
+  "default_tracing_level": "debug",
+
+  "exclude_paths": [
+    "^.*\\.py$",
+    "^.*\\.pyc$",
+    "^/tmp$",
+    "^/tmp/.*$"
+  ],
+
+  "preload": [
+    
+  ],
+
+  "predicates": {
+    "path": {
+      "and": [
+        {
+          "not": "^.*\\.py$"
+        },
+        {
+          "not": "^.*\\.pyc$"
+        },
+        {
+          "not": "^recordings/.*$"
+        }
+      ]
+    }
+  },
+
+  "type_attribute_filter": {
+    "and": [
+      ".*",
+      {
+        "not": [
+          "__hash__",
+          "__getattribute__",
+          "__init_subclass__",
+          "__subclasshook__",
+          "__class__",
+          "__sizeof__",
+          "__new__",
+          "__del__",
+          "__dict__"
+        ]
+      }
+    ]
+  },
+
+  "filters": {
+    "functions": [
+      {
+        "or": [
+          {
+            "type": "types.BuiltinFunctionType"
+          },
+          {
+            "type": "types.FunctionType"
+          }
+        ]
+      }
+    ],
+    "all": {
+      "and": [
+        {
+          "or": [
+            {
+              "type": "type"
+            },
+            {
+              "type": "types.BuiltinFunctionType"
+            },
+            {
+              "type": "types.FunctionType"
+            }
+          ]
+        },
+        {
+          "noneof": [
+            {
+              "subtype": "BaseException"
+            },
+            {
+              "value": "$immutable"
+            },
+            {
+              "name": "__spec__"
+            },
+            {
+              "name": "__package__"
+            },
+            {
+              "name": "__loader__"
+            }
+          ]
+        }
+      ]
+    }
+  },
+
+  "modules": {
+    "_imp": {
+      "patch_extension_exec": ["exec_dynamic", "exec_builtin"]
+    },
+    "sys": {
+      "with_state": {
+        "disabled": ["excepthook"]
+      }
+    },
+    
+    "importlib._bootstrap": {
+      "with_state": {
+        "bootstrap": ["_load_unlocked", "_find_spec"]
+      }
+    },
+
+    "encodings": {
+      "with_state": {
+        "disabled": ["search_function"]
+      }
+    },
+    "_retrace_utils": {
+      "immutable_types": [
+        "PyCFunctionProxy"
+      ]
+    },
+    "os": {
+      "immutable_types": [
+        "stat_result",
+        "terminal_size",
+        "statvfs_result"
+      ]
+    },
+    "mmap": {
+      "proxy": [
+        "mmapXXX"
+      ]
+    },
+    "types": {
+      "immutable_types": [
+        "TracebackType"
+      ]
+    },
+    "builtins": {
+      "replace": {
+        "set": "retracesoftware_utils.set",
+        "frozenset": "retracesoftware_utils.frozenset"
+      },
+      "immutable_types": [
+        "BaseException",
+        "memoryview",
+        "int",
+        "float",
+        "complex",
+        "str",
+        "bytes",
+        "bool",
+        "bytearray",
+        "type",
+        "slice"],
+
+      "patch_exec": "exec"
+    },
+    "_thread": {
+      "proxy": [
+        "allocate",
+        "allocate_lock",
+        "RLock"
+      ],
+      "patch_start_new_thread": ["start_new_thread", "start_new"]
+    },
+    "_datetime": {
+      "immutable_types": [
+        "datetime",
+        "tzinfo",
+        "timezone"
+      ],
+      "proxy_type_attributes": {
+        "datetime": [
+          "now",
+          "utcnow",
+          "today"
+        ]
+      }
+    },
+    "time": {
+      "immutable_types": [
+        "struct_time"
+      ],
+      "proxy": [
+        "perf_counter",
+        "time",
+        "gmtime",
+        "localtime",
+        "monotonic",
+        "monotonic_ns",
+        "time_ns"
+      ]
+    },
+    "io": {
+      "proxy": ["open_code", "open"],
+      "path_predicates": {
+        "open_code": "path",
+        "open": "file"
+      },
+      "wrap": {
+        "FileIO.readinfo": "retracesoftware.install.edgecases.readinto",
+        "BufferedReader.readinfo": "retracesoftware.install.edgecases.readinto",
+        "BufferedRandom.readinfo": "retracesoftware.install.edgecases.readinto"
+      }
+    },
+    "pathlib": {
+      "immutable_types": [
+        "PosixPath"
+      ]
+    },
+    "posix": {
+      "immutable_types": [
+        "times_result",
+        "statvfs_result",
+        "uname_result",
+        "stat_result",
+        "terminal_size",
+        "DirEntry"
+      ],
+
+      "proxy_all_except": [
+        "fork",
+        "register_at_fork",
+        "basename",
+        "readlink",
+        "strerror",
+        "listdir",
+        "_path_normpath"
+      ],
+
+      "wrappers": {
+        "fork_exec": "retracesoftware.install.edgecases.fork_exec",
+        "posix_spawn": "retracesoftware.install.edgecases.posix_spawn"
+      }
+    },
+    "_posixsubprocess": {
+      "proxy_all_except": []
+    },
+    "fcntl": {
+      "proxy_all_except": []
+    },
+    "_signal": {
+      "proxy_all_except": []
+    },
+    "_socket": {
+      "proxy_all_except": [
+        "CAPI"
+      ],
+      "immutable_types": [
+        "error",
+        "herror",
+        "gaierror",
+        "timeout"
+      ],
+      "wrap": {
+        "socket.recvfrom_into": "retracesoftware.install.edgecases.recvfrom_into",
+        "socket.recv_into": "retracesoftware.install.edgecases.recv_into",
+        "socket.recvmsg_into": "retracesoftware.install.edgecases.recvmsg_into"
+      }
+    },
+    "select": {
+      "proxy_all_except": []
+    },
+    "_sqlite3": {
+      "proxy_all_except": []
+    },
+    "_ssl": {
+      "proxy_all_except": [],
+
+      "proxy_functions": ".*",
+      "comment": {
+        "proxy_types": [
+          "MemoryBIO",
+          "_SSLSocket",
+          "_SSLContext"
+          ],
+          "wrap": {
+            "_SSLSocket.write": "write"
+          }
+        },
+      "wrap": {
+        "_SSLSocket.read": "retracesoftware.install.edgecases.read"
+      }
+    },
+    "_random": {
+      "proxy_all_except": [],
+      "proxy_types": [
+        "Random"
+      ],
+      "proxy_functions": ".*"
+    },
+    "_multiprocessing": {
+      "proxy_functions": ".*"
+    },
+    "multiprocessing.context": {      
+      "proxy": ["_default_context"]
+    },
+
+    "PIL._imaging": {
+      "proxy_all_except": ["map_buffer"]
+    },
+    "psycopg2._psycopg": {
+      "proxy_all_except": ["Error"]
+    },
+
+    "_collections": {
+      "sync_types": [
+        "deque"
+      ]
+    },
+    "_queue": {
+      "sync_types": [
+        "SimpleQueue"
+      ]
+    }
+  }
+}

retracesoftware/install/config.py

@@ -0,0 +1,53 @@
+import pkgutil
+import json
+import os
+import datetime
+
+from pathlib import Path
+
+def debug_level(config):
+    if 'RETRACE_DEBUG' in os.environ:
+        debug = env_int('RETRACE_DEBUG')
+    else:
+        debug = config.get('record', {}).get('debug', 1)
+
+    return debug
+
+def env_truthy(key, default=False):
+    value = os.getenv(key)
+    if value is None:
+        return default
+    return value.strip().lower() in ("1", "true", "yes", "on")
+
+def env_int(key, default = 0):
+    value = os.getenv(key)
+    if value is None:
+        return default
+    return int(value)
+
+def get_recording_path(config):
+    return Path(datetime.datetime.now().strftime(config.format(pid = os.getpid())))
+
+def recording_path(config):
+    if 'RETRACE_RECORDING_PATH' in os.environ:
+        return Path(os.environ['RETRACE_RECORDING_PATH'])
+    else:
+        recording_path = get_recording_path(config.get('record_path', 'recordings'))
+        os.environ['RETRACE_RECORDING_PATH'] = str(recording_path)
+        return recording_path
+
+def load_config(filename):
+        
+    data = pkgutil.get_data("retracesoftware", filename)
+    assert data is not None
+
+    config = json.loads(data.decode("utf-8"))
+
+    config['debug_level'] = debug_level(config)
+
+    # config['recording_path'] = recording_path(config)
+
+    config['verbose'] = env_truthy('RETRACE_VERBOSE')
+    
+    return config
+

retracesoftware/install/edgecases.py

@@ -0,0 +1,220 @@
+# from .proxytype import *
+
+import functools
+import os
+
+from retracesoftware.install import globals
+
+def recvfrom_into(target):
+    @functools.wraps(target)
+    def wrapper(self, buffer, nbytes = 0, flags = 0):
+        data, address = self.recvfrom(len(buffer) if nbytes == 0 else nbytes, flags)
+        buffer[0:len(data)] = data
+        return len(data), address
+    return wrapper
+
+def recv_into(target):
+    @functools.wraps(target)
+    def wrapper(self, buffer, nbytes = 0, flags = 0):
+        data = self.recv(len(buffer) if nbytes == 0 else nbytes, flags)
+        buffer[0:len(data)] = data
+        return len(data)
+    return wrapper
+
+def recvmsg_into(target):
+    @functools.wraps(target)
+    def wrapper(self, buffers, ancbufsize = 0, flags = 0):
+        raise NotImplementedError('TODO')
+    return wrapper
+
+def read(target):
+    @functools.wraps(target)
+    def wrapper(self, *args):
+        # super_type = super(type(self), self)
+
+        if len(args) == 0:
+            return target(self)
+        else:
+            buflen = args[0]
+
+            # pdb.set_trace()
+
+            data = target(self, buflen)
+
+            if len(args) == 1:
+                return data
+            else:
+                buffer = args[1]
+
+                buffer[0:len(data)] = data
+
+                return len(data)
+    return wrapper
+
+def write(target):
+    @functools.wraps(target)
+    def wrapper(self, byteslike):
+        return target(byteslike.tobytes())
+
+    return wrapper
+
+def readinto(target):
+    @functools.wraps(target)
+    def wrapper(self, buffer):
+        bytes = self.read(buffer.nbytes)
+        buffer[:len(bytes)] = bytes
+        return len(bytes)
+    return wrapper
+
+typewrappers = {
+    '_socket': {
+        'socket': {
+            'recvfrom_into': recvfrom_into,
+            'recv_into': recv_into,
+            'recvmsg_into': recvmsg_into
+        }
+    },
+    '_ssl': {
+        '_SSLSocket': {
+            'read': read,
+            # 'write': write
+        }
+    },
+    'io': {
+        'FileIO': {
+            'readinto': readinto 
+        },
+        'BufferedReader': {
+            'readinto': readinto
+        },
+        'BufferedRandom': {
+            'readinto': readinto
+        }
+    }
+}
+
+def patchtype(module, name, cls : type):
+    if module in typewrappers:
+        if name in typewrappers[module]:
+            for method,patcher in typewrappers[module][name].items():
+                setattr(cls, method, patcher(getattr(cls, method)))
+
+def transform_argument(target : callable, position : int, name : str, transform : callable, default = None):
+    assert callable(transform)
+    assert callable(target)
+
+    @functools.wraps(target)
+    def wrapper(*args, **kwargs):
+        if name in kwargs:
+            kwargs[name] = transform(kwargs[name])
+        elif len(args) > position:
+            args = list(args)
+            args[position] = transform(args[position])
+        else:
+            kwargs[name] = transform(default)
+
+        # print(f'Running: {args} {kwargs}')
+        return target(*args, **kwargs)
+
+    return wrapper 
+
+def retrace_env():
+    import threading
+
+    env = {
+        'RETRACE_RECORDING_PATH': globals.recording_path.next_subprocess_path(),
+        # 'RETRACE_PARENT_THREAD_ID': threading.current_thread().__retrace_thread_id__,
+        # 'RETRACE_PARENT_EXEC_COUNTER': str(next_exec_counter()
+        'RETRACE_MODE': 'record'
+        # 'RETRACE_MODE': 'proxy' if os.getenv('RETRACE_MODE') == 'proxy' else 'record'
+    }
+    # a timestamped directory off current directory
+    # controlled by config
+    # 1. if recording_dir is set use that
+    # 2. if recording_dir is not set look for 
+
+    # env['RETRACE_EXECUTION_ID'] = hashlib.md5(json.dumps(env).encode('UTF8')).hexdigest()
+
+    return env
+
+def fork_exec(target):
+    def transform(env):
+        r = [f'{k}={v}'.encode('utf-8') for k,v in retrace_env().items()]
+        return env + r if env else r
+
+    return transform_argument(target = target, 
+                              position = 5,
+                              name = 'env',
+                              transform = transform, default = os.environ) 
+
+def posix_spawn(target):
+    def transform(env):
+        r = retrace_env()
+        return {**env, **r}
+
+    return transform_argument(target = target, 
+                              position = 2,
+                              name = 'env',
+                              transform = transform, default = os.environ)
+
+function_patchers = {
+    '_posixsubprocess': {
+        'fork_exec': fork_exec
+    },
+    'posix': {
+        'posix_spawn': posix_spawn
+    }
+}
+
+# import traceback
+
+def typepatcher(cls : type):
+
+    # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! type: {cls} created')
+    # traceback.print_stack()
+    
+    return typewrappers.get(cls.__module__, {}).get(cls.__name__, {})
+
+    # if cls.__module__ in typewrappers:
+    #     # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TYPEWRAPPER for {cls}')
+
+    #     mod = typewrappers[cls.__module__]
+
+    #     return mod.get(cls.__name__, {})
+    
+    #     if cls.__name__ in mod:
+    #         # if cls.__name__ == '_SSLSocket':
+    #         #     breakpoint()
+
+    #         log.info("Applying specialized typewrapper to %s, updated slots: %s", cls, list(mod[cls.__name__].keys()))
+
+    #         for name,value in mod[cls.__name__].items():
+    #             setattr(cls, name, value(getattr(cls, name)))
+
+    #         # slots = {'__module__': cls.__module__, '__slots__': ()}
+    #         # slots.update(mod[cls.__name__])
+    #         # return type(cls.__name__, (cls, ), slots)
+
+    # return cls
+
+def typewrapper(cls : type):
+
+    # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! type: {cls} created')
+    # traceback.print_stack()
+    
+    if cls.__module__ in typewrappers:
+        # print(f'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TYPEWRAPPER for {cls}')
+
+        mod = typewrappers[cls.__module__]
+
+        if cls.__name__ in mod:
+            # if cls.__name__ == '_SSLSocket':
+            #     breakpoint()
+
+            log.info("Applying specialized typewrapper to %s, updated slots: %s", classname(cls), list(mod[cls.__name__].keys()))
+
+            slots = {'__module__': cls.__module__, '__slots__': ()}
+            slots.update(mod[cls.__name__])
+            return type(cls.__name__, (cls, ), slots)
+
+    return cls

retracesoftware/install/globals.py

@@ -0,0 +1,17 @@
+class RecordingPath:
+    def __init__(self, path):
+        self.path = path
+        self.subprocess_counter = 0
+        self.fork_counter = 0
+    
+    def next_subprocess_path(self):
+        path = self.path / f'subprocess-{self.subprocess_counter}'
+        self.subprocess_counter = self.subprocess_counter + 1
+        return path
+
+    def next_fork_path(self):
+        path = self.path / f'fork-{self.fork_counter}'
+        self.fork_counter = self.fork_counter + 1
+        return path
+
+recording_path = None