retracesoftware-proxy 0.1.0__py3-none-any.whl

retracesoftware/install/predicate.py

@@ -0,0 +1,92 @@
+from retracesoftware_functional import *
+import re
+import pdb
+import weakref
+from types import MappingProxyType
+
+class RegexPredicate:
+    __slots__ = ['regex']
+
+    def __init__(self, regex):
+        self.regex = re.compile(regex)
+
+    def __hash__(self):
+        return hash(self.regex)
+    
+    def __eq__(self, other):
+        if not isinstance(other, RegexPredicate):
+            return NotImplemented  # Allows Python to attempt `other.__eq__(self)`
+        return self.regex == other.regex
+    
+    def __str__(self):
+        return repr(self.regex.pattern)
+    
+    def __call__(self, obj):
+        return bool(re.fullmatch(self.regex, str(obj)))
+
+def canonical(spec):
+    if isinstance(spec, str):
+        return ('regex', spec)
+    elif isinstance(spec, list):
+        return ('or',) + tuple(map(canonical, spec))
+    elif 'not' in spec:
+        return ('not', canonical(spec['not']))
+    elif 'and' in spec:
+        return ('and',) + tuple(map(canonical, spec['and']))
+    elif 'or' in spec:
+        return ('or',) + tuple(map(canonical, spec['or']))
+    else:
+        pdb.set_trace()
+        raise Exception()    
+
+class PredicateBuilder:
+    __slots__ = ['cache']
+
+    def __init__(self):
+        # self.cache = weakref.WeakKeyDictionary()
+        self.cache = {}
+
+    def build(self, spec):
+        if spec in self.cache:
+            return self.cache[spec]
+    
+        if spec[0] == 'regex':
+            pred = RegexPredicate(spec[1])
+        elif spec[0] == 'not':
+            pred = not_predicate(self.build(spec[1]))
+        elif spec[0] == 'and':
+            pred = and_predicate(*map(self.build, spec[1:]))
+        elif spec[0] == 'or':
+            pred = or_predicate(*map(self.build, spec[1:]))
+        else:
+            raise Exception("TODO")
+        
+        self.cache[spec] = pred
+
+        return pred
+            
+    def __call__(self, spec):
+        return self.build(canonical(spec))
+
+class ParamPredicate:
+    __slots__ = ['pred', 'index', 'param']
+
+    def __init__(self, signature, param, pred):
+        # params = inspect.signature(target).parameters
+        self.index = list(signature.keys()).index(param)
+        self.param = param
+        self.pred = pred
+
+    def __str__(self):
+        return f'ParamPredicate(pred = {self.pred}, index = {self.index}, param = {self.param})'
+    
+    def find_arg(self, *args, **kwargs):
+        if self.param in kwargs:
+            return kwargs[self.param]
+        elif self.index < len(args):
+            return args[self.index]
+        else:
+            raise Exception(f'Cant get arg: {self.param}')
+
+    def __call__(self, *args, **kwargs):
+        return self.pred(self.find_arg(*args, **kwargs))

retracesoftware/install/record.py

@@ -0,0 +1,128 @@
+from retracesoftware.proxy import *
+
+import retracesoftware.functional as functional
+import retracesoftware_utils as utils
+import retracesoftware.stream as stream
+
+from retracesoftware.install.tracer import Tracer
+from retracesoftware.install import globals
+
+import os
+import sys
+from datetime import datetime
+import json
+from pathlib import Path
+
+def write_files(recording_path):
+    with open(recording_path / 'env', 'w') as f:
+        json.dump(dict(os.environ), f, indent=2)
+
+    with open(recording_path / 'exe', 'w') as f:
+        f.write(sys.executable)
+
+    with open(recording_path / 'cwd', 'w') as f:
+        f.write(os.getcwd())
+
+    with open(recording_path / 'cmd', 'w') as f:
+        json.dump(sys.orig_argv, f, indent=2)
+
+def create_recording_path(path):
+    expanded = datetime.now().strftime(path.format(pid = os.getpid()))
+    os.environ['RETRACE_RECORDING_PATH'] = expanded
+    return Path(expanded)
+
+def tracing_level(config):
+    return os.environ.get('RETRACE_DEBUG', config['default_tracing_level'])
+
+# def tracing_config(config):
+#     level = os.environ.get('RETRACE_DEBUG', config['default_tracing_level'])
+#     return config['tracing_levels'].get(level, {})
+
+def thread_aware_writer(writer):
+    on_thread_switch = functional.sequence(utils.thread_id(), writer.handle('THREAD_SWITCH'))
+    return utils.threadawareproxy(on_thread_switch = on_thread_switch, target = writer)
+
+def record_system(thread_state, immutable_types, config):
+
+    recording_path = create_recording_path(config['recording_path'])
+
+    recording_path.mkdir(parents=True, exist_ok=True)
+
+    globals.recording_path = globals.RecordingPath(recording_path)
+
+    write_files(recording_path)
+
+    tracing_config = config['tracing_levels'].get(tracing_level(config), {})
+
+    with open(recording_path / 'tracing_config.json', 'w') as f:
+        json.dump(tracing_config, f, indent=2)
+
+    writer = thread_aware_writer(stream.writer(path = recording_path / 'trace.bin'))
+    
+    # os.register_at_fork(
+    #     # before = self.thread_state.wrap('disabled', self.before_fork),
+    #     before = before,
+    #     after_in_parent = self.thread_state.wrap('disabled', self.after_fork_in_parent),
+    #     after_in_child = self.thread_state.wrap('disabled', self.after_fork_in_child))
+
+    # self.writer = thread_state.wrap(
+    #     desired_state = 'disabled',
+    #     sticky = True,
+    #     function = VerboseWriter(writer)) if verbose else writer
+
+    # def gc_start(self):
+    #     self.before_gc = self.thread_state.value
+    #     self.thread_state.value = 'external'
+
+    # def gc_end(self):
+    #     self.thread_state.value = self.before_gc
+    #     del self.before_gc
+
+    # def gc_hook(self, phase, info):
+    #     if phase == 'start':
+    #         self.gc_start()
+
+    #     elif phase == 'stop':
+    #         self.gc_end()
+    # gc.callbacks.append(self.gc_hook)
+
+    w = writer.handle('TRACE')
+    def trace_writer(*args):
+        print(f'Trace: {args}')
+        w(*args)
+
+    # print(f'Tracing config: {tracing_config(config)}')
+
+    tracer = Tracer(tracing_config, writer = trace_writer)
+    # tracer = Tracer(config = tracing_config(config), writer = writer.handle('TRACE'))
+
+    factory = RecordProxySystem(thread_state = thread_state,
+                                immutable_types = immutable_types, 
+                                tracer = tracer,
+                                writer = writer)
+    
+    # factory.proxy_type = compose(factory.proxy_type, side_effect(...))
+
+    def on_patched(module_name, updates):
+        ...
+        # for name, value in updates.items():
+        #     if isinstance(value, type) and \
+        #        issubclass(value, ExtendingProxy) and \
+        #        not issubclass(value, InternalProxy):
+            
+        #         ref = writer.handle(ExtendingProxySpec(module = module_name, name = name))
+        #         try:
+        #             writer.add_type_serializer(cls = value, serializer = functional.constantly(ref))
+        #         except:
+        #             pass
+
+    factory.on_patched = on_patched
+
+    factory.sync = lambda function: functional.observer(on_call = functional.always(writer.handle('SYNC')), function = function)
+
+    # factory.set_thread_number = writer.thread_number
+    factory.tracer = tracer
+
+    factory.log = functional.partial(writer, 'LOG')
+
+    return factory

retracesoftware/install/references.py

@@ -0,0 +1,65 @@
+import gc
+import sys
+import pdb
+
+def references(obj):
+    refcnt = sys.getrefcount(obj) - 2
+    
+    if refcnt == 1:
+        moddict = sys.modules[obj.__module__].__dict__
+
+        if obj.__name__ in moddict and moddict[obj.__name__] is obj:
+            return [moddict]
+    else:
+        refs = []
+
+        for mod in list(sys.modules.values()):
+            moddict = mod.__dict__
+            if obj.__name__ in moddict and moddict[obj.__name__] is obj:
+                refs.append(moddict)
+
+        if len(refs) == refcnt:
+            return refs
+
+    return gc.get_referrers(obj)
+    
+def replace(container, old, new):
+    if isinstance(container, dict):
+        if old in container:
+            elem = container.pop(old)
+            container[new] = elem
+            replace(container, old, new)
+        else:
+            for key,value in container.items():
+                if value is old:
+                    container[key] = new
+                    
+    elif isinstance(container, list):
+        for i,value in enumerate(container):
+            if value is old:
+                container[i] = new
+
+    elif isinstance(container, set):
+        container.remove(old)
+        container.add(new)
+
+    # elif isinstance(container, tuple):
+    #     pdb.set_trace()
+    # else:
+    #     pdb.set_trace()
+    #     raise Exception('TODO')
+
+def update(f, obj):
+    new = f(obj)
+    if new is not obj:
+        refs = references(obj)
+        for ref in refs:
+            replace(ref, obj, new)
+
+    return new
+
+# import math
+# import os
+# print(references(math.sqrt))
+# print(references(os.nice))
+# print(references(os.open))

retracesoftware/install/replay.py

@@ -0,0 +1,151 @@
+from retracesoftware.proxy import *
+
+import retracesoftware.functional as functional
+import retracesoftware_utils as utils
+import retracesoftware.stream as stream
+from retracesoftware.install.tracer import Tracer
+from retracesoftware.install import globals
+
+import os, json, re, glob, weakref
+from pathlib import Path
+from datetime import datetime
+
+def latest_from_pattern(pattern: str) -> str | None:
+    """
+    Given a strftime-style filename pattern (e.g. "recordings/%Y%m%d_%H%M%S_%f"),
+    return the path to the most recent matching file, or None if no files exist.
+    """
+    # Turn strftime placeholders into '*' for globbing
+    # (very simple replacement: %... -> *)
+    glob_pattern = re.sub(r"%[a-zA-Z]", "*", pattern)
+
+    # Find all matching files
+    candidates = glob.glob(glob_pattern)
+    if not candidates:
+        return None
+
+    # Derive the datetime format from the pattern (basename only)
+    base_pattern = os.path.basename(pattern)
+
+    def parse_time(path: str):
+        name = os.path.basename(path)
+        return datetime.strptime(name, base_pattern)
+
+    # Find the latest by parsed timestamp
+    latest = max(candidates, key=parse_time)
+    return latest
+
+
+
+def thread_aware_reader(reader):
+    def on_thread_switch():
+        ...
+
+    return utils.threadawareproxy(on_thread_switch = on_thread_switch, target = reader)
+
+
+def replay_system(thread_state, immutable_types, config):
+
+    recording_path = Path(latest_from_pattern(config['recording_path']))
+
+    print(f"replay running against path: {recording_path}")
+
+    globals.recording_path = globals.RecordingPath(recording_path)
+
+    assert recording_path.exists()
+    assert recording_path.is_dir()
+
+    with open(recording_path / "env", "r", encoding="utf-8") as f:
+        os.environ.update(json.load(f))
+
+    with open(recording_path / "tracing_config.json", "r", encoding="utf-8") as f:
+        tracing_config = json.load(f)
+
+    handler = None
+
+    # def create_stub_type(proxyspec):
+    #     if isinstance(proxyspec, ExtendingProxySpec):
+    #         mod = sys.modules[proxyspec.module]
+    #         resolved = getattr(mod, proxyspec.name) 
+    #         assert issubclass(resolved, ExtendingProxy)
+
+    #         return resolved
+    #         # unproxied = resolved.__base__            
+    #         # print(f'base: {unproxied} has_generic_new: {utils.has_generic_new(unproxied)}')
+    #         # print(f'base: {unproxied} has_generic_alloc: {utils.has_generic_alloc(unproxied)}')
+    #         # utils.create_stub_object(resolved)
+    #         # os._exit(1)
+    #         # return resolved
+
+    #     elif isinstance(proxyspec, WrappingProxySpec):            
+    #         nonlocal handler
+    #         return proxyspec.create_type(handler)
+    #     else:
+    #         print(f'In create_stub_type!!!! {proxyspec}')
+    #         os._exit(1)
+
+    # deserializer = functional.compose(pickle.loads, functional.when_instanceof(ProxySpec, create_stub_type))
+
+    reader = thread_aware_reader(stream.reader(path = recording_path / 'trace.bin'))
+
+    # reader = utils.threadawareproxy(on_thread_switch = ..., target = reader)
+
+    def readnext():
+        return reader()
+        # print(f'read: {obj}')
+        # return obj
+
+    lookup = weakref.WeakKeyDictionary()
+    
+    # debug = debug_level(config)
+
+    # int_refs = {}
+        
+    def checkpoint(replay):
+        ...
+
+    def read_required(required):
+        obj = readnext()
+        if obj != required:
+            print(f'Expected: {required} but got: {obj}')
+            for i in range(5):
+                readnext()
+
+            utils.sigtrap(None)
+            os._exit(1)
+            raise Exception(f'Expected: {required} but got: {obj}')
+
+    def trace_writer(name, *args):
+        print(f'Trace: {name} {args}')
+        
+        read_required('TRACE')
+        read_required(name)
+
+        for arg in args:
+            read_required(arg)
+
+    tracer = Tracer(tracing_config, writer = trace_writer)
+
+    factory = ReplayProxySystem(thread_state = thread_state, 
+                                immutable_types = immutable_types,
+                                tracer = tracer,
+                                reader = reader)
+    
+    # factory = replaying_proxy_factory(thread_state = thread_state, 
+    #                                   is_immutable_type = is_immutable_type,
+    #                                   tracer = tracer,
+    #                                   bind = reader.supply,
+    #                                   next = next_result, 
+    #                                   checkpoint = checkpoint)
+    
+    factory.tracer = tracer
+
+    def read_sync(): read_required('SYNC')
+
+    factory.sync = lambda function: functional.observer(on_call = functional.always(read_sync), function = function)
+
+    # factory.set_thread_number = writer.thread_number
+
+    factory.log = lambda message: checkpoint({'type': 'log_message', 'message': message})
+
+    return factory

retracesoftware/install/tracer.py

@@ -0,0 +1,144 @@
+import retracesoftware.functional as functional
+import retracesoftware_utils as utils
+from retracesoftware.proxy.proxytype import Proxy
+
+import types
+import os
+
+def format_kwargs(kwargs):
+    result = []
+    for k, v in kwargs.items():
+        result.extend([f"{k} = ", v])
+
+    return tuple(result)
+
+class Tracer:
+    def __init__(self, config, writer):
+        self.config = config
+        serialize = functional.walker(self.serialize)
+        self.writer = functional.mapargs(transform = serialize, function = writer)
+
+    def serialize(self, obj):
+        try:
+            if obj is None: return None
+
+            cls = functional.typeof(obj)
+
+            if issubclass(cls, (int, str)):
+                return obj
+
+            else:
+                return 'FOO!!!!'
+            
+            if issubclass(cls, Proxy):
+                return f'<Proxy>'
+
+            if issubclass(cls, types.TracebackType):
+                return '<traceback>'
+            
+            elif issubclass(cls, utils.wrapped_function):
+                return utils.unwrap(obj).__name__
+            
+            elif hasattr(obj, '__module__') and hasattr(obj, '__name__'):
+                return f'{obj.__module__}.{obj.__name__}'
+            # elif isinstance(obj, type):
+            #     return f'{obj.__module__}.{obj.__name__}'
+            else:
+                return '<other>'
+                # return f'instance type: {str(self.serialize(type(obj)))}'
+        except:
+            print("ERROR in tracer serialize!!!!")
+            os._exit(1)
+
+    def log(self, name, message):
+        if name in self.config:
+            self.writer(name, message)
+         
+    def __call__(self, name, func = None):
+        if name in self.config:
+            if name.endswith('.call'):
+                def write_call(*args, **kwargs):
+                    self.writer(name, args[0].__name__, args[1:], kwargs)
+            
+                return functional.firstof(write_call, func) if func else write_call
+            
+            elif name.endswith('.result'):
+                def write_result(obj):
+                    self.writer(name, obj)
+                    return obj
+                
+                return functional.sequence(func, write_result) if func else write_result
+            
+            elif name.endswith('.error'):
+                def write_error(*args):
+                    self.writer(name, args)
+            
+                return functional.firstof(write_error, func) if func else write_error
+
+            elif name.endswith('.event'):
+                def write_event(*args, **kwargs):
+                    self.writer(name)
+
+                return functional.firstof(write_event, func) if func else write_event
+        
+            elif name.endswith('.wrap'):
+                def wrapper(*args, **kwargs):
+                    self.writer(name, 'enter')
+                    try:
+                        return func(*args, **kwargs)
+                    finally:
+                        self.writer(name, 'exit')
+
+                return wrapper
+            
+            elif name.endswith('.stack'):
+                def write_event(*args, **kwargs):
+                    self.writer(name, utils.stacktrace())
+
+                return functional.firstof(write_event, func) if func else write_event
+
+
+        return func
+
+
+    def write(self, name):
+        if name in self.config:
+            return functional.partial(self.writer, name)
+
+    def write_call(self, name, func = None):
+
+        def writer(*args, **kwargs):
+            self.writer(name, *(args + format_kwargs(kwargs)))
+
+        if name in self.config:
+            return functional.firstof(writer, func) if func else writer
+        else:
+            return func
+
+    def write_result(self, name, func):
+        
+        writer = functional.partial(self.writer, name)
+            
+        if name in self.config:
+            return functional.compose(func, functional.side_effect(writer))
+        else:
+            return func
+
+    def event(self, name):
+        return functional.always(lambda: self.writer(name))
+
+    def event_before(self, name, func = None):
+        if name in self.config:
+            # def foo(*args, **kwargs):
+            #     print("GRRRRR!!!!!")
+
+            # return functional.firstof(foo, func)
+            return functional.firstof(self.event(name), func)
+        else:
+            return func
+
+    def event_after(self, name, func):
+        if name in self.config:
+            return functional.compose(func, functional.side_effect(self.event(name)))
+        else:
+            return func

retracesoftware/install/typeutils.py

@@ -0,0 +1,72 @@
+# from retrace_utils import _intercept
+import retracesoftware_utils as utils
+
+def flags(cls : type):
+    f = utils.type_flags(cls)
+
+    s = set()
+
+    for name,value in utils.TypeFlags.items():
+        if (f & value) != 0:
+            s.add(name)
+        f = f & ~value
+
+    if f != 0:
+        s.add(f)
+
+    return s
+
+class WithoutFlags:
+
+    def __init__(self, cls  , *flags):
+        self.cls = cls
+        self.flags = flags
+
+    def __enter__(self):
+        self.saved = utils.type_flags(self.cls)
+        flags = self.saved
+
+        for flag in self.flags:
+            flags = flags & ~utils.TypeFlags[flag]
+
+        utils.set_type_flags(self.cls, flags)
+        return self.cls
+
+    def __exit__(self, *args):
+        utils.set_type_flags(self.cls, self.saved)
+
+def add_flag(cls : type, flag):
+    if isinstance(flag, str):
+        flag = utils.TypeFlags[flag]
+    
+    flags = utils.type_flags(cls)
+    utils.set_type_flags(cls, flags | flag)
+
+def modify(cls):
+    return WithoutFlags(cls, "Py_TPFLAGS_IMMUTABLETYPE")
+
+def type_has_feature(cls, flag):
+    if isinstance(flag, str):
+        flag = utils.TypeFlags[flag]
+    return (utils.type_flags(cls) & flag) != 0
+
+def type_disallow_instantiation(cls : type):
+    return utils.type_flags(cls) & utils.Py_TPFLAGS_DISALLOW_INSTANTIATION
+
+def is_method_descriptor(cls : type):
+    return (utils.type_flags(cls) & utils.TypeFlags["Py_TPFLAGS_METHOD_DESCRIPTOR"]) != 0
+
+def extend_type(cls : type):
+    subclasses = list(cls.__subclasses__())
+
+    utils.make_extensible(cls)
+
+    print(f'extending: {cls}')
+    
+    extended = utils.extend_type(cls)
+
+    for subclass in subclasses:
+        print(f"updating subclass: {subclass}")
+        subclass.__bases__ = tuple(map(lambda x: extended if x is cls else x, subclass.__bases__))        
+
+    return extended

retracesoftware/proxy/__init__.py

@@ -0,0 +1,3 @@
+from retracesoftware.proxy.record import RecordProxySystem
+from retracesoftware.proxy.replay import ReplayProxySystem
+