retracesoftware-proxy 0.1.21__py3-none-any.whl → 0.2.0__py3-none-any.whl

retracesoftware/install/patchfindspec.py

@@ -0,0 +1,117 @@
+from functools import wraps
+import os
+from pathlib import Path
+import shutil
+import sys
+
+# Set to track copied modules
+copied_modules = set()
+
+def get_relative_path(module_path: Path, sys_path: list) -> Path:
+    """Compute the relative path of module_path relative to sys.path entries."""
+    module_path = module_path.resolve()
+    for base in sys_path:
+        base_path = Path(base).resolve()
+        try:
+            if module_path.is_relative_to(base_path):
+                return module_path.relative_to(base_path)
+        except ValueError:
+            continue
+    return Path(module_path.name)
+
+class patch_find_spec:
+    def __init__(self, cwd, run_path, python_path):
+        self.cwd = cwd
+        self.run_path = run_path
+        self.python_path = python_path
+
+    def __call__(self, spec):
+        if spec is not None and spec.origin and spec.origin != "built-in" and os.path.isfile(spec.origin):
+            module_name = spec.name
+            if module_name not in copied_modules:
+                module_path = Path(spec.origin)
+
+                dest_path = None
+
+                if module_path.is_relative_to(self.cwd):
+                    dest_path = self.run_path / module_path.relative_to(self.cwd)
+                elif self.python_path:
+                    relative_path = get_relative_path(module_path, sys.path)
+                    dest_path = self.python_path / relative_path
+
+                if dest_path:
+                    dest_path.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(module_path, dest_path)
+                    copied_modules.add(module_name)
+
+    # def __call__(self, fullname, path, target=None):
+    #     spec = self.original_find_spec(fullname, path, target)
+
+    #     if spec is not None and spec.origin and spec.origin != "built-in" and os.path.isfile(spec.origin):
+    #         module_name = spec.name
+    #         if module_name not in copied_modules:
+    #             module_path = Path(spec.origin)
+
+    #             dest_path = None
+
+    #             if module_path.is_relative_to(self.cwd):
+    #                 dest_path = self.run_path / module_path.relative_to(self.cwd)
+    #             elif self.python_path:
+    #                 relative_path = get_relative_path(module_path, sys.path)
+    #                 dest_path = self.python_path / relative_path
+
+    #             if dest_path:
+    #                 dest_path.parent.mkdir(parents=True, exist_ok=True)
+    #                 shutil.copy2(module_path, dest_path)
+    #                 copied_modules.add(module_name)
+
+    #     return spec
+
+# def patch_find_spec(cwd, run_path, python_path, original_find_spec):
+#     """Create a patched version of find_spec that copies .py files."""
+#     @wraps(original_find_spec)
+#     def patched_find_spec(fullname, path, target=None):
+#         spec = original_find_spec(fullname, path, target)
+
+#         if spec is not None and spec.origin and spec.origin != "built-in" and os.path.isfile(spec.origin):
+#             module_name = spec.name
+#             if module_name not in copied_modules:
+#                 module_path = Path(spec.origin)
+
+#                 dest_path = None
+
+#                 if module_path.is_relative_to(cwd):
+#                     dest_path = run_path / module_path.relative_to(cwd)
+#                 elif python_path:
+#                     relative_path = get_relative_path(module_path, sys.path)
+#                     dest_path = python_path / relative_path
+
+#                 if dest_path:
+#                     dest_path.parent.mkdir(parents=True, exist_ok=True)
+#                     shutil.copy2(module_path, dest_path)
+#                     copied_modules.add(module_name)
+
+#                 # elif python_path:
+#                 #     relative_path = get_relative_path(module_path, sys.path)
+#                 #     dest_path = python_path / relative_path
+#                 #     dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+#                 #     shutil.copy2(module_path, dest_path)
+#                 #     copied_modules.add(module_name)
+
+#                     # if module_path.suffix == '.py':
+#                 # elif module_path.suffix == '.py':
+#                 #     relative_path = get_relative_path(module_path, sys.path)
+#                 #     dest_path = path / relative_path
+#                 #     dest_path.parent.mkdir(parents=True, exist_ok=True)
+#                 #     try:
+#                 #         shutil.copy2(module_path, dest_path)
+#                 #         print(f"Copied {module_path} to {dest_path} (relative: {relative_path})")
+#                 #         copied_modules.add(module_name)
+#                 #     except Exception as e:
+#                 #         print(f"Failed to copy {module_path}: {e}")
+#                 # else:
+#                 #     print(f"Skipped copying {module_path} (not a .py file)")
+
+#         return spec
+#     return patched_find_spec

retracesoftware/install/phases.py

@@ -0,0 +1,338 @@
+import retracesoftware.functional as functional
+import retracesoftware.utils as utils
+from retracesoftware.proxy.thread import start_new_thread_wrapper
+import threading
+import importlib
+from retracesoftware.install.typeutils import modify, WithFlags, WithoutFlags
+import enum
+
+from retracesoftware.install.typeutils import modify
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, Callable
+import functools
+
+def simple_phase(names, func):
+    """
+    Simplest phase type
+    """
+    return {name: func for name in names}
+
+def simple_patch_phase(names, func):
+    """
+    Simplest phase type
+    """
+    def side_effect(obj):
+        func(obj)
+        return obj
+
+    return {name: side_effect for name in names}
+
+Transform = Callable[[Any], Any]
+Config = Any
+
+Updater = Dict[str, Transform]
+
+Phase = Callable[[Config], Updater]
+
+Patcher = Dict[str, Phase]
+
+def update_class(cls : type, actions : Updater):
+    ...
+
+def type_attributes(patcher, config):
+    result = {}
+
+    for typename, action_attrs in config.items():
+
+        def patch_type(cls):
+            with modify(cls):
+                for action, attrs in action_attrs.items():
+                    update_class(cls, patcher[action](attrs))
+            return cls
+        
+        result[typename] = patch_type
+    
+    return result
+
+def create_patcher(system) -> Patcher:
+    patcher = {}
+
+    def simple_patcher(func): lambda config: {name: func for name in config}
+
+    patcher.update({
+        'type_attributes': functools.partial(type_attributes, patcher),
+        'disable': simple_patcher(system.disable),
+        'patch_types': simple_patcher(system.patch_type),
+
+    })
+    return patcher
+
+class TypeAttributesPhase(Phase):
+
+    def __init__(self, patcher):
+        super().__init__('type_attributes')
+        self.patcher = patcher
+
+    def patch_with_config(self, config, cls):
+        # print(f'TypeAttributesPhase: {config} {cls}')
+        if not isinstance(cls, type):
+            raise Exception("TODO")
+
+        with modify(cls):
+            for phase_name,values in config.items():
+                for attribute_name,func in self.patcher.find_phase(phase_name)(values).items():
+                    utils.update(cls, attribute_name, func)
+
+        return cls
+
+class Phase:
+    def __init__(self, system):
+        self.system = system
+
+        def simple_phase(func): return lambda names: {name: func for name in names}
+
+        self.phases = {
+            'disable': simple_phase(system.disable),
+            'type_attributes': lambda config: simple_phase(config, system.disable)
+        }
+
+    def type_attributes(self, config, cls):
+        with modify(cls):
+            for action_name, attribute_names in config.items():
+                for attribute_name, func in self(action_name, attribute_names).items():
+                    utils.update(cls, attribute_name, func)
+
+        return cls
+    
+    def __call__(self, name, config):
+        match name:
+            case 'disable':
+                return {name: self.system.disable for name in config}
+            case 'type_attributes':
+                return {cls: partial(self.type_attributes, actions) for cls, actions in config.items()}
+
+
+phases = {
+    'disable': lambda system, config: simple_phase(config, system.disable),
+    'type_attributes': lambda system, config: simple_phase(config, system.disable)
+}
+
+class Phase(ABC):
+    """
+    A phase is function when called with system and config yields map of name->func
+    A patch phase 
+    A phase has... keys... and action
+    """
+    def __init__(self, name):
+        self.name = name
+
+    def patch(self, obj):
+        return obj
+
+    def patch_with_config(self, config, obj):
+        return obj
+
+    @abstractmethod
+    def apply(self, name : str, value : Any) -> Any:
+        pass
+
+    @abstractmethod
+    def __call__(self, namespace : Dict[str, Any]) -> None:
+        for name, value in namespace.items():
+            if 
+
+    def __call__(self, config):
+        if isinstance(config, str):
+            return {config: self.patch}
+        elif isinstance(config, list):
+            return {key: self.patch for key in config}
+        elif isinstance(config, dict):
+            return {key: lambda obj: self.patch_with_config(value, obj) for key,value in config.items()}
+        else:
+            raise Exception(f'Unhandled config type: {config}')
+
+
+
+class Proxy(Phase):
+    
+    def proxy()
+    def __call__(self, name : str, value : Any) -> None:
+        if name in self.targets
+
+
+class Patch(Phase):
+
+    def __init__(self, targets, action):
+        self.targets = targets
+
+    def select(self, name : str) -> bool:
+        pass
+    
+    def __call__(self, name : str, value : Any) -> None:
+        if name in self.targets
+
+class SimplePhase(Phase):
+    def __init__(self, name, patch):
+        super().__init__(name)
+        self.patch = patch
+
+class DisablePhase(Phase):
+    def __init__(self, thread_state):
+        super().__init__('disable')
+        self.thread_state = thread_state
+
+    def patch(self, obj):
+        print(f'Disabling: {obj}')
+        return self.thread_state.wrap('disabled', obj)
+
+class TryPatchPhase(Phase):
+    def __init__(self, patcher):
+        super().__init__('try_patch')
+        self.patcher = patcher
+
+    def patch(self, obj):
+        try:
+            return self.patcher(obj)
+        except:
+            return obj
+
+class PatchTypesPhase(Phase):
+    def __init__(self, patcher):
+        super().__init__('patch_types')
+        self.patcher = patcher
+    
+    def patch(self, obj):
+        if not isinstance(obj, type):
+            raise Exception("TODO")
+
+        self.patcher(obj)
+        return obj
+
+class BindPhase(Phase):
+    def __init__(self, bind):
+        super().__init__('bind')
+        self.bind = bind
+
+    def patch(self, obj):
+        # print(f'In BindPhase: {obj} {isinstance(obj, type)}')
+        if issubclass(obj, enum.Enum):
+            for member in obj:
+                # print(f'binding member: {member}')
+                # utils.sigtrap(member)
+                self.bind(member)
+        else:
+            self.bind(obj)
+
+        return obj
+
+class ImmutableTypePhase(Phase):
+
+    def __init__(self, types):
+        super().__init__('immutable')
+        self.types = types
+
+    def patch(self, obj):
+        if not isinstance(obj, type):
+            raise Exception("TODO")
+
+        self.types.add(obj)
+        return obj
+
+class PatchThreadPhase(Phase):
+    def __init__(self, thread_state, on_exit):
+        super().__init__('patch_start_new_thread')
+        self.thread_state = thread_state
+        self.on_exit = on_exit
+
+    def patch(self, obj):
+        return start_new_thread_wrapper(thread_state = self.thread_state, 
+                                        on_exit = self.on_exit,
+                                        start_new_thread = obj)
+
+def resolve(path):
+    module, sep, name = path.rpartition('.')
+    if module == None: module = 'builtins'
+    
+    return getattr(importlib.import_module(module), name)
+
+class WrapPhase(Phase):
+    def __init__(self):
+        super().__init__('wrap')
+
+    def patch_with_config(self, wrapper, obj):
+        return resolve(wrapper)(obj)
+
+class PatchClassPhase(Phase):
+    def __init__(self):
+        super().__init__('patch_class')
+
+    def patch_with_config(self, config, cls):
+
+        patchers = utils.map_values(resolve, config)
+
+        assert cls is not None
+
+        with WithoutFlags(cls, "Py_TPFLAGS_IMMUTABLETYPE"):
+            for name,func in patchers.items():                
+                utils.update(cls, name, func)
+
+        return cls
+
+class ProxyWrapPhase(Phase):
+    def __init__(self):
+        super().__init__('wrap_proxy')
+
+    def patch_with_config(self, config, cls):
+
+        patchers = utils.map_values(resolve, config)
+        
+        def patch(proxytype):
+            for name,func in patchers.items():
+                utils.update(proxytype, name, func)
+
+        cls.__retrace_patch_proxy__ = patch
+
+        return cls
+        # return resolve(wrapper)(obj)
+
+class TypeAttributesPhase(Phase):
+
+    def __init__(self, patcher):
+        super().__init__('type_attributes')
+        self.patcher = patcher
+
+    def patch_with_config(self, config, cls):
+        # print(f'TypeAttributesPhase: {config} {cls}')
+        if not isinstance(cls, type):
+            raise Exception("TODO")
+
+        with modify(cls):
+            for phase_name,values in config.items():
+                for attribute_name,func in self.patcher.find_phase(phase_name)(values).items():
+                    utils.update(cls, attribute_name, func)
+
+        return cls
+
+class PerThread(threading.local):
+    def __init__(self):
+        self.internal = utils.counter()
+        self.external = utils.counter()
+
+class PatchHashPhase(Phase):
+
+    def __init__(self, thread_state):
+        super().__init__('patch_hash')
+
+        per_thread = PerThread()
+        
+        self.hashfunc = thread_state.dispatch(
+            functional.constantly(None),
+            internal = functional.repeatedly(functional.partial(getattr, per_thread, 'internal')),
+            external = functional.repeatedly(functional.partial(getattr, per_thread, 'external')))
+
+    def patch(self, obj):
+        if not isinstance(obj, type):
+            raise Exception("TODO")
+
+        utils.patch_hash(cls = obj, hashfunc = self.hashfunc)
+        return obj

retracesoftware/install/record.py

@@ -6,23 +6,34 @@ import retracesoftware.stream as stream
 
 from retracesoftware.install.tracer import Tracer
 from retracesoftware.install import globals
+from retracesoftware.install.config import env_truthy
+from retracesoftware.install.patchfindspec import patch_find_spec
 
 import os
 import sys
 from datetime import datetime
 import json
 from pathlib import Path
+import shutil
 
 # class ThreadSwitch:
 #     def __init__(self, id):
 #         self.id = id
-    
+
 #     def __repr__(self):
 #         return f'ThreadSwitch<{self.id}>'
 
 #     def __str__(self):
 #         return f'ThreadSwitch<{self.id}>'
 
+def code_workspace():
+    return {
+        'folders': [
+            {'path': '../..', 'name': 'Application'},
+            {'path': '.', 'name': 'Recording'}
+        ]
+    }
+
 def write_files(recording_path):
     with open(recording_path / 'env', 'w') as f:
         json.dump(dict(os.environ), f, indent=2)
@@ -36,6 +47,9 @@ def write_files(recording_path):
     with open(recording_path / 'cmd', 'w') as f:
         json.dump(sys.orig_argv, f, indent=2)
 
+    with open(recording_path / 'replay.code-workspace', 'w') as f:
+        json.dump(code_workspace(), f, indent=2)
+    
 def create_recording_path(path):
     expanded = datetime.now().strftime(path.format(pid = os.getpid()))
     os.environ['RETRACE_RECORDING_PATH'] = expanded
@@ -55,10 +69,14 @@ def merge_config(base, override):
         return override
 
 
+
+def dump_as_json(path, obj):
+    with open(path, 'w') as f:
+        json.dump(obj, f, indent=2)
+
 def record_system(thread_state, immutable_types, config):
 
     recording_path = create_recording_path(config['recording_path'])
-
     recording_path.mkdir(parents=True, exist_ok=True)
 
     globals.recording_path = globals.RecordingPath(recording_path)
@@ -67,48 +85,90 @@ def record_system(thread_state, immutable_types, config):
 
     tracing_config = config['tracing_levels'].get(tracing_level(config), {})
 
-    with open(recording_path / 'tracing_config.json', 'w') as f:
-        json.dump(tracing_config, f, indent=2)
+    dump_as_json(path = recording_path / 'tracing_config.json', obj = tracing_config)
 
     def write_main_path(path):
         with open(recording_path / 'mainscript', 'w') as f:
             f.write(path)
 
-    # writer = stream.writer(path = recording_path / 'trace.bin')
-    
-    # os.register_at_fork(
-    #     # before = self.thread_state.wrap('disabled', self.before_fork),
-    #     before = before,
-    #     after_in_parent = self.thread_state.wrap('disabled', self.after_fork_in_parent),
-    #     after_in_child = self.thread_state.wrap('disabled', self.after_fork_in_child))
-
-    # self.writer = thread_state.wrap(
-    #     desired_state = 'disabled',
-    #     sticky = True,
-    #     function = VerboseWriter(writer)) if verbose else writer
-
-    # def gc_start(self):
-    #     self.before_gc = self.thread_state.value
-    #     self.thread_state.value = 'external'
-
-    # def gc_end(self):
-    #     self.thread_state.value = self.before_gc
-    #     del self.before_gc
-
-    # def gc_hook(self, phase, info):
-    #     if phase == 'start':
-    #         self.gc_start()
-
-    #     elif phase == 'stop':
-    #         self.gc_end()
-    # gc.callbacks.append(self.gc_hook)
-
-    # print(f'Tracing config: {tracing_config(config)}')
-
-    # tracer = Tracer(config = tracing_config(config), writer = writer.handle('TRACE'))
+    run_path = recording_path / 'run'
+    run_path.mkdir(parents=True, exist_ok=False)
+
+    shutil.copy2(sys.argv[0], run_path)
+
+    python_path = recording_path / 'pythonpath'
+    python_path.mkdir(parents=True, exist_ok=False)
+
+    vscode = recording_path / '.vscode'
+    vscode.mkdir(parents=True, exist_ok=False)
+
+    # launch_json = {
+    #     # // Use IntelliSense to learn about possible attributes.
+    #     # // Hover to view descriptions of existing attributes.
+    #     # // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    #     "version": "0.2.0",
+    #     "configurations": [
+    #         {
+    #             "name": "Python Debugger: Python File",
+    #             "type": "debugpy",
+    #             "request": "launch",
+    #             "cwd": "${workspaceFolder}/run",
+    #             "env": {
+    #                 "RETRACE_MODE": "replay"
+    #             },
+    #             "python": sys.executable,
+    #             "program": sys.argv[0]
+    #         }
+    #     ]
+    # }
+
+    # dump_as_json(vscode / 'launch.json', launch_json)
+
+    workspace = {
+        "folders": [{ 'path': '.' }],
+        "settings": {
+            # This is the one that works reliably in .code-workspace files
+            # "python.defaultInterpreterPath":
+            "python.defaultInterpreterPath": sys.executable,
+            # "python.interpreterInfo": {
+            #     "run": {"path": sys.executable}
+            # }
+        },
+        "launch": {
+            "version": "0.2.0",
+            "configurations": [
+                {
+                    "justMyCode": False,
+                    "name": "Python Debugger: Python File",
+                    "type": "debugpy",
+                    "request": "launch",
+                    "cwd": "${workspaceFolder}/run",
+                    "env": {
+                        "RETRACE_RECORDING_PATH": "..",
+                        "RETRACE_MODE": "replay"
+                    },
+                    # "python": ["${command:python.interpreterPath}"],
+                    "python": sys.executable,
+                    # "python": sys.executable,
+                    "program": sys.argv[0]
+                }
+            ]
+        },
+    }
+
+    dump_as_json(recording_path / 'replay.code-workspace', workspace)
+
+    copy_source = thread_state.wrap('disabled', patch_find_spec(cwd = Path(os.getcwd()), run_path = run_path, python_path = python_path))
+
+    for finder in sys.meta_path:
+        finder.find_spec = functional.sequence(finder.find_spec, functional.side_effect(copy_source))
 
     return RecordProxySystem(thread_state = thread_state,
                              immutable_types = immutable_types, 
                              tracing_config = tracing_config,
                              write_main_path = write_main_path,
-                             path = recording_path / 'trace.bin')
+                             path = recording_path / 'trace.bin',
+                             tracecalls = env_truthy('RETRACE_ALL', False),
+                             verbose = env_truthy('RETRACE_VERBOSE', False),
+                             stacktraces = env_truthy('RETRACE_STACKTRACES', False),
+                             magic_markers = env_truthy('RETRACE_MAGIC_MARKERS', False))

retracesoftware/install/replace.py

@@ -0,0 +1,28 @@
+import gc
+
+def container_replace(container, old, new):
+    if isinstance(container, dict):
+        if old in container:
+            elem = container.pop(old)
+            container[new] = elem
+            container_replace(container, old, new)
+        else:
+            for key,value in container.items():
+                if key != '__retrace_unproxied__' and value is old:
+                    container[key] = new
+        return True
+    elif isinstance(container, list):
+        for i,value in enumerate(container):
+            if value is old:
+                container[i] = new
+        return True
+    elif isinstance(container, set):
+        container.remove(old)
+        container.add(new)
+        return True
+    else:
+        return False
+
+def update(old, new):
+    for ref in gc.get_referrers(old):
+        container_replace(container = ref, old = old, new = new)