retracesoftware-proxy 0.1.21__py3-none-any.whl → 0.2.0__py3-none-any.whl

retracesoftware/__main__.py

@@ -0,0 +1,266 @@
+import sys
+import runpy
+import os
+import argparse
+from typing import Tuple, List
+import retracesoftware.utils as utils
+import retracesoftware.functional as functional
+from retracesoftware.stackdifference import on_stack_difference
+from pathlib import Path
+from retracesoftware.proxy.record import RecordProxySystem
+from retracesoftware.proxy.replay import ReplayProxySystem
+import retracesoftware.stream as stream
+from retracesoftware.proxy.startthread import thread_id
+import datetime
+import json
+from shutil import copy2
+import threading
+import time
+import gc
+import hashlib
+
+from retracesoftware.run import install, run_with_retrace, ImmutableTypes, thread_states
+
+def expand_recording_path(path):
+    return datetime.datetime.now().strftime(path.format(pid = os.getpid()))
+
+def load_json(file):
+    with open(file, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+def dump_as_json(path, obj):
+    with open(path, 'w') as f:
+        json.dump(obj, f, indent=2)
+
+vscode_workspace = {
+    "folders": [{ 'path': '.' }],
+    "settings": {
+        "python.defaultInterpreterPath": sys.executable,
+    },
+    "launch": {
+        "version": "0.2.0",
+        "configurations": [{
+            "name": "replay",
+            "type": "debugpy",
+            "request": "launch",
+            
+            "python": sys.executable,
+            "module": "retracesoftware",
+            "args": [
+                "--recording", "..",
+                "--skip_weakref_callbacks",
+                "--read_timeout", "1000"
+            ],
+            
+            "cwd": "${workspaceFolder}/run",
+            "console": "integratedTerminal",
+            "justMyCode": False
+        }]
+    },
+}
+
+def scriptname(argv):
+    return argv[1] if argv[0] == "-m" else argv[0]
+
+def collector(multiplier):
+    collect_gen = utils.CollectPred(multiplier = multiplier)
+
+    return functional.lazy(functional.sequence(collect_gen, functional.when_not_none(gc.collect)))
+
+def file_md5(path):
+    return hashlib.md5(path.read_bytes()).hexdigest()
+
+def checksum(path):
+    return file_md5(path) if path.is_file() else {entry.name: checksum(entry) for entry in path.iterdir() if entry.name != '__pycache__'}
+
+def retrace_extension_paths():
+    names = ['retracesoftware_utils', 'retracesoftware_functional', 'retracesoftware_stream']
+    return {name: Path(sys.modules[name].__file__) for name in names}
+
+def retrace_module_paths():
+    paths = retrace_extension_paths()
+    paths['retracesoftware'] = Path(sys.modules['retracesoftware'].__file__).parent
+    return paths
+
+def checksums():
+    return {name: checksum(path) for name, path in retrace_module_paths().items()}
+
+def record(options, args):
+    
+    path = Path(expand_recording_path(options.recording))
+    # ensure the path exists
+    path.mkdir(parents=True, exist_ok=True) 
+
+    # write various recording files to directory
+    dump_as_json(path / 'settings.json', {
+        'argv': args,
+        'magic_markers': options.magic_markers,
+        'trace_inputs': options.trace_inputs,
+        'trace_shutdown': options.trace_shutdown,
+        'env': dict(os.environ),
+        'python_version': sys.version,
+        'md5_checksums': checksums(),
+    })
+
+    rundir = path / 'run'
+    rundir.mkdir(exist_ok=True)
+
+    script = Path(scriptname(args))
+    if script.exists():
+        copy2(script, rundir)
+
+    dump_as_json(path / 'replay.code-workspace', vscode_workspace)
+
+    with stream.writer(path = path / 'trace.bin',
+                       thread = thread_id, 
+                       verbose = options.verbose, 
+                       stacktraces = options.stacktraces, 
+                       magic_markers = options.magic_markers) as writer:
+
+        # flusher = threading.Timer(5.0, periodic_task)
+        # flusher.start()
+
+        writer.exclude_from_stacktrace(record)
+        writer.exclude_from_stacktrace(main)
+
+        thread_state = utils.ThreadState(*thread_states)
+
+        tracing_config = {}
+
+        multiplier = 2
+        gc.set_threshold(*map(lambda x: x * multiplier, gc.get_threshold()))
+
+        system = RecordProxySystem(
+            writer = writer,
+            thread_state = thread_state,
+            immutable_types = ImmutableTypes(), 
+            tracing_config = tracing_config,
+            maybe_collect = collector(multiplier = multiplier),
+            traceargs = options.trace_inputs)
+
+        # force a full collection
+        install(system)
+
+        gc.collect()
+        gc.callbacks.append(system.on_gc_event)
+    
+        run_with_retrace(system, args, options.trace_shutdown)
+
+        gc.callbacks.remove(system.on_gc_event)
+
+def replay(args):
+    path = Path(args.recording)
+
+    if not path.exists():
+        raise Exception(f"Recording path: {path} does not exist")
+
+    settings = load_json(path / "settings.json")
+
+    if settings['md5_checksums'] != checksums():
+        raise Exception("Checksums for Retrace do not match, cannot run replay with different version of retrace to record")
+
+    if settings['python_version'] != sys.version:
+        raise Exception("Python version does not match, cannot run replay with different version of Python to record")
+
+    os.environ.update(settings['env'])
+
+    thread_state = utils.ThreadState(*thread_states)
+
+    # with stream.reader(path = path / 'trace.bin',
+    #                    thread = thread_id, 
+    #                    timeout_seconds = args.timeout,
+    #                    verbose = args.verbose,
+    #                    on_stack_difference = thread_state.wrap('disabled', on_stack_difference),
+    #                    magic_markers = settings['magic_markers']) as reader:
+
+    with stream.reader1(path = path / 'trace.bin',
+                        read_timeout = args.read_timeout,
+                        verbose = args.verbose,
+                        magic_markers = settings['magic_markers']) as reader:
+
+        tracing_config = {}
+
+        system = ReplayProxySystem(
+            reader = reader,
+            thread_state = thread_state,
+            immutable_types = ImmutableTypes(), 
+            tracing_config = tracing_config,
+            traceargs = settings['trace_inputs'],
+            verbose = args.verbose,
+            skip_weakref_callbacks = args.skip_weakref_callbacks)
+
+        install(system)
+
+        gc.collect()
+        gc.disable()
+
+        run_with_retrace(system, settings['argv'], settings['trace_shutdown'])
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="python -m retracesoftware",
+        description="Run a Python module with debugging, logging, etc."
+    )
+
+    parser.add_argument(
+        '--verbose', 
+        action='store_true', 
+        help='Enable verbose output'
+    )
+
+    parser.add_argument(
+        '--recording',   # or '-r'
+        type = str,      # ensures it's a string (optional, but safe)
+        default = '.',    # default value if not provided
+        help = 'the directory to place the recording files'
+    )
+
+    if '--' in sys.argv:
+        parser.add_argument(
+            '--stacktraces', 
+            action='store_true', 
+            help='Capture stacktrace for every event'
+        )
+
+        parser.add_argument(
+            '--magic_markers', 
+            action='store_true', 
+            help='Write magic markers to tracefile, used for debugging'
+        )
+
+        parser.add_argument(
+            '--trace_shutdown',
+            action='store_true', 
+            help='Whether to trace system shutdown and cleanup hooks'
+        )
+
+        parser.add_argument(
+            '--trace_inputs', 
+            action='store_true', 
+            help='Whether to write call parameters, used for debugging'
+        )
+
+        parser.add_argument('rest', nargs = argparse.REMAINDER, help='target application and arguments')
+
+        args = parser.parse_args()
+
+        record(args, args.rest[1:])
+    else:
+
+        parser.add_argument(
+            '--skip_weakref_callbacks',
+            action='store_true',
+            help = 'whether to disable retrace in weakref callbacks on replay'
+        )
+
+        parser.add_argument(
+            '--read_timeout',   # or '-r'
+            type = int,      # ensures it's a string (optional, but safe)
+            default = 1000,    # default value if not provided
+            help = 'timeout in millseconds for incomplete read of element to timeout'
+        )
+
+        replay(parser.parse_args())
+
+if __name__ == "__main__":
+    main()

retracesoftware/autoenable.py

@@ -0,0 +1,53 @@
+if __name__ == "__main__":
+    import sysconfig
+    import pathlib
+
+    # 'purelib' is the name of the directory where non-platform-specific modules are installed.
+    file = pathlib.Path(sysconfig.get_paths()["purelib"]) / 'retrace.pth'
+    file.write_text('import retracesoftware.autoenable;', encoding='utf-8')
+
+    print(f'Retrace autoinstall enabled by creating: {file}')
+else:
+    import os
+
+    def is_true(name):
+        if name in os.environ:
+            return os.environ[name].lower() in {'true', '1', 't', 'y', 'yes'}
+        else:
+            return False
+            
+    def is_running_retrace():
+        return sys.orig_argv[1] == '-m' and sys.orig_argv[2].startswith('retracesoftware')
+    
+    # only do anything is the RETRACE env variable is set
+    if is_true('RETRACE'):
+        import sys
+
+        if not is_running_retrace():
+            
+            new_argv = [sys.orig_argv[0], '-m', 'retracesoftware']
+
+            if is_true('RETRACE_VERBOSE'):
+                new_argv.append('--verbose')
+
+            if 'RETRACE_RECORDING_PATH' in os.environ:
+                new_argv.append('--recording')
+                new_argv.append(os.environ['RETRACE_RECORDING_PATH'])
+
+            if is_true('RETRACE_STACKTRACES'):
+                new_argv.append('--stacktraces')
+
+            if is_true('RETRACE_SHUTDOWN'):
+                new_argv.append('--trace_shutdown')
+
+            if is_true('RETRACE_MAGIC_MARKERS'):
+                new_argv.append('--magic_markers')
+
+            if is_true('RETRACE_TRACE_INPUTS'):
+                new_argv.append('--trace_inputs')
+
+            new_argv.append('--')
+            new_argv.extend(sys.orig_argv[1:])
+            
+            # print(f'Running: {new_argv}')
+            os.execv(sys.executable, new_argv)

retracesoftware/config.json

@@ -4,15 +4,18 @@
     "internal", {"comment": "Default state when retrace is disabled for a thread"},
     "external", {"comment": "When target thread is running outside the python system to be recorded"},
     "retrace", {"comment": "When target thread is running outside the retrace system"},
+    "importing", {"comment": "When target thread is running outside the retrace system"},
     "gc", {"comment": "When the target thread is running inside the pyton garbage collector"}
   ],
 
+  "trace_calls": false,
+
   "recording_path": "recordings/%Y%m%d_%H%M%S_%f",
 
   "tracing_levels": {
     "none": [],
     "all": [
-      "tracecalls",
+      "tracecalls.call",
       "proxy.ext.disabled.call",
       "proxy.int.disabled.call",
       "proxy.wrapping.new",
@@ -36,6 +39,7 @@
       "install.module.phase.results"
     ],
     "debug": [
+      "tracecalls.call",
       "proxy.wrapping.new",
       "proxy.wrapping.new.method",
       "proxy.int_to_ext.stack",
@@ -55,7 +59,8 @@
       "install.module",
       "install.module.phase",
       "install.module.phase.results"
-    ]
+    ],
+    "release": []
   },
 
   "default_tracing_level": "debug",
@@ -68,6 +73,18 @@
   ],
 
   "preload": [
+    "logging",
+    "http.client",
+    "queue",
+    "mimetypes",
+    "encodings.idna",
+    "hmac",
+    "ipaddress",
+    "tempfile",
+    "zipfile",
+    "importlib.resources",
+    "atexit",
+    "weakref"
   ],
 
   "predicates": {
@@ -154,298 +171,5 @@
         }
       ]
     }
-  },
-
-  "modules": {
-    "_frozen_importlib_external": {
-      "comment": {
-        "with_state": {
-          "disabled": ["_path_stat"]
-        }
-      }
-    },
-    "_imp": {
-      "patch_extension_exec": ["exec_dynamic", "exec_builtin"],
-      "comment": {
-        "with_state": {
-          "internal": ["create_dynamic"]
-        }
-      }
-    },
-    "bdb": {
-      "methods_with_state": {
-        "disabled": {
-          "Bdb": ["trace_dispatch"]
-        }
-      },
-      "with_state": {
-        "disabled": ["Bdb.trace_dispatch"]
-      }
-    },
-
-    "sys": {
-      "with_state": {
-        "disabled": ["excepthook"]
-      },
-      "comment": {
-        "with_state_recursive": {
-          "disabled": ["settrace", "setprofile"]
-        }
-      }
-    },
-    
-    "importlib": {
-      "with_state": {
-        "disabled": ["import_module"]
-      }
-    },
-
-    "importlib._bootstrap": {
-      "comment": {
-        "with_state": {
-          "internal": [
-            "_load_unlocked",
-            "_find_spec",
-            "_lock_unlock_module",
-            "_get_module_lock"]
-        }
-      }
-    },
-
-    "encodings": {
-      "with_state": {
-        "disabled": ["search_function"]
-      }
-    },
-    "_retrace_utils": {
-      "immutable_types": [
-        "PyCFunctionProxy"
-      ]
-    },
-    "enum": {
-      "immutable_types": [
-        "Enum"
-      ]
-    },
-    "os": {
-      "immutable_types": [
-        "stat_result",
-        "terminal_size",
-        "statvfs_result"
-      ]
-    },
-    "mmap": {
-      "proxy": [
-        "mmapXXX"
-      ]
-    },
-    "types": {
-      "patch_hash": ["FunctionType"],
-      "immutable_types": [
-        "TracebackType",
-        "ModuleType"
-      ]
-    },
-    "builtins": {
-      "patch_hash": ["object"],
-
-      "comment": {
-        "with_state": {
-          "disabled": ["__import__"]
-        }
-      },
-
-      "immutable_types": [
-        "BaseException",
-        "memoryview",
-        "int",
-        "float",
-        "complex",
-        "str",
-        "bytes",
-        "bool",
-        "bytearray",
-        "type",
-        "slice"],
-
-      "patch_exec": "exec"
-    },
-    "_thread": {
-      "proxy": [
-        "allocate",
-        "allocate_lock",
-        "RLock"
-      ],
-      "patch_start_new_thread": ["start_new_thread", "start_new"]
-    },
-    "_datetime": {
-      "immutable_types": [
-        "datetime",
-        "tzinfo",
-        "timezone"
-      ],
-      "proxy_type_attributes": {
-        "datetime": [
-          "now",
-          "utcnow",
-          "today"
-        ]
-      }
-    },
-    "time": {
-      "immutable_types": [
-        "struct_time"
-      ],
-      "proxy": [
-        "perf_counter",
-        "time",
-        "gmtime",
-        "localtime",
-        "monotonic",
-        "monotonic_ns",
-        "time_ns"
-      ]
-    },
-    "io": {
-      "comment": {
-        "proxy": ["open_code", "open"]
-      },
-      "path_predicates": {
-        "open_code": "path",
-        "open": "file"
-      },
-      "patchtype": {
-        "FileIO": {
-          "readinfo": "retracesoftware.install.edgecases.readinto"
-        },
-        "BufferedReader": {
-          "readinfo": "retracesoftware.install.edgecases.readinto"
-        },
-        "BufferedRandom": {
-          "readinfo": "retracesoftware.install.edgecases.readinto"
-        }
-      }
-    },
-    "pathlib": {
-      "immutable_types": [
-        "PosixPath"
-      ]
-    },
-    "posix": {
-      "immutable_types": [
-        "times_result",
-        "statvfs_result",
-        "uname_result",
-        "stat_result",
-        "terminal_size"
-      ],
-
-      "proxy_all_except": [
-        "fork",
-        "register_at_fork",
-        "basename",
-        "readlink",
-        "strerror",
-        "listdir",
-        "_path_normpath",
-        "DirEntry"
-      ],
-
-      "with_state_recursive0": {
-        "disabled": ["register_at_fork"]
-      },
-
-      "wrappers": {
-        "fork_exec": "retracesoftware.install.edgecases.fork_exec",
-        "posix_spawn": "retracesoftware.install.edgecases.posix_spawn"
-      }
-    },
-    "_posixsubprocess": {
-      "proxy_all_except": []
-    },
-    "fcntl": {
-      "proxy_all_except": []
-    },
-    "_signal": {
-      "proxy_all_except": []
-    },
-    "_socket": {
-      "proxy_all_except": [
-        "CAPI"
-      ],
-      "immutable_types": [
-        "error",
-        "herror",
-        "gaierror",
-        "timeout"
-      ],
-      "patchtype": {
-        "socket": {
-          "recvfrom_into": "retracesoftware.install.edgecases.recvfrom_into",
-          "recv_into": "retracesoftware.install.edgecases.recv_into",
-          "recvmsg_into": "retracesoftware.install.edgecases.recvmsg_into"
-        }
-      }
-    },
-    "select": {
-      "proxy_all_except": []
-    },
-    "_sqlite3": {
-      "proxy_all_except": []
-    },
-    "_ssl": {
-      "proxy_all_except": [],
-
-      "proxy_functions": ".*",
-      "comment": {
-        "proxy_types": [
-          "MemoryBIO",
-          "_SSLSocket",
-          "_SSLContext"
-          ],
-          "wrap": {
-            "_SSLSocket.write": "write"
-          }
-        },
-      "patchtype": {
-        "_SSLSocket": {
-          "read": "retracesoftware.install.edgecases.read"
-        }
-      }
-    },
-    "_random": {
-      "proxy_all_except": [],
-      "proxy_types": [
-        "Random"
-      ],
-      "proxy_functions": ".*"
-    },
-    "_multiprocessing": {
-      "proxy_functions": ".*"
-    },
-    "multiprocessing.context": {      
-      "proxy": ["_default_context"]
-    },
-
-    "PIL._imaging": {
-      "proxy_all_except": ["map_buffer"]
-    },
-    "psycopg2._psycopg": {
-      "proxy_all_except": ["Error"]
-    },
-
-    "_weakrefset": {
-      "sync_types": ["WeakSet", "_IterationGuard"]
-    },
-    "_collections": {
-      "sync_types": [
-        "deque"
-      ]
-    },
-    "_queue": {
-      "sync_types": [
-        "SimpleQueue"
-      ]
-    }
   }
 }

retracesoftware/install/config.py

@@ -2,6 +2,7 @@ import pkgutil
 import json
 import os
 import datetime
+import tomllib
 
 from pathlib import Path
 
@@ -36,6 +37,11 @@ def recording_path(config):
         os.environ['RETRACE_RECORDING_PATH'] = str(recording_path)
         return recording_path
 
+def load_module_config(filename):
+    data = pkgutil.get_data("retracesoftware", filename)
+    assert data is not None
+    return tomllib.loads(data.decode("utf-8"))
+
 def load_config(filename):
         
     data = pkgutil.get_data("retracesoftware", filename)