pyrust-engine 0.6.0__py3-none-any.whl

pyrust_engine/__init__.py

@@ -0,0 +1,5 @@
+from .runtime import optimize, OptimizationError, OptimizedFunction, PyRustFallbackWarning
+from .compiler import Unsupported, F64Array, I64Array
+from .cache import prune_cache
+__all__ = ['optimize', 'OptimizationError', 'OptimizedFunction', 'PyRustFallbackWarning', 'Unsupported', 'F64Array', 'I64Array', 'prune_cache']
+__version__ = '0.6.0'

pyrust_engine/__main__.py

@@ -0,0 +1,60 @@
+import argparse
+import inspect
+import json
+from pathlib import Path
+import runpy
+import sys
+from .profiling import profile_script
+from .cache import prune_cache
+
+
+def check_file(path, max_iterations=10_000_000):
+    """Load a trusted file (without running its `if __name__ == '__main__'` block) and
+    explain, for every function defined in it, whether PyRust can compile it."""
+    from .runtime import optimize
+    path = Path(path).resolve()
+    old_path = sys.path[:]
+    try:
+        sys.path.insert(0, str(path.parent))
+        namespace = runpy.run_path(str(path), run_name='__pyrust_check__')
+    finally:
+        sys.path[:] = old_path
+    results = []
+    for value in namespace.values():
+        original = getattr(value, 'python', value)
+        if not inspect.isfunction(original) or Path(original.__code__.co_filename).resolve() != path:
+            continue
+        text = optimize(original, max_iterations=max_iterations, warn_on_fallback=False).explain()
+        results.append({'function': original.__qualname__, 'line': original.__code__.co_firstlineno,
+                        'native': ' compiles to Rust.' in text.splitlines()[0], 'explanation': text})
+    return sorted(results, key=lambda r: r['line'])
+
+
+def main():
+    parser=argparse.ArgumentParser(prog='python -m pyrust_engine',description='PyRust Engine developer tools')
+    commands=parser.add_subparsers(dest='command',required=True)
+    p=commands.add_parser('check',help='Explain which functions in a trusted file compile to Rust, and why not')
+    p.add_argument('script');p.add_argument('--json',action='store_true')
+    p=commands.add_parser('profile',help='Execute a trusted script once; rank native candidates')
+    p.add_argument('script');p.add_argument('arguments',nargs=argparse.REMAINDER)
+    p=commands.add_parser('cache',help='List old cache entries; delete only with --delete')
+    p.add_argument('--directory');p.add_argument('--days',type=float,default=30)
+    p.add_argument('--delete',action='store_true')
+    args=parser.parse_args()
+    if args.command=='check':
+        results=check_file(args.script)
+        if args.json:
+            print(json.dumps(results,indent=2))
+        else:
+            for row in results:
+                print(row['explanation'])
+                print()
+            native=sum(r['native'] for r in results)
+            print(f'{native} of {len(results)} functions compile to Rust.')
+        return
+    result=(profile_script(args.script,args.arguments) if args.command=='profile' else
+            prune_cache(args.directory,older_than_days=args.days,dry_run=not args.delete))
+    print(json.dumps(result,indent=2))
+
+if __name__=='__main__':
+    main()

pyrust_engine/cache.py

@@ -0,0 +1,116 @@
+"""Private native cache with OS process locks, hashes and atomic publication."""
+from contextlib import contextmanager
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+import time
+
+SUFFIX = '.dll' if sys.platform=='win32' else '.dylib' if sys.platform=='darwin' else '.so'
+
+def cache_root(path=None):
+    root = Path(path) if path else Path.home()/'.cache'/'pyrust_engine'
+    root.mkdir(mode=0o700, parents=True, exist_ok=True)
+    root=root.resolve()
+    if os.name=='posix':
+        info=root.stat()
+        if info.st_uid!=os.getuid() or info.st_mode & 0o022:
+            raise PermissionError('Native cache must be owned by this user and not group/world writable')
+    return root
+
+def digest(path):
+    h=hashlib.sha256()
+    with open(path,'rb') as f:
+        for chunk in iter(lambda:f.read(1024*1024),b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+@contextmanager
+def process_lock(path, timeout=150):
+    # Advisory OS locks are released automatically if the process dies.
+    f=open(path,'a+b')
+    try:
+        f.seek(0,2)
+        if f.tell()==0:
+            f.write(b'0');f.flush()
+        start=time.monotonic()
+        while True:
+            try:
+                f.seek(0)
+                if sys.platform=='win32':
+                    import msvcrt
+                    msvcrt.locking(f.fileno(),msvcrt.LK_NBLCK,1)
+                else:
+                    import fcntl
+                    fcntl.flock(f.fileno(),fcntl.LOCK_EX|fcntl.LOCK_NB)
+                break
+            except (BlockingIOError,OSError):
+                if time.monotonic()-start >= timeout:
+                    raise TimeoutError('Timed out waiting for native cache lock')
+                time.sleep(0.02)
+        try:
+            yield
+        finally:
+            if sys.platform=='win32':
+                import msvcrt
+                f.seek(0);msvcrt.locking(f.fileno(),msvcrt.LK_UNLCK,1)
+            else:
+                import fcntl
+                fcntl.flock(f.fileno(),fcntl.LOCK_UN)
+    finally:
+        f.close()
+
+def cached_path(root,key):
+    try:
+        meta=json.loads((root/(key+'.json')).read_text())
+        sha=meta['sha256']
+        if not isinstance(sha,str) or not re.fullmatch('[0-9a-f]{64}',sha):
+            return None
+        path=root/(key+'-'+sha+SUFFIX)
+        return path if path.is_file() and digest(path)==sha else None
+    except (OSError,ValueError,KeyError,TypeError):
+        return None
+
+def compile_library(root,key,compiler,source):
+    with tempfile.TemporaryDirectory(dir=root,prefix='build-') as work:
+        rs=Path(work)/'kernel.rs';binary=Path(work)/('kernel'+SUFFIX)
+        rs.write_text(source,encoding='utf-8')
+        result=subprocess.run([compiler,str(rs),'--crate-type','cdylib','--edition','2021',
+                               '-C','opt-level=3','-C','panic=abort','-o',str(binary)],
+                              capture_output=True,text=True,timeout=120)
+        if result.returncode:
+            raise RuntimeError('Rust compilation failed: '+result.stderr[-3000:])
+        sha=digest(binary)
+        path=root/(key+'-'+sha+SUFFIX)
+        # Never truncate a library another process could have mapped.
+        if not path.exists() or digest(path)!=sha:
+            os.replace(binary,path)
+        meta=Path(work)/'metadata.json'
+        meta.write_text(json.dumps({'sha256':sha,'created':time.time()}))
+        os.replace(meta,root/(key+'.json'))
+        return path
+
+def prune_cache(path=None, *, older_than_days=30, dry_run=True):
+    """List/remove old cache entries. Defaults to a dry run. Lock files are retained."""
+    if older_than_days < 0:
+        raise ValueError('older_than_days must be nonnegative')
+    root=cache_root(path);entries=[];errors=[]
+    for meta in root.glob('*.json'):
+        key=meta.stem
+        if not re.fullmatch('[0-9a-f]{64}',key):
+            continue
+        with process_lock(root/(key+'.lock')):
+            if not meta.exists() or time.time()-meta.stat().st_mtime < older_than_days*86400:
+                continue
+            for file in [*root.glob(key+'-*'+SUFFIX),meta]:
+                entries.append(str(file))
+                if not dry_run:
+                    try:
+                        file.unlink(missing_ok=True)
+                    except OSError as exc:
+                        errors.append(str(exc))
+    return {'dry_run':dry_run,'entries':entries,'errors':errors}