externum 2.0.0__py3-none-any.whl

externum/__init__.py

@@ -0,0 +1,81 @@
+"""
+Externum - The First LLM-Native Programming Language
+======================================================
+Externum v3 is a complete programming language that fuses:
+- Python readability + dynamic typing
+- Binary performance + SIMD vectorization
+- Bash system control + process orchestration
+
+One source compiles to Python, Bash and binary targets, or runs directly
+(``externum run``), with a REPL, a module system and a standard library
+written in the language itself.
+
+Roadmap modules (llm, neural, distributed, types, spec, debug) keep their
+reserved API surface below; the package stays fully importable without them.
+"""
+
+__version__ = "3.0.0"
+__codename__ = "Sentient"
+
+from .lexer import Lexer
+from .parser import Parser
+from .compiler import Compiler
+
+try:
+    from .runtime import Runtime
+except ImportError:  # pragma: no cover
+    Runtime = None
+
+
+def _guarded(name):
+    try:
+        return __import__(f"{__name__}.{name}", fromlist=["*"])
+    except ImportError:
+        return None
+
+
+_llm = _guarded("llm")
+_neural = _guarded("neural")
+_distributed = _guarded("distributed")
+_types = _guarded("types")
+_spec = _guarded("spec")
+_debug = _guarded("debug")
+
+LLMClient = getattr(_llm, "LLMClient", None)
+PromptTemplate = getattr(_llm, "PromptTemplate", None)
+FunctionSchema = getattr(_llm, "FunctionSchema", None)
+
+Tensor = getattr(_neural, "Tensor", None)
+Module = getattr(_neural, "Module", None)
+Linear = getattr(_neural, "Linear", None)
+Conv2d = getattr(_neural, "Conv2d", None)
+Attention = getattr(_neural, "Attention", None)
+Autograd = getattr(_neural, "Autograd", None)
+
+Actor = getattr(_distributed, "Actor", None)
+Cluster = getattr(_distributed, "Cluster", None)
+Stream = getattr(_distributed, "Stream", None)
+Channel = getattr(_distributed, "Channel", None)
+
+Type = getattr(_types, "Type", None)
+DependentType = getattr(_types, "DependentType", None)
+RefinementType = getattr(_types, "RefinementType", None)
+EffectType = getattr(_types, "EffectType", None)
+
+Spec = getattr(_spec, "Spec", None)
+Theorem = getattr(_spec, "Theorem", None)
+Proof = getattr(_spec, "Proof", None)
+Verify = getattr(_spec, "Verify", None)
+
+TimeTravelDebugger = getattr(_debug, "TimeTravelDebugger", None)
+HotReloader = getattr(_debug, "HotReloader", None)
+
+__all__ = [
+    "Lexer", "Parser", "Compiler", "Runtime",
+    "LLMClient", "PromptTemplate", "FunctionSchema",
+    "Tensor", "Module", "Linear", "Conv2d", "Attention", "Autograd",
+    "Actor", "Cluster", "Stream", "Channel",
+    "Type", "DependentType", "RefinementType", "EffectType",
+    "Spec", "Theorem", "Proof", "Verify",
+    "TimeTravelDebugger", "HotReloader",
+]

externum/__main__.py

@@ -0,0 +1,205 @@
+"""Main entry point for the Externum compiler, runtime and REPL."""
+
+import sys
+import argparse
+
+from .lexer import Lexer
+from .parser import Parser
+from .compiler import Compiler
+from . import __version__
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog='externum',
+        description='Externum - Python + Binary + Bash Language',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  externum run program.ext [args...]   # execute a program
+  externum repl                        # interactive shell
+  externum program.ext --target python # compile to Python
+  externum program.ext -o out.py       # compile to a file
+        """,
+    )
+    p.add_argument('--version', action='version', version=f'Externum {__version__}')
+    sub = p.add_subparsers(dest='command')
+
+    run_p = sub.add_parser('run', help='Execute an .ext program')
+    run_p.add_argument('file', help='Source file to run')
+    run_p.add_argument('args', nargs='*', help='Arguments passed to the program')
+    run_p.add_argument('--protect', action='store_true',
+                       help='Apply the DRM stack to the program before running')
+    run_p.add_argument('--app-id', default=None, help='Application id for DRM')
+    run_p.add_argument('--author', default=None, help='Author name for DRM')
+    run_p.add_argument('--secret', default=None, help='DRM signing secret (compile-time only)')
+    run_p.add_argument('--build-id', default=None, help='Build id baked into the DRM watermark')
+
+    sub.add_parser('repl', help='Start the interactive Externum shell')
+
+    comp = sub.add_parser('compile', help='Compile an .ext program (default)')
+    comp.add_argument('file', help='Source file to compile')
+    comp.add_argument('--target', choices=['python', 'binary', 'bash', 'all'],
+                      default='all', help='Output target')
+    comp.add_argument('--output', '-o', help='Output file')
+    comp.add_argument('--protect', action='store_true',
+                      help='Embed the full DRM stack (license, watermark, tamper check, obfuscation)')
+    comp.add_argument('--app-id', default=None, help='Application id for DRM')
+    comp.add_argument('--author', default=None, help='Author name for DRM')
+    comp.add_argument('--secret', default=None, help='DRM signing secret (compile-time only)')
+    comp.add_argument('--build-id', default=None, help='Build id baked into the DRM watermark')
+
+    key = sub.add_parser('keygen', help='Generate DRM license keys')
+    key.add_argument('--app-id', required=True, help='Application id')
+    key.add_argument('--author', default='', help='Author name')
+    key.add_argument('--secret', required=True, help='Signing secret')
+    key.add_argument('--expires', type=int, default=None,
+                     help='Unix timestamp expiry (0/absent = never)')
+    key.add_argument('--count', type=int, default=1, help='How many keys')
+    return p
+
+
+def _read_source(path: str) -> str:
+    try:
+        with open(path, 'r', encoding='utf-8') as fh:
+            return fh.read()
+    except FileNotFoundError:
+        print(f"Error: File '{path}' not found", file=sys.stderr)
+        sys.exit(1)
+    except IOError as exc:
+        print(f'Error reading file: {exc}', file=sys.stderr)
+        sys.exit(1)
+
+
+def cmd_run(args) -> None:
+    try:
+        from .runtime import Runtime
+
+        protect = None
+        if args.protect:
+            protect = {
+                'app_id': args.app_id or 'externum-app',
+                'author': args.author or 'unknown',
+                'secret': args.secret or 'externum-drm',
+                'build_id': args.build_id,
+            }
+        Runtime().run_file(args.file, argv=args.args, protect=protect)
+    except SyntaxError as exc:
+        print(f'Syntax Error: {exc}', file=sys.stderr)
+        sys.exit(1)
+    except SystemExit:
+        raise
+    except Exception as exc:  # noqa: BLE001 - CLI boundary
+        print(f'Runtime Error: {exc}', file=sys.stderr)
+        sys.exit(1)
+
+
+def cmd_keygen(args) -> None:
+    from .drm import make_license
+    from . import __version__
+
+    print(f'Externum {__version__} — DRM keygen (secret is never stored)')
+    for _ in range(args.count):
+        print(make_license(args.secret, args.app_id, args.author, args.expires))
+
+
+def cmd_repl(args) -> None:
+    try:
+        from .runtime import Runtime
+
+        Runtime().repl()
+    except KeyboardInterrupt:
+        print()
+    except Exception as exc:  # noqa: BLE001
+        print(f'Error: {exc}', file=sys.stderr)
+        sys.exit(1)
+
+
+def cmd_compile(args) -> None:
+    source = _read_source(args.file)
+    try:
+        from .runtime import Runtime
+
+        protect = None
+        if args.protect:
+            protect = {
+                'app_id': args.app_id or 'externum-app',
+                'author': args.author or 'unknown',
+                'secret': args.secret or 'externum-drm',
+                'build_id': args.build_id,
+            }
+        rt = Runtime()
+        py = rt.compile_to_python(source, protect=protect)
+        if args.target == 'all':
+            # DRM-protected Python plus the raw bash/binary targets
+            tokens = Lexer(source).tokenize()
+            ast = list(Parser(tokens).parse())
+            compiled = Compiler(ast).compile('all')
+            result = {
+                'python': py,
+                'bash': compiled['bash'],
+                'binary': compiled['binary'],
+            }
+        elif args.target == 'python':
+            result = py
+        else:
+            tokens = Lexer(source).tokenize()
+            ast = list(Parser(tokens).parse())
+            result = Compiler(ast).compile(args.target)
+            if args.target == 'bash':
+                result = result['bash']
+            elif args.target == 'binary':
+                result = result['binary']
+    except SyntaxError as exc:
+        print(f'Syntax Error: {exc}', file=sys.stderr)
+        sys.exit(1)
+    except Exception as exc:  # noqa: BLE001 - CLI boundary
+        print(f'Error: {exc}', file=sys.stderr)
+        sys.exit(1)
+
+    if args.target == 'all':
+        # bash/binary sections are embedded as comments so the combined
+        # artifact stays a valid, runnable Python file
+        def _comment(lines: str) -> str:
+            return '\n'.join('# ' + ln for ln in lines.splitlines())
+
+        output = (
+            '# Externum Generated Code\n'
+            '# Python target:\n'
+            f"{result['python']}\n\n"
+            '# Bash target (commentary — not Python):\n'
+            f"{_comment(result['bash'])}\n\n"
+            '# Binary target (commentary — not Python):\n'
+            f"{_comment(result['binary'])}\n"
+        )
+    else:
+        output = result if isinstance(result, str) else '\n'.join(result)
+
+    if args.output:
+        with open(args.output, 'w', encoding='utf-8') as fh:
+            fh.write(output)
+        print(f'Output written to {args.output}')
+    else:
+        print(output)
+
+
+def main(argv=None) -> None:
+    argv = list(sys.argv[1:] if argv is None else argv)
+
+    # Backwards compatible form: `externum file.ext [--target ...]`
+    if argv and argv[0] not in ('run', 'repl', 'compile', 'keygen') and not argv[0].startswith('-'):
+        argv = ['compile'] + argv
+
+    args = _build_parser().parse_args(argv)
+    if args.command == 'run':
+        cmd_run(args)
+    elif args.command == 'repl':
+        cmd_repl(args)
+    elif args.command == 'keygen':
+        cmd_keygen(args)
+    else:
+        cmd_compile(args)
+
+
+if __name__ == '__main__':
+    main()

externum/analysis.py

@@ -0,0 +1,88 @@
+"""Externum — static analysis pipeline.
+
+Externum is a strict language by design: every compilation runs the full
+static analysis. There is no "easy mode" — these checks *are* the language.
+
+`preprocess()` expands `macro` definitions before lexing (compile-time
+metaprogramming).
+
+`check()` runs the static type checker + ownership analyser over a parsed
+AST using the metadata the parser captured (declared variable annotations,
+function signatures, traits, impls, `mut` declarations). Any violation
+raises `ExternumTypeError` with all diagnostics joined.
+"""
+
+import re
+from typing import List, Optional, Set, Tuple
+
+from .parser import ASTNode
+from .typesys import TypeChecker, ExternumTypeError
+
+
+_MACRO_DEF = re.compile(
+    r'macro\s+([A-Za-z_][A-Za-z0-9_]*)\s*(?:\(([^)]*)\))?\s*\{([\s\S]*?)\}',
+    re.MULTILINE,
+)
+
+
+class MacroError(Exception):
+    """Raised when a macro is misused."""
+
+
+def _split_args(raw: str) -> List[str]:
+    return [p.strip() for p in raw.split(',') if p.strip()]
+
+
+def preprocess(source: str) -> Tuple[str, dict]:
+    """Expand macro definitions. Returns (processed_source, macros)."""
+    macros = {}
+    for m in _MACRO_DEF.finditer(source):
+        name, params, body = m.group(1), m.group(2), m.group(3)
+        macros[name] = {'params': _split_args(params or ''), 'body': body}
+    if not macros:
+        return source, macros
+
+    # strip definitions from the source
+    cleaned = _MACRO_DEF.sub('', source)
+
+    # expand NAME(...) and bare NAME invocations
+    for name, macro in macros.items():
+        cleaned = _expand(cleaned, name, macro)
+    return cleaned, macros
+
+
+def _expand(source: str, name: str, macro: dict) -> str:
+    params = macro['params']
+    body = macro['body']
+    pattern = re.compile(r'\b' + re.escape(name) + r'\s*\(([^()]*)\)')
+    pos = 0
+    out = []
+    for m in pattern.finditer(source):
+        out.append(source[pos:m.start()])
+        raw_args = m.group(1)
+        args = _split_args(raw_args) if raw_args.strip() else []
+        if len(args) != len(params):
+            raise MacroError(
+                f'macro `{name}` expects {len(params)} argument(s), got {len(args)}')
+        expanded = body
+        for pname, arg in zip(params, args):
+            expanded = re.sub(r'\b' + re.escape(pname) + r'\b', arg, expanded)
+        out.append(expanded)
+        pos = m.end()
+    out.append(source[pos:])
+    return ''.join(out)
+
+
+def check(ast: List[ASTNode], annotations: dict, signatures: dict,
+          traits: dict, impls: dict, mutable: Set[str] = None) -> List[str]:
+    """Run Externum static analysis. Returns the list of diagnostics."""
+    checker = TypeChecker(ast, annotations, signatures, traits, impls, mutable or set())
+    return checker.check()
+
+
+def check_or_raise(ast: List[ASTNode], annotations: dict, signatures: dict,
+                   traits: dict, impls: dict, mutable: Set[str] = None) -> None:
+    errors = check(ast, annotations, signatures, traits, impls, mutable)
+    if errors:
+        raise ExternumTypeError(
+            'Externum: ' + '; '.join(errors))