monogate-forge 0.1.0__py3-none-any.whl

lang/loader/__init__.py

@@ -0,0 +1,26 @@
+"""Module loader for `use` declarations.
+
+Resolves `use <root>::<name>(...);` paths to `.eml` source files
+on disk, parses them once, and caches the results so a function
+imported by ten different modules is parsed only once per
+process.
+
+Search-path table:
+
+  stdlib    -> lang/spec/stdlib/<name>.eml
+
+(More roots will land as the import system grows -- e.g.
+`local::X` for sibling files relative to the importing file.)
+
+Cycles are detected by tracking the in-progress load set; when a
+module already on that set is requested again, ImportError is
+raised with the cycle path.
+"""
+
+from lang.loader.resolver import (
+    LoaderError,
+    ModuleLoader,
+    resolve_imports,
+)
+
+__all__ = ["LoaderError", "ModuleLoader", "resolve_imports"]

lang/loader/resolver.py

@@ -0,0 +1,288 @@
+"""Import resolver -- finds, parses, caches, and merges modules."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from lang.parser.ast_nodes import (
+    EMLConstant,
+    EMLFunction,
+    EMLImport,
+    EMLModule,
+    EMLTypeAlias,
+)
+
+
+# Repo root -- two levels up from this file (lang/loader/resolver.py).
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+# Search-path table mapping the FIRST path segment to a directory.
+# When `use stdlib::math;` is parsed, the loader looks for
+# `<DEFAULT_SEARCH_PATHS["stdlib"]>/math.eml`.
+#
+# The reserved root `"local"` does NOT appear here -- it resolves
+# relative to the importing file's directory at load time.
+DEFAULT_SEARCH_PATHS: dict[str, Path] = {
+    "stdlib": _REPO_ROOT / "lang" / "spec" / "stdlib",
+}
+
+LOCAL_ROOT = "local"
+
+
+class LoaderError(Exception):
+    """Loader-level error: not-found, cycle, or symbol clash."""
+
+
+@dataclass
+class ModuleLoader:
+    """Resolves + parses + caches imports.
+
+    Construct one per parse session if you want test isolation;
+    construct one and reuse it across many parse calls if you
+    want maximum cache hit rate.
+
+    Cache keys are RESOLVED file-paths (strings), not joined
+    import paths -- so `local::helpers` from `/a/x.eml` and
+    `/b/x.eml` correctly cache as two different modules even
+    though they share a joined-path spelling."""
+    search_paths: dict[str, Path] = field(
+        default_factory=lambda: dict(DEFAULT_SEARCH_PATHS),
+    )
+    cache: dict[str, EMLModule] = field(default_factory=dict)
+    _in_progress: set[str] = field(default_factory=set)
+
+    def load(
+        self,
+        joined_path: str,
+        *,
+        source_dir: Path | None = None,
+    ) -> EMLModule:
+        """Load the module identified by `joined_path` (e.g.
+        'stdlib::math' or 'local::helpers').
+
+        `source_dir` is the directory of the file doing the import.
+        Required when `joined_path` starts with `local::`; ignored
+        otherwise.
+
+        Raises LoaderError on:
+          - unknown root segment (not in search_paths and not 'local')
+          - missing `local::` source_dir
+          - missing .eml file
+          - import cycle (resolved file path already in progress)
+        """
+        file_path = self.resolve(joined_path, source_dir=source_dir)
+        cache_key = str(file_path.resolve())
+
+        if cache_key in self.cache:
+            return self.cache[cache_key]
+
+        if cache_key in self._in_progress:
+            raise LoaderError(
+                f"import cycle while loading {joined_path!r} "
+                f"({file_path}) -- in progress: "
+                f"{sorted(self._in_progress)}"
+            )
+
+        if not file_path.is_file():
+            raise LoaderError(
+                f"module {joined_path!r} not found "
+                f"(looked for {file_path})"
+            )
+
+        # Local import to avoid a parser <-> loader cycle.
+        from lang.parser.parser import parse_source
+
+        self._in_progress.add(cache_key)
+        try:
+            text = file_path.read_text(encoding="utf-8")
+            mod = parse_source(text, source_file=str(file_path))
+            # Recursively resolve transitive imports BEFORE caching.
+            mod = resolve_imports(mod, loader=self)
+            self.cache[cache_key] = mod
+            return mod
+        finally:
+            self._in_progress.discard(cache_key)
+
+    def resolve(
+        self,
+        joined_path: str,
+        *,
+        source_dir: Path | None = None,
+    ) -> Path:
+        """Translate `<root>::<name>(::<sub>...)` into a file path
+        without loading.
+
+        `local::name` resolves against `source_dir`; every other
+        root resolves against `self.search_paths`."""
+        parts = joined_path.split("::")
+        if len(parts) < 2:
+            raise LoaderError(
+                f"path {joined_path!r} must have at least 2 segments"
+            )
+        root, *rest = parts
+
+        if root == LOCAL_ROOT:
+            if source_dir is None:
+                raise LoaderError(
+                    f"`use local::{ '::'.join(rest) };` requires a "
+                    f"source-file directory but none was supplied "
+                    f"(use parse_file or pass source_dir=)"
+                )
+            base = source_dir
+        else:
+            if root not in self.search_paths:
+                raise LoaderError(
+                    f"unknown import root {root!r} "
+                    f"(known: {sorted(self.search_paths) + [LOCAL_ROOT]})"
+                )
+            base = self.search_paths[root]
+
+        # rest = ["math"]      -> math.eml
+        # rest = ["foo", "bar"] -> foo/bar.eml
+        rel = Path(*rest[:-1]) / f"{rest[-1]}.eml"
+        return base / rel
+
+
+def resolve_imports(
+    mod: EMLModule,
+    *,
+    loader: ModuleLoader | None = None,
+) -> EMLModule:
+    """Return a new EMLModule with every `use ...;` import resolved
+    and its constants/types/functions merged into `mod`'s namespace.
+
+    `local::` imports are resolved against the directory of
+    `mod.source_file`; if that is "<unknown>" or "<string>"
+    (parsed from a string, not a file), local imports raise
+    LoaderError -- callers wanting local imports must set the
+    EMLModule.source_file before calling.
+
+    Conflicts (an imported name collides with a local one, or two
+    imports both define the same name) raise LoaderError.
+    """
+    if not mod.imports:
+        return mod
+
+    if loader is None:
+        loader = ModuleLoader()
+
+    # Compute the importing file's directory once so every
+    # local::... lookup uses the same base.
+    source_dir: Path | None = None
+    src = mod.source_file
+    if src and not src.startswith("<"):
+        source_dir = Path(src).resolve().parent
+
+    out = deepcopy(mod)
+
+    # Build name tables seeded with the local module's own decls.
+    # Imports CANNOT shadow a locally-defined name; the local
+    # definition wins and the import is rejected as a clash so the
+    # author sees the conflict explicitly.
+    const_names = {c.name for c in out.constants}
+    type_names = {t.name for t in out.types}
+    func_names = {f.name for f in out.functions}
+
+    # Track which import provided each name so error messages can
+    # point at the offending source.
+    provided_by: dict[str, str] = {}
+
+    for imp in mod.imports:
+        sub = loader.load(imp.joined, source_dir=source_dir)
+
+        # Selective-import filter. When `imp.only` is set, only
+        # names in that allowlist are merged. Names outside it are
+        # NOT brought into the importing module's namespace; they
+        # remain in the imported sub-module and the tree-shaker
+        # later sees them as unreached.
+        wanted = set(imp.only) if imp.only is not None else None
+
+        # Alias map (None -> empty dict). Key is the original
+        # name in the imported module; value is what it should be
+        # called in the importing module's namespace.
+        alias_map: dict[str, str] = imp.aliases or {}
+
+        # Validate the selective-import allowlist against what the
+        # imported module actually exports -- a typo in the user's
+        # `use ::{misspelled}` should surface as a clear error
+        # rather than silently importing nothing.
+        if wanted is not None:
+            exported_names = (
+                {c.name for c in sub.constants}
+                | {t.name for t in sub.types}
+                | {f.name for f in sub.functions}
+            )
+            unknown = sorted(wanted - exported_names)
+            if unknown:
+                raise LoaderError(
+                    f"`use {imp.joined}::{{...}};` requested name(s) "
+                    f"{unknown} not exported by {imp.joined!r} "
+                    f"(available: {sorted(exported_names)[:20]})"
+                )
+
+        for c in sub.constants:
+            if wanted is not None and c.name not in wanted:
+                continue
+            local_name = alias_map.get(c.name, c.name)
+            _check_clash(local_name, "constant", imp, provided_by,
+                         local_names=const_names)
+            new_c = deepcopy(c)
+            new_c.name = local_name
+            out.constants.append(new_c)
+            const_names.add(local_name)
+            provided_by[local_name] = imp.joined
+
+        for t in sub.types:
+            if wanted is not None and t.name not in wanted:
+                continue
+            local_name = alias_map.get(t.name, t.name)
+            _check_clash(local_name, "type alias", imp, provided_by,
+                         local_names=type_names)
+            new_t = deepcopy(t)
+            new_t.name = local_name
+            out.types.append(new_t)
+            type_names.add(local_name)
+            provided_by[local_name] = imp.joined
+
+        for f in sub.functions:
+            if wanted is not None and f.name not in wanted:
+                continue
+            local_name = alias_map.get(f.name, f.name)
+            _check_clash(local_name, "function", imp, provided_by,
+                         local_names=func_names)
+            new_fn = deepcopy(f)
+            new_fn.name = local_name
+            # Tag for the tree-shaker: this function arrived via
+            # `use ...;` and is droppable if no local function
+            # ends up calling it.
+            new_fn.imported_from = imp.joined
+            out.functions.append(new_fn)
+            func_names.add(local_name)
+            provided_by[local_name] = imp.joined
+
+    return out
+
+
+def _check_clash(
+    name: str, kind: str, imp: EMLImport,
+    provided_by: dict[str, str],
+    *,
+    local_names: set[str],
+) -> None:
+    """Raise if `name` is already in scope, distinguishing local
+    redefinitions from import conflicts in the message."""
+    if name in local_names:
+        # Was it locally-defined or imported earlier?
+        if name in provided_by:
+            raise LoaderError(
+                f"`use {imp.joined};` brings in {kind} {name!r}, "
+                f"but it was already imported via "
+                f"`{provided_by[name]}`",
+            )
+        raise LoaderError(
+            f"`use {imp.joined};` brings in {kind} {name!r}, "
+            f"but the importing module already defines that name",
+        )

lang/optimizer/__init__.py

@@ -0,0 +1,111 @@
+"""SuperBEST routing + fusion + CSE + constant folding.
+
+The optimizer is the DEFAULT (not opt-in). Every function passes
+through it before any backend emits code, so all backends share
+the same optimal node count + the same per-node operator choice.
+
+Public entries:
+
+  optimize_module(mod)    Run the full pass sequence over every
+                          function in `mod`. Mutates the module
+                          in place and returns it for chaining.
+  optimize_function(fn)   Same, single function.
+  fold_constants(node)    Just the constant-folding pass.
+  apply_cse(fn)           Just the CSE pass.
+  route_superbest(node)   Just SuperBEST routing (today: identity).
+
+Default pass sequence (in order):
+
+  1. constant_folding  -- fold pure-literal sub-trees first so
+                          downstream passes see a smaller tree.
+  2. cse               -- hoist remaining duplicates into lets.
+  3. superbest         -- per-node operator-family selection
+                          (placeholder; wired in for Phase 2.1).
+"""
+
+from copy import deepcopy
+
+from lang.optimizer.constant_folding import fold_constants, fold_in_place
+from lang.optimizer.cse import apply_cse, apply_cse_module
+from lang.optimizer.inliner import inline_calls
+from lang.optimizer.ml_routing import route_ml_activations_module
+from lang.optimizer.superbest import (
+    route_superbest,
+    superbest_function,
+    superbest_module,
+)
+from lang.optimizer.tree_shaker import shake_imports
+from lang.parser.ast_nodes import EMLFunction, EMLModule, NodeKind
+
+
+def optimize_function(fn: EMLFunction) -> EMLFunction:
+    """Run the per-function pass sequence (constant_folding + CSE +
+    superbest). Returns a new function -- input is not mutated.
+
+    NOTE: the call-inliner is a MODULE-level pass (it needs to
+    look up callees in the module's function table) so it lives
+    in `optimize_module`, not here. Calling `optimize_function`
+    directly skips inlining."""
+    if fn.body is None:
+        return fn
+
+    out = deepcopy(fn)
+
+    # Pass 1: constant folding (in-place on the deep copy).
+    if out.body is not None:
+        out.body = fold_in_place(out.body)
+    # Pass 2: CSE -- operates on the whole function.
+    out = apply_cse(out)
+    # Pass 3: superbest (no-op today).
+    if out.body is not None:
+        out.body = route_superbest(out.body)
+    return out
+
+
+def optimize_module(mod: EMLModule, *,
+                    ml_routing: bool = False) -> EMLModule:
+    """Run the default pass sequence on every function in `mod`.
+    Returns a new module; the input is not mutated.
+
+    Pass order:
+      0. inline_calls       -- substitute eligible same-module
+                               CALLs with the callee's body
+      1. constant_folding   -- fold pure-literal sub-trees
+                               (incl. ones exposed by inlining)
+      2. cse                -- hoist remaining duplicates
+      3. superbest          -- per-node operator-family selection
+      3.5 ml_routing        -- (opt-in) pattern-rewrite sigmoid /
+                               softplus to libmonogate runtime calls
+                               for HIGH-drift functions
+      4. shake_imports      -- drop unused imports
+
+    `ml_routing` defaults to False because the pass emits CALL
+    nodes targeting libmonogate runtime symbols, which only the
+    C and Rust backends know how to resolve. Enable when
+    targeting C / Rust on a known-drifty workload.
+    """
+    # Pass 0: module-level inliner.
+    out = inline_calls(mod)
+    # Passes 1 -> 2: per-function (constant_folding + CSE).
+    out.functions = [optimize_function(fn) for fn in out.functions]
+    # Pass 3: module-level SuperBEST routing -- needs SymPy
+    # bridge access so it lives outside optimize_function.
+    out = superbest_module(out)
+    # Pass 3.5: opt-in ML pattern rewriter (libmonogate runtime).
+    if ml_routing:
+        out = route_ml_activations_module(out)
+    # Pass 4: drop unused imports (after inlining so reachable set
+    # reflects the post-inline call graph).
+    out = shake_imports(out)
+    return out
+
+
+__all__ = [
+    "optimize_function",
+    "optimize_module",
+    "fold_constants",
+    "apply_cse",
+    "apply_cse_module",
+    "route_superbest",
+    "route_ml_activations_module",
+]