multilingualprogramming 0.2.0__py3-none-any.whl

multilingualprogramming/imports.py

@@ -0,0 +1,150 @@
+#
+# SPDX-FileCopyrightText: 2026 John Samuel <johnsamuelwrites@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+"""Import hook for loading `.ml` modules/packages."""
+
+import importlib.abc
+import importlib.util
+import sys
+from pathlib import Path
+
+from multilingualprogramming.codegen.python_generator import PythonCodeGenerator
+from multilingualprogramming.codegen.runtime_builtins import RuntimeBuiltins
+from multilingualprogramming.core.lowering import lower_to_core_ir
+from multilingualprogramming.lexer.lexer import Lexer
+from multilingualprogramming.parser.parser import Parser
+from multilingualprogramming.parser.semantic_analyzer import SemanticAnalyzer
+
+
+class _MLLoader(importlib.abc.Loader):
+    """Loader that transpiles and executes `.ml` source modules."""
+
+    _CODE_CACHE = {}
+
+    def __init__(self, source_path, is_package=False):
+        self.source_path = Path(source_path)
+        self._is_package = is_package
+
+    def is_package(self, fullname):
+        """Return whether this loader target is a package."""
+        del fullname
+        return self._is_package
+
+    def _build_code_object(self):
+        """Compile multilingual source to a cached Python code object."""
+        stat = self.source_path.stat()
+        cache_key = str(self.source_path.resolve())
+        cache_entry = self._CODE_CACHE.get(cache_key)
+        stamp = (stat.st_mtime_ns, stat.st_size)
+        if cache_entry and cache_entry["stamp"] == stamp:
+            return cache_entry["code"], cache_entry["language"]
+
+        with open(self.source_path, "r", encoding="utf-8") as handle:
+            source = handle.read()
+
+        lexer = Lexer(source, language=None)
+        tokens = lexer.tokenize()
+        detected_language = lexer.language or "en"
+
+        parser = Parser(tokens, source_language=detected_language)
+        program = parser.parse()
+        core_program = lower_to_core_ir(
+            program, detected_language, frontend_name="lexer_parser"
+        )
+
+        analyzer = SemanticAnalyzer(source_language=detected_language)
+        builtins_ns = RuntimeBuiltins(detected_language).namespace()
+        for name in builtins_ns:
+            analyzer.symbol_table.define(name, "variable", line=0, column=0)
+
+        semantic_errors = analyzer.analyze(core_program.ast)
+        if semantic_errors:
+            rendered = "; ".join(str(err) for err in semantic_errors)
+            raise ImportError(
+                f"Semantic checks failed for {self.source_path}: {rendered}"
+            )
+
+        python_source = PythonCodeGenerator().generate(core_program)
+        code_obj = compile(python_source, str(self.source_path), "exec")
+
+        self._CODE_CACHE[cache_key] = {
+            "stamp": stamp,
+            "code": code_obj,
+            "language": detected_language,
+        }
+        return code_obj, detected_language
+
+    def exec_module(self, module):
+        """Execute the transpiled module code in the module namespace."""
+        code_obj, language = self._build_code_object()
+        for name, builtin_obj in RuntimeBuiltins(language).namespace().items():
+            module.__dict__.setdefault(name, builtin_obj)
+        exec(code_obj, module.__dict__)  # pylint: disable=exec-used
+
+
+class _MLFinder(importlib.abc.MetaPathFinder):
+    """Meta-path finder for `.ml` modules and packages."""
+
+    @staticmethod
+    def _candidate_spec(fullname, base_dir):
+        module_name = fullname.rsplit(".", 1)[-1]
+        package_dir = Path(base_dir) / module_name
+        package_init = package_dir / "__init__.ml"
+        if package_init.is_file():
+            loader = _MLLoader(package_init, is_package=True)
+            return importlib.util.spec_from_file_location(
+                fullname,
+                package_init,
+                loader=loader,
+                submodule_search_locations=[str(package_dir)],
+            )
+
+        module_file = Path(base_dir) / f"{module_name}.ml"
+        if module_file.is_file():
+            loader = _MLLoader(module_file, is_package=False)
+            return importlib.util.spec_from_file_location(
+                fullname,
+                module_file,
+                loader=loader,
+            )
+        return None
+
+    def find_spec(self, fullname, path=None, target=None):
+        """Find an import spec for modules/packages backed by `.ml` files."""
+        del target
+        search_paths = path if path is not None else sys.path
+        for base in search_paths:
+            if not base:
+                base = "."
+            try:
+                spec = self._candidate_spec(fullname, base)
+            except OSError:
+                continue
+            if spec is not None:
+                return spec
+        return None
+
+
+_FINDER = None
+
+
+def enable_multilingual_imports():
+    """Enable import of `.ml` modules in the active Python process."""
+    global _FINDER  # pylint: disable=global-statement
+    if _FINDER is not None and _FINDER in sys.meta_path:
+        return _FINDER
+    finder = _MLFinder()
+    sys.meta_path.insert(0, finder)
+    _FINDER = finder
+    return finder
+
+
+def disable_multilingual_imports():
+    """Disable import of `.ml` modules in the active Python process."""
+    global _FINDER  # pylint: disable=global-statement
+    if _FINDER is not None and _FINDER in sys.meta_path:
+        sys.meta_path.remove(_FINDER)
+    _FINDER = None

multilingualprogramming/keyword/__init__.py

@@ -0,0 +1,13 @@
+#
+# SPDX-FileCopyrightText: 2024 John Samuel <johnsamuelwrites@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+"""Keyword subpackage for multilingual programming."""
+
+from multilingualprogramming.keyword.keyword_registry import KeywordRegistry
+from multilingualprogramming.keyword.keyword_validator import KeywordValidator
+from multilingualprogramming.keyword.language_pack_validator import (
+    LanguagePackValidator,
+)

multilingualprogramming/keyword/keyword_registry.py

@@ -0,0 +1,249 @@
+#
+# SPDX-FileCopyrightText: 2024 John Samuel <johnsamuelwrites@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+"""Registry for multilingual keyword lookups based on the Universal Semantic Model."""
+# pylint: disable=unsubscriptable-object
+
+import json
+from pathlib import Path
+from typing import Any, cast
+from multilingualprogramming.exceptions import (
+    UnknownKeywordError,
+    UnsupportedLanguageError,
+)
+
+
+class KeywordRegistry:
+    """
+    Loads and queries the USM keyword ontology.
+
+    Provides forward lookup (concept -> keyword), reverse lookup
+    (keyword -> concept), and language detection from a set of keywords.
+    """
+
+    _instance = None
+    _data: dict[str, Any] | None = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        if KeywordRegistry._data is None:
+            self._load()
+        if not hasattr(self, "_reverse_index"):
+            self._reverse_index = {}
+        if not hasattr(self, "_concept_map"):
+            self._concept_map = {}
+
+    def _load(self):
+        """Load keywords.json from the resources directory."""
+        resources_dir = Path(__file__).parent.parent / "resources" / "usm"
+        keywords_path = resources_dir / "keywords.json"
+        with open(keywords_path, "r", encoding="utf-8") as f:
+            KeywordRegistry._data = json.load(f)
+
+        # Build reverse index: {language: {keyword_string: [concept_id, ...]}}
+        self._reverse_index = {}
+        # Build flat concept map: {concept_id: {language: keyword | [keywords]}}
+        self._concept_map = {}
+        data = cast(dict[str, Any] | None, KeywordRegistry._data)
+        if data is None:
+            raise UnsupportedLanguageError("unknown")
+
+        for category_concepts in data["categories"].values():
+            for concept_id, translations in category_concepts.items():
+                self._concept_map[concept_id] = translations
+                for lang, keyword_value in translations.items():
+                    if lang not in self._reverse_index:
+                        self._reverse_index[lang] = {}
+                    keywords = (
+                        keyword_value
+                        if isinstance(keyword_value, list)
+                        else [keyword_value]
+                    )
+                    for keyword in keywords:
+                        if keyword not in self._reverse_index[lang]:
+                            self._reverse_index[lang][keyword] = []
+                        self._reverse_index[lang][keyword].append(concept_id)
+
+    def _check_language(self, language):
+        """Raise UnsupportedLanguageError if language is not in the registry."""
+        data = cast(dict[str, Any] | None, self._data)
+        if data is None:
+            raise UnsupportedLanguageError(language)
+        if language not in data["languages"]:
+            raise UnsupportedLanguageError(language)
+
+    def check_language(self, language):
+        """Public language validation helper."""
+        self._check_language(language)
+
+    def get_keyword(self, concept_id, language):
+        """
+        Look up a keyword for a concept in a given language.
+
+        Parameters:
+            concept_id (str): The USM concept ID (e.g., "COND_IF")
+            language (str): Language code (e.g., "en", "fr", "hi")
+
+        Returns:
+            str: The keyword string in the requested language
+
+        Raises:
+            UnsupportedLanguageError: If the language is not supported
+            UnknownKeywordError: If the concept_id is not found
+        """
+        self._check_language(language)
+        if concept_id not in self._concept_map:
+            raise UnknownKeywordError(concept_id)
+        translations = self._concept_map[concept_id]
+        if language not in translations:
+            raise UnsupportedLanguageError(
+                f"{language} (not available for concept {concept_id})"
+            )
+        keyword_value = translations[language]
+        if isinstance(keyword_value, list):
+            return keyword_value[0]
+        return keyword_value
+
+    def get_concept(self, keyword, language):
+        """
+        Reverse lookup: given a keyword in a language, return the concept ID.
+
+        Parameters:
+            keyword (str): The keyword string (e.g., "si", "अगर")
+            language (str): Language code
+
+        Returns:
+            str: The concept ID
+
+        Raises:
+            UnsupportedLanguageError: If the language is not supported
+            UnknownKeywordError: If the keyword is not found
+        """
+        self._check_language(language)
+        lang_index = self._reverse_index.get(language, {})
+        concepts = lang_index.get(keyword)
+        if concepts is None:
+            raise UnknownKeywordError(f"'{keyword}' in language '{language}'")
+        if len(concepts) == 1:
+            return concepts[0]
+        # Return the first match; ambiguity is checked via KeywordValidator
+        return concepts[0]
+
+    def get_concepts(self, keyword, language):
+        """
+        Get all concept IDs that match a keyword in a language.
+
+        Parameters:
+            keyword (str): The keyword string
+            language (str): Language code
+
+        Returns:
+            list[str]: List of matching concept IDs
+        """
+        self._check_language(language)
+        lang_index = self._reverse_index.get(language, {})
+        return lang_index.get(keyword, [])
+
+    def is_keyword(self, word, language):
+        """
+        Check if a word is a recognized keyword in the given language.
+
+        Parameters:
+            word (str): The word to check
+            language (str): Language code
+
+        Returns:
+            bool: True if the word is a keyword
+        """
+        lang_index = self._reverse_index.get(language, {})
+        return word in lang_index
+
+    def get_all_keywords(self, language):
+        """
+        Get all keywords for a language.
+
+        Parameters:
+            language (str): Language code
+
+        Returns:
+            dict: Mapping of concept_id -> keyword string
+        """
+        self._check_language(language)
+        result = {}
+        for concept_id, translations in self._concept_map.items():
+            if language in translations:
+                keyword_value = translations[language]
+                if isinstance(keyword_value, list):
+                    result[concept_id] = keyword_value[0]
+                else:
+                    result[concept_id] = keyword_value
+        return result
+
+    def get_supported_languages(self):
+        """
+        Get list of all supported languages.
+
+        Returns:
+            list[str]: Language codes
+        """
+        data = cast(dict[str, Any] | None, self._data)
+        if data is None:
+            return []
+        return list(data["languages"])
+
+    def detect_language(self, keywords):
+        """
+        Given a list of keyword strings, detect which language they belong to.
+
+        Scores each language by how many of the provided keywords match.
+
+        Parameters:
+            keywords (list[str]): List of keyword strings
+
+        Returns:
+            str | None: The language code with the most matches, or None
+        """
+        data = cast(dict[str, Any] | None, self._data)
+        if data is None:
+            return None
+        scores = {}
+        for lang in data["languages"]:
+            lang_index = self._reverse_index.get(lang, {})
+            score = sum(1 for kw in keywords if kw in lang_index)
+            if score > 0:
+                scores[lang] = score
+
+        if not scores:
+            return None
+        return max(scores, key=scores.get)
+
+    def get_all_concept_ids(self):
+        """
+        Get all concept IDs in the registry.
+
+        Returns:
+            list[str]: All concept IDs
+        """
+        return list(self._concept_map.keys())
+
+    def get_language_index(self, language):
+        """Get reverse index for a language."""
+        self._check_language(language)
+        return self._reverse_index.get(language, {})
+
+    def get_concept_map(self):
+        """Get concept map used for validation workflows."""
+        return self._concept_map
+
+    @classmethod
+    def reset(cls):
+        """Reset the singleton instance (useful for testing)."""
+        cls._instance = None
+        cls._data = None

multilingualprogramming/keyword/keyword_validator.py

@@ -0,0 +1,59 @@
+#
+# SPDX-FileCopyrightText: 2024 John Samuel <johnsamuelwrites@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+"""Validator for keyword mappings, checking ambiguity and completeness."""
+
+from multilingualprogramming.keyword.keyword_registry import KeywordRegistry
+
+
+class KeywordValidator:
+    """
+    Validates keyword mappings for a language.
+
+    Checks for:
+    - Ambiguity: same string maps to multiple concepts
+    - Completeness: all concepts have a translation
+    """
+
+    def __init__(self):
+        self.registry = KeywordRegistry()
+
+    def validate_no_ambiguity(self, language):
+        """
+        Check if any keywords in a language map to multiple concepts.
+
+        Parameters:
+            language (str): Language code
+
+        Returns:
+            list[tuple[str, list[str]]]: List of (keyword, [concept_ids])
+                for ambiguous keywords. Empty list if no ambiguities.
+        """
+        self.registry.check_language(language)
+        ambiguities = []
+        lang_index = self.registry.get_language_index(language)
+        for keyword, concepts in lang_index.items():
+            if len(concepts) > 1:
+                ambiguities.append((keyword, concepts))
+        return ambiguities
+
+    def validate_completeness(self, language):
+        """
+        Check if a language has translations for all concepts.
+
+        Parameters:
+            language (str): Language code
+
+        Returns:
+            list[str]: List of concept IDs missing translations.
+                Empty list if complete.
+        """
+        self.registry.check_language(language)
+        missing = []
+        for concept_id, translations in self.registry.get_concept_map().items():
+            if language not in translations:
+                missing.append(concept_id)
+        return missing

multilingualprogramming/keyword/language_pack_validator.py

@@ -0,0 +1,110 @@
+#
+# SPDX-FileCopyrightText: 2026 John Samuel <johnsamuelwrites@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+"""Validation helpers for language pack smoke checks."""
+
+import json
+from pathlib import Path
+
+from multilingualprogramming.codegen.executor import ProgramExecutor
+from multilingualprogramming.keyword.keyword_registry import KeywordRegistry
+from multilingualprogramming.keyword.keyword_validator import KeywordValidator
+
+
+class LanguagePackValidator:
+    """Validate consistency and basic execution for one language pack."""
+
+    def __init__(self):
+        self._registry = KeywordRegistry()
+        self._keyword_validator = KeywordValidator()
+        resources = Path(__file__).resolve().parent.parent / "resources"
+        self._error_messages_path = resources / "parser" / "error_messages.json"
+        self._repl_commands_path = resources / "repl" / "commands.json"
+
+    @staticmethod
+    def _load_json(path):
+        with open(path, "r", encoding="utf-8-sig") as handle:
+            return json.load(handle)
+
+    def _validate_error_messages(self, language):
+        data = self._load_json(self._error_messages_path)
+        missing = []
+        for message_key, by_language in data.get("messages", {}).items():
+            if language not in by_language:
+                missing.append(message_key)
+        if not missing:
+            return None
+        return f"missing parser error messages for {len(missing)} keys"
+
+    def _validate_repl_catalog(self, language):
+        data = self._load_json(self._repl_commands_path)
+
+        if language not in data.get("help_titles", {}):
+            return "missing REPL help title"
+
+        for message_key, by_language in data.get("messages", {}).items():
+            if language not in by_language:
+                return f"missing REPL message '{message_key}'"
+
+        commands = data.get("commands", {})
+        for command_name, meta in commands.items():
+            aliases = meta.get("aliases", {}).get(language)
+            descriptions = meta.get("descriptions", {}).get(language)
+            if aliases is None:
+                return f"missing REPL aliases for '{command_name}'"
+            if not aliases:
+                return f"empty REPL aliases for '{command_name}'"
+            if descriptions is None:
+                return f"missing REPL description for '{command_name}'"
+        return None
+
+    def _validate_executor_smoke(self, language):
+        let_kw = self._registry.get_keyword("LET", language)
+        print_kw = self._registry.get_keyword("PRINT", language)
+        source = f"{let_kw} x = 1\n{print_kw}(x)\n"
+        result = ProgramExecutor(language=language).execute(source)
+        if not result.success:
+            return "executor smoke program failed"
+        if result.output.strip() != "1":
+            return f"executor smoke output mismatch: {result.output!r}"
+        return None
+
+    def get_supported_languages(self):
+        """Return supported language codes from the keyword registry."""
+        return self._registry.get_supported_languages()
+
+    def validate(self, language):
+        """
+        Return a list of validation errors for a language.
+
+        Empty list means validation passed.
+        """
+        self._registry.check_language(language)
+        errors = []
+
+        missing_concepts = self._keyword_validator.validate_completeness(language)
+        if missing_concepts:
+            errors.append(
+                f"missing keyword translations for {len(missing_concepts)} concepts"
+            )
+
+        ambiguities = self._keyword_validator.validate_no_ambiguity(language)
+        if ambiguities:
+            errors.append(f"ambiguous keywords found: {len(ambiguities)}")
+
+        error_msg_issue = self._validate_error_messages(language)
+        if error_msg_issue:
+            errors.append(error_msg_issue)
+
+        repl_issue = self._validate_repl_catalog(language)
+        if repl_issue:
+            errors.append(repl_issue)
+
+        exec_issue = self._validate_executor_smoke(language)
+        if exec_issue:
+            errors.append(exec_issue)
+
+        return errors

multilingualprogramming/lexer/__init__.py

@@ -0,0 +1,11 @@
+#
+# SPDX-FileCopyrightText: 2024 John Samuel <johnsamuelwrites@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+"""Lexer subpackage for multilingual programming."""
+
+from multilingualprogramming.lexer.token_types import TokenType
+from multilingualprogramming.lexer.token import Token
+from multilingualprogramming.lexer.lexer import Lexer