optalph 1.0.0__py3-none-any.whl

optalph/__init__.py

@@ -0,0 +1,43 @@
+try:
+    from importlib.metadata import version as _version
+    __version__ = _version("optalph")
+except Exception:
+    __version__ = "0.0.0"
+
+from .config import ENGLISH_ALPHABET, RUSSIAN_ALPHABET, OptimizerConfig, RulesConfig, RunConfig
+from .dictionary import detect_alphabet, download_dictionary, is_valid_word, load_dictionary
+from .evaluator import IncrementalEvaluator, annotate, evaluate, evaluate_detailed
+from .frequencies import load_frequencies
+from .grid_evaluator import IncrementalGridEvaluator
+from .optimizer import SwapEvaluator, greedy_init, iterated_local_search, multistart_sa, simulated_annealing
+from .rules import Adjacent, Compose, ExtractionRule, GridRule, SkipAdjacent, build_rules
+
+__all__ = [
+    "__version__",
+    "RUSSIAN_ALPHABET",
+    "ENGLISH_ALPHABET",
+    "OptimizerConfig",
+    "RulesConfig",
+    "RunConfig",
+    "load_dictionary",
+    "detect_alphabet",
+    "is_valid_word",
+    "download_dictionary",
+    "Adjacent",
+    "SkipAdjacent",
+    "Compose",
+    "ExtractionRule",
+    "GridRule",
+    "build_rules",
+    "evaluate",
+    "evaluate_detailed",
+    "annotate",
+    "IncrementalEvaluator",
+    "IncrementalGridEvaluator",
+    "SwapEvaluator",
+    "greedy_init",
+    "simulated_annealing",
+    "multistart_sa",
+    "iterated_local_search",
+    "load_frequencies",
+]

optalph/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

optalph/_base.py

@@ -0,0 +1,63 @@
+from abc import ABC, abstractmethod
+from typing import Any, Generic, TypeVar
+
+_SK = TypeVar("_SK")
+
+
+class _IncrementalBase(ABC, Generic[_SK]):
+    dictionary: set[str]
+    weights: dict[str, float] | None
+    n: int
+    _current: list[str]
+    _current_score: float
+    _slots: dict[_SK, str]
+    _word_refs: dict[str, set[_SK]]
+    _valid_score: float
+
+    def _word_weight(self, word: str) -> float:
+        if word not in self.dictionary:
+            return 0.0
+        if self.weights is None:
+            return 1.0
+        return self.weights.get(word, 1.0)
+
+    def _put_slot(self, key: _SK, word: str) -> None:
+        old_word = self._slots.get(key)
+        if old_word is not None:
+            self._del_slot(key)
+        self._slots[key] = word
+        if word not in self._word_refs:
+            self._word_refs[word] = set()
+            self._valid_score += self._word_weight(word)
+        self._word_refs[word].add(key)
+
+    def _del_slot(self, key: _SK) -> None:
+        word = self._slots.pop(key)
+        refs = self._word_refs[word]
+        refs.discard(key)
+        if not refs:
+            del self._word_refs[word]
+            self._valid_score -= self._word_weight(word)
+
+    @abstractmethod
+    def full_eval(self, permutation: tuple[str, ...] | list[str]) -> float: ...
+
+    @abstractmethod
+    def try_swap(self, i: int, j: int) -> tuple[float, list[tuple[_SK, str]], Any]: ...
+
+    @abstractmethod
+    def undo_swap(self, i: int, j: int, snapshot: list[tuple[_SK, str]], affected: Any) -> float: ...
+
+    @property
+    def score(self) -> float:
+        return self._current_score
+
+    def get_permutation(self) -> tuple[str, ...]:
+        return tuple(self._current)
+
+    def _clone_common(self, target: Any) -> None:
+        target._current = self._current[:]
+        target._current_score = self._current_score
+        target._slots = dict(self._slots)
+        target._word_refs = {w: set(refs) for w, refs in self._word_refs.items()}
+        target._valid_score = self._valid_score

optalph/_cli.py

@@ -0,0 +1,132 @@
+import argparse
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Optimize alphabet permutation for max dictionary words"
+    )
+    parser.add_argument(
+        "--alphabet", "-a", type=str, default=None,
+        help="Custom alphabet string (default: Russian)",
+    )
+    parser.add_argument(
+        "--window", "-w", type=int, nargs="+", default=[3],
+        help="Adjacent window size(s), e.g. -w 3 4 5",
+    )
+    parser.add_argument(
+        "--skip", "-s", type=int, nargs="+", default=[],
+        help="Skip rules as k:step pairs, e.g. -s 3 2 means SkipAdjacent(3,2)",
+    )
+    parser.add_argument(
+        "--method", choices=["sa", "ils"], default="sa",
+        help="Optimization method",
+    )
+    parser.add_argument(
+        "--iter", "-i", type=int, default=500000,
+        help="Max iterations for simulated annealing",
+    )
+    parser.add_argument(
+        "--t-start", type=float, default=100.0,
+        help="Starting temperature (default: 100)",
+    )
+    parser.add_argument(
+        "--t-end", type=float, default=0.01,
+        help="End temperature",
+    )
+    parser.add_argument(
+        "--cooling", type=float, default=0.9999,
+        help="Cooling rate per iteration (default: 0.9999)",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=None,
+        help="Random seed for reproducibility",
+    )
+    parser.add_argument(
+        "--dict-file", type=str, default=None,
+        help="External dictionary file (one word per line)",
+    )
+    parser.add_argument(
+        "--no-dawg", action="store_true",
+        help="Skip DAWG dictionary, use only external file",
+    )
+    parser.add_argument(
+        "--download", choices=["english", "russian"],
+        help="Download dictionary automatically (cached in ~/.optalph/)",
+    )
+    parser.add_argument(
+        "--restarts", "-r", type=int, default=1,
+        help="Number of multistart restarts",
+    )
+    parser.add_argument(
+        "--freq-file", type=str, default=None,
+        help="Word frequency file for weighted scoring (format: word count)",
+    )
+    parser.add_argument(
+        "--incremental", action="store_true",
+        help="Use incremental evaluator (faster for large alphabets/grids)",
+    )
+    parser.add_argument(
+        "--save", type=str, default=None,
+        help="Save results to JSON file",
+    )
+    parser.add_argument(
+        "--detailed", action="store_true",
+        help="Show detailed results (found words)",
+    )
+    parser.add_argument(
+        "--show-invalid", action="store_true",
+        help="Also show invalid candidate words",
+    )
+    parser.add_argument(
+        "--quiet", "-q", action="store_true",
+        help="Minimal output",
+    )
+    parser.add_argument(
+        "--tabu", action="store_true",
+        help="Enable tabu list for SA (prevents cycling)",
+    )
+    parser.add_argument(
+        "--tabu-tenure", type=int, default=20,
+        help="Tabu list tenure (default: 20)",
+    )
+    parser.add_argument(
+        "--random-init", action="store_true",
+        help="Use random initial permutation instead of greedy bigram init",
+    )
+    parser.add_argument(
+        "--grid-rows", type=int, default=None,
+        help="Grid rows for 2D Boggle-like extraction (requires --grid-cols)",
+    )
+    parser.add_argument(
+        "--grid-cols", type=int, default=None,
+        help="Grid columns for 2D Boggle-like extraction (requires --grid-rows)",
+    )
+    parser.add_argument(
+        "--grid-min-len", type=int, default=3,
+        help="Minimum word length for grid extraction (default: 3)",
+    )
+    parser.add_argument(
+        "--ils", action="store_true",
+        help="Use Iterated Local Search (SA + perturbations)",
+    )
+    parser.add_argument(
+        "--ils-perturb", type=int, default=3,
+        help="ILS perturbation strength (number of swaps, default: 3)",
+    )
+    parser.add_argument(
+        "--ils-steps", type=int, default=10,
+        help="ILS number of perturbation steps (default: 10)",
+    )
+    parser.add_argument(
+        "--parallel", action="store_true",
+        help="Run multistart restarts in parallel (score_fn mode only)",
+    )
+    return parser.parse_args(argv)
+
+
+def build_skip_rules(skip_args: list[int]) -> list[tuple[int, int]]:
+    if not skip_args:
+        return []
+    if len(skip_args) % 2 != 0:
+        raise ValueError("--skip requires pairs of arguments (k step k step ...)")
+    return [(skip_args[i], skip_args[i + 1]) for i in range(0, len(skip_args), 2)]

optalph/config.py

@@ -0,0 +1,67 @@
+from dataclasses import dataclass, field
+
+__all__ = ["RUSSIAN_ALPHABET", "ENGLISH_ALPHABET", "OptimizerConfig", "RulesConfig", "RunConfig"]
+
+RUSSIAN_ALPHABET = tuple("абвгдеёжзийклмнопрстуфхцчшщъыьэюя")
+ENGLISH_ALPHABET = tuple("abcdefghijklmnopqrstuvwxyz")
+
+
+@dataclass
+class OptimizerConfig:
+    """Simulated annealing configuration."""
+    t_start: float = 100.0
+    t_end: float = 0.01
+    cooling_rate: float = 0.9999
+    max_iterations: int = 500000
+    seed: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.t_start <= 0:
+            raise ValueError("t_start must be > 0")
+        if self.t_end < 0:
+            raise ValueError("t_end must be >= 0")
+        if self.t_end >= self.t_start:
+            raise ValueError("t_end must be < t_start")
+        if not 0 < self.cooling_rate < 1:
+            raise ValueError("cooling_rate must be in (0, 1)")
+        if self.max_iterations < 1:
+            raise ValueError("max_iterations must be >= 1")
+
+
+@dataclass
+class RulesConfig:
+    """Extraction rule configuration."""
+    adjacent_windows: list[int] = field(default_factory=lambda: [3])
+    skip_rules: list[tuple[int, int]] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if not all(w >= 1 for w in self.adjacent_windows):
+            raise ValueError("All adjacent_windows must be >= 1")
+        for k, s in self.skip_rules:
+            if k < 1:
+                raise ValueError("Skip rule k must be >= 1")
+            if s < 1:
+                raise ValueError("Skip rule step must be >= 1")
+        if not self.adjacent_windows and not self.skip_rules:
+            raise ValueError("At least one rule must be specified")
+
+
+@dataclass
+class RunConfig:
+    """Top-level run configuration."""
+    alphabet: tuple[str, ...] = RUSSIAN_ALPHABET
+    optimizer: OptimizerConfig = field(default_factory=OptimizerConfig)
+    rules: RulesConfig = field(default_factory=RulesConfig)
+    dict_file: str | None = None
+    use_dawg: bool = False
+    min_word_len: int = 3
+
+    def __post_init__(self) -> None:
+        if self.min_word_len < 1:
+            raise ValueError("min_word_len must be >= 1")
+        if len(self.alphabet) < 2:
+            raise ValueError("alphabet must contain at least 2 characters")
+        if self.min_word_len > len(self.alphabet):
+            raise ValueError(f"min_word_len ({self.min_word_len}) > alphabet size ({len(self.alphabet)})")
+        if len(set(self.alphabet)) != len(self.alphabet):
+            raise ValueError("alphabet must not contain duplicate characters")

optalph/dictionary.py

@@ -0,0 +1,249 @@
+import logging
+import os
+import unicodedata
+import urllib.request
+from pathlib import Path
+
+from . import __version__
+
+__all__ = ["load_dictionary", "detect_alphabet", "download_dictionary"]
+
+logger = logging.getLogger(__name__)
+
+_CACHE_DIR = Path.home() / ".optalph"
+
+_DICTIONARY_URLS: dict[str, str] = {
+    "english": "https://raw.githubusercontent.com/dwyl/english-words/master/words_alpha.txt",
+    "russian": "https://raw.githubusercontent.com/hingston/russian/master/russian.txt",
+}
+
+
+def _ensure_cache_dir() -> Path:
+    _CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    return _CACHE_DIR
+
+
+def _cache_path(name: str) -> Path:
+    return _ensure_cache_dir() / f"{name}.txt"
+
+
+def download_dictionary(name: str, force: bool = False) -> str:
+    """Download a dictionary to ~/.optalph/ and return the cached file path.
+
+    Supported names: 'english', 'russian'.
+    Downloads only if not already cached. Use force=True to re-download.
+    """
+    name = name.lower()
+    if name not in _DICTIONARY_URLS:
+        raise ValueError(f"Unknown dictionary '{name}'. Available: {', '.join(_DICTIONARY_URLS)}")
+
+    url = _DICTIONARY_URLS[name]
+    cached = _cache_path(name)
+
+    if cached.is_file() and not force:
+        logger.info("Using cached dictionary: %s", cached)
+        return str(cached)
+
+    logger.info("Downloading dictionary '%s' from %s...", name, url)
+
+    tmp = cached.with_suffix(".tmp")
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": f"optalph/{__version__}"})
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            with open(tmp, "wb") as f:
+                while True:
+                    chunk = resp.read(8192)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+        tmp.rename(cached)
+    except Exception as e:
+        if tmp.is_file():
+            tmp.unlink()
+        raise RuntimeError(f"Failed to download dictionary: {e}") from e
+
+    logger.info("Downloaded %s (%.1f MB)", name, cached.stat().st_size / 1e6)
+    return str(cached)
+
+
+def is_valid_word(word: str, allowed_chars: set[str] | None = None) -> bool:
+    if allowed_chars is not None:
+        return all(c in allowed_chars for c in word)
+    return word.isalpha()
+
+
+def load_from_dawg(min_len: int = 1, max_len: int = 33, allowed_chars: set[str] | None = None) -> set[str]:
+    """Load words from pymorphy2 DAWG dictionary files."""
+    if min_len > max_len:
+        raise ValueError(f"min_len ({min_len}) > max_len ({max_len})")
+    try:
+        import dawg_python
+    except ImportError:
+        raise ImportError("dawg_python not installed. Run: uv add dawg-python")
+
+    dict_path = None
+    try:
+        from pymorphy2_dicts_ru import get_path
+        dict_path = get_path()
+    except ImportError:
+        try:
+            import pymorphy2_dicts_ru
+            dict_path = os.path.dirname(pymorphy2_dicts_ru.__file__)
+        except ImportError:
+            pass
+
+    if dict_path is None:
+        raise RuntimeError("Cannot find pymorphy2_dicts_ru path")
+
+    words: set[str] = set()
+
+    words_file = os.path.join(dict_path, "words.dawg")
+    if not os.path.isfile(words_file):
+        raise FileNotFoundError(f"words.dawg not found at {dict_path}")
+
+    try:
+        d = dawg_python.BytesDAWG()
+        d.load(words_file)
+        for key in d.keys():
+            if isinstance(key, bytes):
+                word = key.decode("utf-8").lower()
+            else:
+                word = key.lower()
+            if min_len <= len(word) <= max_len and is_valid_word(word, allowed_chars):
+                words.add(word)
+    except (OSError, ValueError) as e:
+        raise ValueError(f"Failed to load words.dawg: {e}") from e
+
+    return words
+
+
+_ENCODINGS = ["utf-8-sig", "cp1251", "koi8-r", "cp866"]
+
+
+def detect_alphabet(filepath: str, sample: int = 1000) -> set[str]:
+    """Detect unique alphabetic characters from the first `sample` words.
+
+    Returns empty set if <90% of detected characters are letters (category L*)
+    or if fewer than 4 unique letters found — guards against garbage decoded
+    via latin-1 fallback.
+    """
+    chars: set[str] = set()
+    count = 0
+    for enc in _ENCODINGS + ["latin-1"]:
+        try:
+            with open(filepath, "r", encoding=enc) as f:
+                for line in f:
+                    word = line.strip().lower()
+                    if word:
+                        chars.update(c for c in word if c.isalpha())
+                        count += 1
+                        if count >= sample:
+                            break
+            if chars:
+                letter_count = sum(
+                    1 for c in chars
+                    if unicodedata.category(c).startswith("L")
+                )
+                if letter_count >= 4 and letter_count / len(chars) >= 0.9:
+                    return chars
+                chars.clear()
+        except (UnicodeDecodeError, UnicodeError):
+            continue
+    return set()
+
+
+def load_from_file(
+    filepath: str, min_len: int = 1, max_len: int = 33,
+    allowed_chars: set[str] | None = None,
+) -> set[str]:
+    """Load words from a text file. Tries utf-8, cp1251, koi8-r, cp866, latin-1."""
+    if not os.path.isfile(filepath):
+        raise FileNotFoundError(f"Dictionary file not found: {filepath}")
+    if min_len > max_len:
+        raise ValueError(f"min_len ({min_len}) > max_len ({max_len})")
+
+    words: set[str] = set()
+    encodings = _ENCODINGS
+    for enc in encodings:
+        try:
+            with open(filepath, "r", encoding=enc) as f:
+                for line in f:
+                    word = line.strip().lower()
+                    if min_len <= len(word) <= max_len and is_valid_word(word, allowed_chars):
+                        words.add(word)
+            if not words:
+                raise ValueError(f"No valid words found in {filepath}")
+            return words
+        except (UnicodeDecodeError, UnicodeError):
+            words.clear()
+            continue
+
+    if not words:
+        with open(filepath, "r", encoding="latin-1") as f:
+            for line in f:
+                word = line.strip().lower()
+                if min_len <= len(word) <= max_len and is_valid_word(word, allowed_chars):
+                    words.add(word)
+
+    if not words:
+        raise ValueError(f"No valid words found in {filepath}")
+    return words
+
+
+def load_dictionary(
+    filepath: str | None = None,
+    use_dawg: bool = True,
+    min_len: int = 1,
+    max_len: int = 33,
+    quiet: bool = False,
+    alphabet: tuple[str, ...] | None = None,
+    download: str | None = None,
+) -> set[str]:
+    """Load dictionary from DAWG and/or external file. Returns set of words.
+    alphabet: if provided, only words using characters from alphabet are kept.
+    download: 'english' or 'russian' — auto-download if no other source works.
+    """
+    allowed_chars = set(alphabet) if alphabet is not None else None
+    words: set[str] = set()
+
+    if use_dawg:
+        try:
+            if not quiet:
+                logger.info("Loading dictionary from DAWG (pymorphy2-dicts-ru)...")
+            words.update(load_from_dawg(min_len, max_len, allowed_chars))
+            if not quiet:
+                logger.info("  dawg: %d words", len(words))
+        except (ImportError, RuntimeError, FileNotFoundError) as e:
+            if not quiet:
+                logger.info("  DAWG not available: %s", e)
+
+    if filepath:
+        if not quiet:
+            logger.info("Loading dictionary from %s...", filepath)
+        try:
+            file_words = load_from_file(filepath, min_len, max_len, allowed_chars)
+            if not quiet:
+                logger.info("  file: %d words", len(file_words))
+            words.update(file_words)
+        except (ValueError, FileNotFoundError) as e:
+            if not quiet:
+                logger.info("  file load failed: %s", e)
+
+    if not words and download:
+        cached = download_dictionary(download)
+        if not quiet:
+            logger.info("Loading downloaded dictionary...")
+        dl_words = load_from_file(cached, min_len, max_len, allowed_chars)
+        if not quiet:
+            logger.info("  downloaded: %d words", len(dl_words))
+        words.update(dl_words)
+
+    if not words:
+        raise ValueError(
+            "No dictionary words loaded. Provide --dict-file, install pymorphy2-dicts-ru, "
+            "or use --download english/russian"
+        )
+
+    if not quiet:
+        logger.info("  total unique: %d words", len(words))
+    return words