stark-engine 4.4.0__tar.gz → 4.4.1__tar.gz

{stark_engine-4.4.0 → stark_engine-4.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stark-engine
-Version: 4.4.0
+Version: 4.4.1
 Summary: S.T.A.R.K - Speech and Text Algorithmic Recognition Kit. Modern framework for creating powerfull voice assistants.
 License: CC BY-NC-SA 4.0
 License-File: LICENSE.md

{stark_engine-4.4.0 → stark_engine-4.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "stark-engine"
-version = "4.4.0"
+version = "4.4.1"
 description = "S.T.A.R.K - Speech and Text Algorithmic Recognition Kit. Modern framework for creating powerfull voice assistants."
 authors = ["MarkParker5 <mark@parker-programs.com>"]
 license = "CC BY-NC-SA 4.0"

stark_engine-4.4.1/stark/__init__.py

@@ -0,0 +1,79 @@
+import asyncer
+
+from stark.core import (
+    Command,
+    CommandsContext,
+    CommandsManager,
+    Pattern,
+    Response,
+    ResponseStatus,
+)
+from stark.core.commands_context_processor import CommandsContextProcessor
+from stark.core.health_check import health_check
+from stark.core.processors.search_processor import SearchProcessor
+from stark.general.blockage_detector import BlockageDetector
+from stark.general.localisation import Localizer
+from stark.interfaces.protocols import (
+    SpeechRecognizer,
+    SpeechRecognizerDelegate,
+    SpeechSynthesizer,
+    SpeechSynthesizerResult,
+)
+from stark.voice_assistant import Mode, VoiceAssistant
+
+
+async def run(
+    manager: CommandsManager,
+    speech_recognizer: SpeechRecognizer | list[SpeechRecognizer],
+    speech_synthesizer: SpeechSynthesizer,
+    processors: list[CommandsContextProcessor] | None = None,
+    localizer: Localizer | None = None,
+):
+    if processors is None:
+        if localizer:
+            from stark.core.processors.corrections_processor import CorrectionsProcessor
+            processors = [CorrectionsProcessor(), SearchProcessor()]
+        else:
+            processors = [SearchProcessor()]
+
+    async with asyncer.create_task_group() as main_task_group:
+        context = CommandsContext(
+            task_group=main_task_group,
+            commands_manager=manager,
+            processors=processors,
+            localizer=localizer,
+        )
+
+        from stark.interfaces.microphone import Microphone
+
+        recognizers = speech_recognizer if isinstance(speech_recognizer, list) else [speech_recognizer]
+        use_relay = len(recognizers) > 1
+
+        if use_relay:
+            from stark.interfaces.recognizer_relay import SpeechRecognizerRelay
+            effective_recognizer = SpeechRecognizerRelay(recognizers)
+        else:
+            effective_recognizer = recognizers[0]
+
+        voice_assistant = VoiceAssistant(
+            speech_recognizer=effective_recognizer,
+            speech_synthesizer=speech_synthesizer,
+            commands_context=context,
+        )
+        effective_recognizer.delegate = voice_assistant
+        context.delegate = voice_assistant
+
+        health_check(context.pattern_parser, manager.commands)
+
+        if use_relay:
+            effective_recognizer.start_speech_recognizers(main_task_group)
+        else:
+            main_task_group.soonify(effective_recognizer.start_listening)()
+
+        microphone = Microphone(effective_recognizer.microphone_did_receive_sample)
+        main_task_group.soonify(microphone.start_listening)()
+
+        main_task_group.soonify(context.handle_responses)()
+
+        detector = BlockageDetector()
+        main_task_group.soonify(detector.monitor)()

{stark_engine-4.4.0 → stark_engine-4.4.1}/stark/core/command.py

@@ -43,10 +43,10 @@ CommandRunner = TypeVar("CommandRunner", bound=SyncCommandRunner | AsyncCommandR
 
 class Command(Generic[CommandRunner]):
     name: str
-    patterns: dict[str, Pattern]
+    patterns: dict[LanguageCode, Pattern]
     _runner: CommandRunner
 
-    def __init__(self, name: str, patterns: dict[str, Pattern], runner: CommandRunner):
+    def __init__(self, name: str, patterns: dict[LanguageCode, Pattern], runner: CommandRunner):
         assert patterns and all(isinstance(p, Pattern) for p in patterns.values())
         self.name = name
         self.patterns = patterns

{stark_engine-4.4.0 → stark_engine-4.4.1}/stark/core/commands_context.py

@@ -16,8 +16,9 @@ from stark.core.commands_context_processor import (
 from stark.core.parsing import PatternParser
 from stark.core.types.object import Object
 from stark.general.localisation import LocaleString, Localizer
+from stark.general.localisation.language_code import LanguageCode
 
-from ..general.dependencies import DependencyManager, default_dependency_manager
+from ..general.dependencies import Dependency, DependencyManager, default_dependency_manager
 from .command import (
     AsyncResponseHandler,
     Command,
@@ -104,9 +105,7 @@ class CommandsContext:
 
         for processor in self.processors:
             logger.debug(f"Processing context {processor=} with {string=} {recognized_entities=} {self.context_queue=}")
-            search_results, context_pops = await processor.process_string(
-                string, self, recognized_entities
-            )
+            search_results, context_pops = await processor.process_string(string, self, recognized_entities)
             if search_results:
                 # Pop contexts as directed by processor
                 for _ in range(context_pops):
@@ -123,7 +122,15 @@ class CommandsContext:
             parameters: dict[str, Object] = {}
             parameters.update(current_context.parameters)
             parameters.update(search_result.match_result.parameters)
+
+            substring = search_result.match_result.substring
+            lang = substring.language_code if isinstance(substring, LocaleString) else string.language_code
+            lang_dep = Dependency(None, LanguageCode, lang)
+            self.dependency_manager.dependencies.add(lang_dep)
             parameters.update(self.dependency_manager.resolve(search_result.command._runner))
+            # language is a command-specific temporary dependency
+            self.dependency_manager.dependencies.discard(lang_dep)
+
             self.run_command(search_result.command, parameters)
 
         return search_results or []

{stark_engine-4.4.0 → stark_engine-4.4.1}/stark/core/health_check.py

@@ -1,7 +1,7 @@
 import warnings
 
 from stark.core.command import Command
-from stark.core.parsing import PatternParser, _LOCALIZER_KEY_REGEX
+from stark.core.parsing import _LOCALIZER_KEY_REGEX, PatternParser
 
 
 def health_check(pattern_parser: PatternParser, commands: list[Command]) -> None:
@@ -32,17 +32,19 @@ def health_check(pattern_parser: PatternParser, commands: list[Command]) -> None
         for lang, cmd_pattern in command.patterns.items():
             if pattern_parser.localizer and pattern_parser._has_localizer_keys(cmd_pattern):
                 for key in _LOCALIZER_KEY_REGEX.findall(cmd_pattern._origin):
-                    pattern_parser.localizer.verify_key_exists(key)
+                    pattern_parser.localizer.verify_recognizable(key)
         for param in command.get_pattern("base").parameters.values():
             assert param.type_name in pattern_parser.parameter_types_by_name, (
-                f"Unknown parameter type '{param.type_name}' in pattern '{command.get_pattern("base")}' of command '{command.name}'. Known types: {sorted(pattern_parser.parameter_types_by_name.keys())}"
+                f"Unknown parameter type '{param.type_name}' in pattern '{command.get_pattern('base')}' of command '{command.name}'. Known types: {sorted(pattern_parser.parameter_types_by_name.keys())}"
             )
 
     # 5. Commands: all pattern params are expected by the runner func
     for command in commands:
         pattern_params = set(command.get_pattern("base").parameters.keys())
         runner_params = set(command._runner.__annotations__.keys())
-        assert pattern_params <= runner_params, f"Command '{command.name}' function missing parameters: {pattern_params - runner_params}"
+        assert pattern_params <= runner_params, (
+            f"Command '{command.name}' function missing parameters: {pattern_params - runner_params}"
+        )
 
     # 6. Pattern syntax validity (TODO)
     # for command in commands:
@@ -54,7 +56,9 @@ def health_check(pattern_parser: PatternParser, commands: list[Command]) -> None
             try:
                 pattern_parser._compile_pattern(p, language_code=lang)
             except Exception as e:
-                warnings.warn(f"Failed to compile pattern '{p}' of type '{reg_type.type.__name__}' (language '{lang}'): {e}")
+                warnings.warn(
+                    f"Failed to compile pattern '{p}' of type '{reg_type.type.__name__}' (language '{lang}'): {e}"
+                )
     for command in commands:
         for lang, p in command.patterns.items():
             try:

{stark_engine-4.4.0 → stark_engine-4.4.1}/stark/core/parsing.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import re
 from abc import ABC
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import AsyncGenerator
 
 from stark.core.patterns.pattern import Pattern
@@ -11,7 +11,16 @@ from stark.core.types import Object
 from stark.core.types.string import String
 from stark.core.types.word import Word
 from stark.general.cache import alru_cache
+from typing import NamedTuple
+
 from stark.general.localisation import LanguageCode, LocaleString, Localizer
+from stark.models.transcription_string import Correction
+from stark.tools.common.span import Span
+
+
+class CorrectionMatch(NamedTuple):
+    span: Span
+    correction: Correction
 
 type ObjectType = type[Object]
 
@@ -46,6 +55,8 @@ class MatchResult:
     start: int
     end: int
     parameters: dict[str, Object | None]  # TODO: use ParameterMatch?
+    corrections: list[CorrectionMatch] = field(default_factory=list)
+    corrected_string: str = ""
 
 
 @dataclass
@@ -127,7 +138,7 @@ class PatternParser:
                     f"Pattern '{pattern._origin}' of type '{object_type.__name__}' contains @key references but no Localizer is configured on PatternParser"
                 )
             for key in keys:
-                self.localizer.verify_key_exists(key)
+                self.localizer.verify_recognizable(key)
 
     @staticmethod
     def _has_localizer_keys(pattern: Pattern) -> bool:
@@ -203,6 +214,9 @@ class PatternParser:
         language_code = string.language_code
         recognized_entities = recognized_entities or []
         compiled = self._compile_pattern(pattern, language_code=language_code)
+
+        compiled, correction_groups = self._expand_corrections(compiled, string)
+
         logger.debug(f'Starting looking for "{pattern=}" "{compiled=}" in "{string}"')
 
         matches = []
@@ -237,6 +251,21 @@ class PatternParser:
             end = match.start() + new_match.end()
             command_str = string[start:end].strip()
 
+            # Backtrack corrections via named groups
+            match_corrections: list[CorrectionMatch] = []
+            for group_name, keyword in correction_groups.items():
+                if group_name not in match.groupdict():
+                    continue
+                captured = match.group(group_name)
+                if captured and captured != keyword:
+                    span = Span(match.start(group_name) - start, match.end(group_name) - start)
+                    match_corrections.append(CorrectionMatch(span, Correction(captured, keyword)))
+
+            # Reconstruct corrected string by applying replacements right-to-left
+            corrected = str(command_str)
+            for cm in sorted(match_corrections, key=lambda cm: cm.span.start, reverse=True):
+                corrected = corrected[: cm.span.start] + cm.correction.keyword + corrected[cm.span.end :]
+
             matches.append(
                 MatchResult(
                     substring=command_str,
@@ -246,6 +275,8 @@ class PatternParser:
                         name: (parameter.parsed_obj if parameter else None)
                         for name, parameter in all_parameters.items()
                     },
+                    corrections=match_corrections,
+                    corrected_string=corrected,
                 )
             )
             logger.debug(f"Match result: {matches[-1]}")
@@ -253,6 +284,53 @@ class PatternParser:
         matches = self._filter_overlapping_matches(matches)
         return sorted(matches, key=lambda m: len(m.substring), reverse=True)
 
+    def _expand_corrections(self, compiled: str, string: LocaleString) -> tuple[str, dict[str, str]]:
+        """Inject corrections into compiled regex as named groups.
+
+        Returns (expanded_regex, group_to_keyword_map) where group_to_keyword_map
+        maps named group names to their correction keywords for back-tracking.
+        """
+        from stark.models.transcription_string import TranscriptionString
+
+        empty_map: dict[str, str] = {}
+        if not isinstance(string, TranscriptionString):
+            return compiled, empty_map
+
+        all_corrections = list(string.corrections)
+        for track_corrections in string._corrections_by_track.values():
+            all_corrections.extend(track_corrections)
+
+        if not all_corrections:
+            return compiled, empty_map
+
+        grouped: dict[str, set[str]] = {}
+        for c in all_corrections:
+            grouped.setdefault(c.keyword, set()).add(c.variant)
+
+        # each occurrence of a keyword gets a unique named group because re module
+        # doesn't allow duplicate group names
+        # (the third-party `regex` module does via DUPNAMES, but some bechmarks show it's ~2x slower — not worth swapping for the hot path)
+        # TODO: review regex vs re for STARK again later
+        group_map: dict[str, str] = {}
+        for keyword, variants in grouped.items():
+            alternatives = "|".join([re.escape(v) for v in variants])
+            result_parts: list[str] = []
+            remaining = compiled
+            found = False
+            while keyword in remaining:
+                idx = remaining.index(keyword)
+                group_name = f"_corr{len(group_map)}"
+                group_map[group_name] = keyword
+                replacement = f"(?P<{group_name}>{re.escape(keyword)}|{alternatives})"
+                result_parts.append(remaining[:idx])
+                result_parts.append(replacement)
+                remaining = remaining[idx + len(keyword) :]
+                found = True
+            if found:
+                result_parts.append(remaining)
+                compiled = "".join(result_parts)
+        return compiled, group_map
+
     def _find_initial_matches(self, compiled: str, string: str) -> list[re.Match]:
         return sorted(re.finditer(compiled, string), key=lambda match: match.start())
 
@@ -278,6 +356,7 @@ class PatternParser:
 
             # re-run regex only in the current command_str
             compiled = self._compile_pattern(pattern, prefill=prefill, language_code=language_code)
+            compiled, _ = self._expand_corrections(compiled, string)
             new_matches = list(re.finditer(compiled, string))
 
             logger.debug(f'Re capturing parameters {string} prefill={prefill} compiled="{compiled}"')

{stark_engine-4.4.0 → stark_engine-4.4.1}/stark/core/processors/__init__.py

@@ -1,2 +1,3 @@
+from .corrections_processor import CorrectionsProcessor
 from .search_processor import SearchProcessor
 from .spacy_ner_processor import SpacyNERProcessor

stark_engine-4.4.1/stark/core/processors/corrections_processor.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, override
+
+from stark.core.commands_context_processor import CommandsContextProcessor
+from stark.core.commands_manager import SearchResult
+from stark.core.parsing import RecognizedEntity
+from stark.general.localisation import LocaleString
+
+from ..commands_context import CommandsContext
+
+if TYPE_CHECKING:
+    from stark.tools.dictionary.dictionary import Dictionary, LookupMode
+
+logger = logging.getLogger(__name__)
+
+
+class CorrectionsProcessor(CommandsContextProcessor):
+    """Pre-processor that generates phonetic corrections from dictionary lookups.
+
+    Accepts any Dictionary instance(s) — including one built from recognizable.strings
+    via build_recognizable_dictionary(). For each input word/phrase, runs dictionary
+    sentence search and appends matching corrections to TranscriptionString.corrections.
+    These are used by PatternParser._expand_corrections to widen compiled regexes.
+
+    For multilingual input with alternative tracks, runs dictionary search per track
+    and stores per-track corrections.
+
+    Place this processor before SearchProcessor in the pipeline.
+    """
+
+    def __init__(
+        self,
+        dictionaries: list[Dictionary] | None = None,
+        mode: LookupMode | None = None,
+    ):
+        self._dictionaries = dictionaries or []
+        if mode is None:
+            from stark.tools.dictionary.dictionary import LookupMode
+            mode = LookupMode.AUTO
+        self._mode = mode
+
+    @override
+    async def process_string(
+        self,
+        string: LocaleString,
+        context: CommandsContext,
+        recognized_entities: list[RecognizedEntity],
+    ) -> tuple[list[SearchResult], int]:
+        if not self._dictionaries:
+            return [], 0
+
+        from stark.models.transcription_string import TranscriptionString
+        from stark.models.transcription_string import Correction
+
+        if not isinstance(string, TranscriptionString):
+            return [], 0
+
+        self._find_corrections(str(string), string.language_code, string.corrections)
+
+        if string.alternative_texts:
+            for alt_lang, alt_text in string.alternative_texts.items():
+                if alt_lang == string.language_code:
+                    continue
+                corrections: list[Correction] = []
+                self._find_corrections(str(alt_text), alt_lang, corrections)
+                if corrections:
+                    string._corrections_by_track[alt_lang] = corrections
+
+        return [], 0
+
+    def _find_corrections(self, text: str, language_code: str, output: list):
+        from stark.models.transcription_string import Correction
+
+        for dictionary in self._dictionaries:
+            for result in dictionary.search_in_sentence(text, language_code, self._mode):
+                variant = text[result.span.slice]
+                keyword = result.item.name
+                if variant != keyword:
+                    output.append(Correction(variant=variant, keyword=keyword))

stark_engine-4.4.1/stark/core/processors/search_processor.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from typing import override
+
+from asyncer import SoonValue, create_task_group
+
+from stark.core.parsing import MatchResult, PatternParser, RecognizedEntity
+from stark.general.feature_flags import FeatureFlag, get_flag
+from stark.general.localisation import LocaleString
+from stark.general.localisation.language_code import LanguageCode
+
+from ..command import Command
+from ..commands_context import CommandsContext, CommandsContextLayer
+from ..commands_context_processor import CommandsContextProcessor
+from ..commands_manager import SearchResult
+
+
+class SearchProcessor(CommandsContextProcessor):
+    async def search(
+        self,
+        string: str | LocaleString,
+        pattern_parser: PatternParser,
+        commands: list[Command],
+        recognized_entities: list[RecognizedEntity],
+    ) -> list[SearchResult]:
+        string = string if isinstance(string, LocaleString) else LocaleString(string)
+        language_code = string.language_code
+
+        # collect all tracks to match (primary + alternatives)
+        tracks_to_match: list[tuple[str | LocaleString, str]] = [(string, language_code)]
+
+        if get_flag(FeatureFlag.ENABLE_MULTILANG_MATRIX):
+            from stark.models.transcription_string import TranscriptionString
+
+            if isinstance(string, TranscriptionString) and string.alternative_texts:
+                for alt_lang, alt_text in string.alternative_texts.items():
+                    if alt_lang == language_code:
+                        continue
+                    tracks_to_match.append((alt_text, alt_lang))
+
+        # match all tracks concurrently, tag results with source text and language
+        all_futures: list[tuple[str, str, SoonValue[list[SearchResult]]]] = []
+        async with create_task_group() as group:
+            for track_string, track_lang in tracks_to_match:
+                all_futures.append(
+                    (
+                        str(track_string),
+                        track_lang,
+                        group.soonify(self._match_commands)(
+                            track_string, track_lang, pattern_parser, commands, recognized_entities
+                        ),
+                    )
+                )
+
+        tagged: list[tuple[str, str, SearchResult]] = []
+        global_index = 0
+        for source_text, track_lang, future in all_futures:
+            for r in future.value:
+                r.index = global_index
+                global_index += 1
+                tagged.append((source_text, track_lang, r))
+
+        # sort by start position (translated to primary track for ordering)
+        # primary = str(string)
+        # tagged.sort(key=lambda t: string.translate_position(t[2].match_result.start, t[0], primary))
+        tagged.sort(key=lambda t: t[2].match_result.start)
+
+        # resolve overlaps — each result lives in its own source track's coordinates
+        # translate between tracks on demand for comparison and cutting
+        for prev_entry, current_entry in zip(tagged.copy(), tagged[1:]):
+            prev_src, prev_lang, prev = prev_entry
+            current_src, current_lang, current = current_entry
+
+            # check for overlaps
+            try:
+                current_start_in_prev = string.translate_position(current.match_result.start, current_src, prev_src)
+            except ValueError:
+                # can't translate between tracks — drop lower priority
+                loser_entry = current_entry if prev.index < current.index else prev_entry
+                if loser_entry in tagged:
+                    tagged.remove(loser_entry)
+                continue
+
+            if current_start_in_prev is None or prev.match_result.end <= current_start_in_prev:
+                # no overlap
+                continue
+
+            # overlap — try to cut each in its own source text
+            try:
+                prev_end_in_current = string.translate_position(prev.match_result.end, prev_src, current_src)
+            except ValueError:
+                loser_entry = current_entry if prev.index < current.index else prev_entry
+                if loser_entry in tagged:
+                    tagged.remove(loser_entry)
+                continue
+
+            prev_cut = (
+                await pattern_parser.match(
+                    prev.command.get_pattern(prev_lang),
+                    prev_src[prev.match_result.start : current_start_in_prev],
+                    recognized_entities,
+                )
+                if current_start_in_prev > prev.match_result.start
+                else []
+            )
+
+            current_cut = (
+                await pattern_parser.match(
+                    current.command.get_pattern(current_lang),
+                    current_src[prev_end_in_current : current.match_result.end],
+                    recognized_entities,
+                )
+                if prev_end_in_current is not None and prev_end_in_current < current.match_result.end
+                else []
+            )
+
+            # less index = more priority to save full match
+            is_prev_priority = prev.index < current.index
+            loser_cut = current_cut if is_prev_priority else prev_cut
+            winner_cut = prev_cut if is_prev_priority else current_cut
+            loser_entry = current_entry if is_prev_priority else prev_entry
+            winner_entry = prev_entry if is_prev_priority else current_entry
+
+            if new_matches := loser_cut:
+                loser_entry[2].match_result = new_matches[0]
+            elif new_matches := winner_cut:
+                winner_entry[2].match_result = new_matches[0]
+            else:
+                if loser_entry in tagged:
+                    tagged.remove(loser_entry)
+
+        return [r for _, _, r in tagged]
+
+    async def _match_commands(
+        self,
+        string: str | LocaleString,
+        language_code: LanguageCode,
+        pattern_parser: PatternParser,
+        commands: list[Command],
+        recognized_entities: list[RecognizedEntity],
+    ) -> list[SearchResult]:
+        results: list[SearchResult] = []
+        futures: list[tuple[Command, SoonValue[list[MatchResult]]]] = []
+
+        # run concurent commands match
+        async with create_task_group() as group:
+            for command in commands:
+                futures.append(
+                    (
+                        command,
+                        group.soonify(pattern_parser.match)(
+                            command.get_pattern(language_code), string, recognized_entities
+                        ),
+                    )
+                )
+
+        # read all finished matches
+        i = 0
+        for command, future in futures:
+            for match in future.value:  # empty for most of commands (not matched)
+                results.append(SearchResult(command=command, match_result=match, index=i))
+                i += 1
+
+        return results
+
+    # Implement CommandsContextProcessor
+
+    @override
+    async def process_context_layer(
+        self,
+        string: LocaleString,
+        context: CommandsContext,
+        context_layer: CommandsContextLayer,
+        recognized_entities: list[RecognizedEntity],
+    ) -> list[SearchResult]:
+        return await self.search(string, context.pattern_parser, context_layer.commands, recognized_entities)

stark_engine-4.4.1/stark/general/feature_flags.py

@@ -0,0 +1,17 @@
+import os
+from enum import StrEnum
+
+
+class FeatureFlag(StrEnum):
+    ENABLE_VOICE_CLI = "STARK_ENABLE_VOICE_CLI"
+    ENABLE_MULTILANG_MATRIX = "STARK_ENABLE_MULTILANG_MATRIX"
+
+
+_DEFAULTS: dict[FeatureFlag, str] = {
+    FeatureFlag.ENABLE_VOICE_CLI: "0",
+    FeatureFlag.ENABLE_MULTILANG_MATRIX: "1",
+}
+
+
+def get_flag(flag: FeatureFlag) -> bool:
+    return os.getenv(flag.value, _DEFAULTS.get(flag, "0")) == "1"

{stark_engine-4.4.0 → stark_engine-4.4.1}/stark/general/localisation/__init__.py

@@ -3,3 +3,4 @@ from .locale_string import LocaleString
 from .localizable_string import LocalizableString
 from .localizer import Localizer
 from .strings import StringsFile, String
+from stark.models.transcription_string import TranscriptionString, TranscriptionWord