dshellInterpreter 1.2.7.3__tar.gz → 1.2.7.4__tar.gz

{dshellinterpreter-1.2.7.3 → dshellinterpreter-1.2.7.4}/Dshell/DISCORD_COMMANDS/utils/utils_type_validation.py

@@ -117,7 +117,7 @@ def _validate_optional_permission(value, param_name: StrNode, command_name: StrN
     :param command_name: The command name for error messages (optional)
     :raises TypeError: If the value is not None and not a PermissionNode
     """
-    if (value is not None or not isinstance(value, _MissingSentinel)) and not isinstance(value, PermissionNode):
+    if value is not None and not isinstance(value, _MissingSentinel) and not isinstance(value, PermissionNode):
         raise TypeError(f"[{command_name}] -> {param_name} must be a PermissionNode or None, not {type(value).__name__}")
 
 

dshellinterpreter-1.2.7.4/Dshell/DshellPreProcess/dshell_preprocess.py

@@ -0,0 +1,73 @@
+from ..full_import import sub, MULTILINE, StrEnum, search, compile, DOTALL
+
+class PreProcessorInstructions(StrEnum):
+    DEFINE = "define"
+
+class PreProcessorData:
+    def __init__(self, instruction: str, symbol: str, value: str):
+        self.instruction = instruction
+        self.symbol = symbol
+        self.value = value
+
+def define(code: str, pre_processor_data: PreProcessorData) -> str:
+    """
+    Replace all symbol with the current value
+    :param code: the code before tokenization
+    :param pre_processor_data: all data for the ##define pre-processor
+    :return:
+    """
+    return sub(f"(?<![a-zA-Z]){pre_processor_data.symbol}(?![a-zA-Z])", pre_processor_data.value, code)
+
+pre_processor_pattern = compile(r"^\s*## *([a-zA-Z]+) +([a-zA-Z]+)(?: +(.*))?$", flags=MULTILINE|DOTALL)
+def preProcessor(code: list[str]) -> list[str]:
+    """
+    Execute preprocessor line
+    :param code: a list of code line before tokenization
+    :return: a new list of string after pre-processor and since pre-processor instructions
+    """
+    pre_processor_data: list[PreProcessorData] = list()
+    new_code: list[str] = []
+
+    for line in code:
+
+        line = removeCommentPreProcessor(line)
+
+        if pre_processor_match := search(pre_processor_pattern, line):
+
+            instruction: str = pre_processor_match.group(1).lower()
+            symbol: str = pre_processor_match.group(2)
+            value: str = pre_processor_match.group(3)
+            if instruction in PreProcessorInstructions:
+                pre_processor_data.append(PreProcessorData(instruction, symbol, value))
+
+        else:
+            # apply pre-processor instructions
+            line = applyPreProcessor(line, pre_processor_data)
+            new_code.append(line)
+
+    return new_code
+
+
+
+def applyPreProcessor(code: str, pre_processor_data: list[PreProcessorData]) -> str:
+    """
+    Apply preprocessor data to code
+    :param line: the current line before tokenization
+    :param pre_processor_data:
+    :return:
+    """
+    for pre_processor in pre_processor_data:
+        match pre_processor.instruction:
+            case "define":
+                code = define(code, pre_processor)
+
+    return code
+
+def removeCommentPreProcessor(code: str) -> str:
+    """
+    Remove all comment in the code
+    :param code:
+    :return:
+    """
+    # add \ before or in :: like \:: or :\: disable comment. Usable in string to add :: characters sequence.
+    return sub(r"(?<!\\):(?<!\\)?:.*", "", code, flags=MULTILINE)

{dshellinterpreter-1.2.7.3 → dshellinterpreter-1.2.7.4}/Dshell/DshellTokenizer/dshell_keywords.py

@@ -10,7 +10,7 @@ __all__ = [
 
 from ..DISCORD_COMMANDS import *
 
-from ..full_import import Callable
+from ..full_import import Callable, ChainMap
 from ..DshellInterpreteur.dshell_iterators import IntIterator
 
 
@@ -175,7 +175,16 @@ Tuple format: (function, precedence, number of operands min, number of operands
                 Optional[index to translate into value])
 """
 
-dshell_mathematical_operators: dict[str, tuple[Callable, int, int, int, Optional[int]]] = {
+Operator = tuple[
+    Callable,
+    int,
+    int,
+    int,
+    Optional[int],
+]
+
+
+dshell_mathematical_operators: dict[str, Operator] = {
 
     r"++": (lambda a: a + 1, 6, 1, 1, None),
     r"--": (lambda a: a - 1, 6, 1, 1, None),
@@ -198,14 +207,14 @@ dshell_mathematical_operators: dict[str, tuple[Callable, int, int, int, Optional
 
 }
 
-dshell_logical_word_operators: dict[str, tuple[Callable, int, int, int, Optional[int]]] = {
+dshell_logical_word_operators: dict[str, Operator] = {
     r"and": (lambda a, b: bool(a and b), 2, 2, 2, None),
     r"or": (lambda a, b: bool(a or b), 1, 2, 2, None),
     r"not": (lambda a: not a, 3, 1, 1, None),
     r"in": (lambda a, b: a in b, 4, 2, 2, None),
 }
 
-dshell_logical_operators: dict[str, tuple[Callable, int, int, int, Optional[int]]] = {
+dshell_logical_operators: dict[str, Operator] = {
 
     r"==": (lambda a, b: a == b, 4, 2, 2, None),
     r"<=": (lambda a, b: a <= b, 4, 2, 2, None),
@@ -223,10 +232,13 @@ dshell_logical_operators: dict[str, tuple[Callable, int, int, int, Optional[int]
     r">": (lambda a, b: a > b, 4, 2, 2, None),
     r"!": (lambda a: not a, 3, 1, 1, None),
     r"?": (lambda condition, first, second: first if condition else second, 1, 3, 3, 1),
-    #r".": (lambda target, attribute: target.call(attribute) if hasattr(target, attribute) and hasattr(target, 'call') else None, 9, 2, 2), # attribute access operator
-
 }
 
-dshell_operators: dict[str, tuple[Callable, int, int, int, Optional[int]]] = dshell_logical_operators.copy()
+dshell_operators: ChainMap[str, Operator] = ChainMap(
+    dshell_logical_operators,
+    dshell_logical_word_operators,
+    dshell_mathematical_operators)
+
+"""dshell_operators: dict[str, tuple[Callable, int, int, int, Optional[int]]] = dshell_logical_operators.copy()
 dshell_operators.update(dshell_logical_word_operators)
-dshell_operators.update(dshell_mathematical_operators)
+dshell_operators.update(dshell_mathematical_operators)"""

{dshellinterpreter-1.2.7.3 → dshellinterpreter-1.2.7.4}/Dshell/DshellTokenizer/dshell_tokenizer.py

@@ -12,10 +12,8 @@ from Dshell.full_import import (
                            ASCII,
                            DOTALL,
                            IGNORECASE,
-                           MULTILINE,
                            compile,
                            escape,
-                           findall,
                            finditer,
                            Match,
                            sub)
@@ -27,7 +25,7 @@ from .dshell_keywords import (dshell_keyword,
                               dshell_logical_operators,
                               dshell_logical_word_operators)
 
-from ..DshellPreProcess.dshell_preprocess import preProcessor, applyPreProcessor, PreProcessorData
+from ..DshellPreProcess.dshell_preprocess import preProcessor
 
 MASK_CHARACTER = '§'
 
@@ -37,10 +35,9 @@ def is_line_empty(line: str) -> bool:
     :param line: The line to check.
     :return: True if the line is empty, False otherwise.
     """
-    return all(c in (MASK_CHARACTER, ' ') for c in line)
+    return all(c in (MASK_CHARACTER, ' ', '\n') for c in line)
 
 table_regex: dict[DTT, Pattern] = {
-    DTT.COMMENT: compile(r"::(.*)", flags=MULTILINE),
     DTT.STR: compile(r'"((?:[^\\"]|\\.)*)"', flags=DOTALL),
     DTT.EVAL_L_EXPRESSION: compile(r"({)"),
     DTT.EVAL_R_EXPRESSION: compile(r"(})"),
@@ -54,7 +51,7 @@ table_regex: dict[DTT, Pattern] = {
     DTT.PARAMETER: compile(rf"--\s*([A-Za-z_]+)\s*", flags=ASCII),
     DTT.MENTION: compile(r'<(?:@[!&]?|#)([0-9]+)>'),
     DTT.KEYWORD: compile(rf"(?<!\w)(#?{'|'.join(dshell_keyword)})(?!\w)"),
-    DTT.DISCORD_KEYWORD: compile(rf"(?<!\w|-)(#?{'|'.join(dshell_discord_keyword)})(?!\w|-)", flags=IGNORECASE),
+    DTT.DISCORD_KEYWORD: compile(rf"(?<![\w\-])(#?{'|'.join(dshell_discord_keyword)})(?![\w\-])" , flags=IGNORECASE),
     DTT.COMMAND: compile(rf"\b({'|'.join(dshell_commands.keys())})\b", flags=IGNORECASE),
     DTT.FLOAT: compile(r"(\d+\.\d+)"),
     DTT.HEXA: compile(r"(0[Xx][0-9a-fA-F]+)"),
@@ -66,21 +63,19 @@ table_regex: dict[DTT, Pattern] = {
     DTT.BOOL: compile(r"(True|False)", flags=IGNORECASE),
     DTT.NONE: compile(r"(None)", flags=IGNORECASE),
     DTT.IDENT: compile(rf"([A-Za-z0-9_]+)"),
-    DTT.ANY_CHARACTER: compile(rf"([^{MASK_CHARACTER}\n]+)"),
 }
 
+backslash_pattern = compile(r"\\(.)", flags=DOTALL)
 
 class DshellTokenizer:
 
-    def __init__(self, code: str, match_any_character: bool = False):
+    def __init__(self, code: str):
         """
         Initialize the tokenizer.
         :param code: The code to tokenize
         :param match_any_character: Whether to match any character
         """
         self.code: str = code
-        self.match_any_character: bool = match_any_character
-        self.data_pre_processor: list[PreProcessorData] = []
 
     def start(self):
         """
@@ -88,6 +83,7 @@ class DshellTokenizer:
         Returns an array of tokens per line (normally separated by \\n)
         """
         split_commands = self.split(self.code)
+        split_commands = preProcessor(split_commands)
         return self.tokenizer(split_commands)
 
     def tokenizer(self, command_lines: list[str]) -> list[list[Token]]:
@@ -96,74 +92,47 @@ class DshellTokenizer:
         :param command_lines: The code separated into multiple lines by the split method
         """
         tokens: list[list[Token]] = []
+        ident_tokens = 0
 
-        line_number = 1
-        for line in command_lines:  # iterate each line of code
-            tokens_per_line: list[Token] = []
-            is_comment: bool = False
+        for line_number, line in enumerate(command_lines, start=1):
+            line: str = command_lines[line_number-1] # get the current line
 
+            # if the line is empty or already tokenized, pass the line.
             if is_line_empty(line):
-                line_number += 1
                 continue
 
-            if pre_processor_data := preProcessor(line):
-                self.data_pre_processor.append(pre_processor_data)
-                continue
-            else:
-                if self.data_pre_processor:
-                    for processor_data in self.data_pre_processor:
-                        line = applyPreProcessor(line, processor_data)
-
-            for token_type, pattern in table_regex.items():  # iterate the regex table to test all patterns on the line
-
-                if is_comment:
-                    is_comment = False
-                    break
-
-                if is_line_empty(line):
-                    break
+            tokens.append([])
+            tokens_per_line = tokens[ident_tokens]
 
-                if not self.match_any_character and token_type == DTT.ANY_CHARACTER:
-                    continue
-
-                for match in finditer(pattern, line):  # iterate the match results to get their positions
+            for token_type, regex in table_regex.items():
+                for match in finditer(regex, line):
                     match: Match
 
-                    if token_type == DTT.COMMENT:  # if we encounter a comment, stop tokenizing the line
-                        if len(match.group(0)) == len(line):
-                            is_comment = True
-                            break
-                        else:
-                            line = line[:match.start()]
-                            continue
-
-                    start_match = match.start()  # start position of the match
+                    start_match = match.start()
+                    end_match = match.end()
 
-                    token = Token(token_type, match.group(1), (line_number, start_match))  # record its token
+                    token = Token(token_type, match.group(1), (line_number, start_match))
                     tokens_per_line.append(token)
 
                     if token_type == DTT.STR:
-                        token.value = token.value.replace(r'\"', '"')
-
-                    len_match = len(match.group(0))  # length of the match found
-                    line = line[:start_match] + (MASK_CHARACTER * len_match) + line[
-                                                                                    match.end():]  # replace the match to avoid matching it a second time
-
-            tokens_per_line.sort(key=lambda
-                token: token.position[1])  # sort the position based on token match positions to have them in code order
-            if tokens_per_line:
+                        token.value = sub(backslash_pattern, lambda m: escape(m.group(1)), token.value)
 
-                tokens_per_line = self.parse_group(tokens_per_line,
-                                                  line_number)  # parse list in the current line to regroup them in a single token with all the elements of the list as value
+                    len_match = len(match.group(0))
+                    line = line[:start_match] + MASK_CHARACTER * len_match + line[end_match:]
 
-                tokens.append(tokens_per_line)
+            tokens_per_line.sort(key=lambda t: t.position[1])
 
-            line_number += 1  # increment the line number for the next line
+            if tokens_per_line:
+                tokens[ident_tokens] = self.parse_group(tokens_per_line, line_number)
 
+            ident_tokens += 1
         return tokens
 
     @staticmethod
-    def split(command: str, global_split='\n', keep_grouping_character=True, grouping_character='"') -> list[
+    def split(command: str,
+              global_split='\n',
+              keep_grouping_character=True,
+              grouping_character='"') -> list[
         str]:
         """
         Separate commands into a list while respecting strings between quotes.
@@ -175,31 +144,39 @@ class DshellTokenizer:
         :return: A list of split commands with restored strings.
         """
 
-        commands: str = command.strip()
-        temporary_replacement = '[REPLACE]'
-        pattern_find_regrouped_part = compile(rf'({grouping_character}(?:[^\\{grouping_character}]|\\.)*{grouping_character})', flags=DOTALL)
-        between_grouping_character = findall(pattern_find_regrouped_part, commands)  # find parts between quotes and save them
-
-        res = sub(pattern_find_regrouped_part,
-                  temporary_replacement,
-                  commands,
-                  )  # replace parts between quotes
-
-        res = res.split(global_split)  # split commands without quotes
-
-        # restore quotes to their place
-        result = []
-        for i in res:
-            while temporary_replacement in i:
-                i = i.replace(temporary_replacement,
-                              between_grouping_character[0][1: -1] if not keep_grouping_character else
-                              between_grouping_character[0], 1)
-                between_grouping_character.pop(0)
-            result.append(i)
+        result: list[str] = []
+        in_string: bool = False # True if we are in a string
+        backslash: bool = False
+        line: list[str] = [] # buffer for each line
+
+        for char in command:
+
+            if backslash:
+                line.append('\\')
+                line.append(char)
+                backslash = False
+
+            elif char == global_split and not in_string:
+                result.append(''.join(line))
+                line.clear()
+
+            elif char == grouping_character:
+                in_string = not in_string
+                if keep_grouping_character:
+                    line.append(char)
+
+            elif char == '\\':
+                backslash = True
+
+            else:
+                line.append(char)
+
+        result.append(''.join(line))
+
         return result
 
     @staticmethod
-    def parse_group(line: list[Token],
+    def parse_group(tokens_in_line: list[Token],
                    line_position: int):
         """
         Parse all list in the current token line and return the modified list line.
@@ -227,8 +204,8 @@ class DshellTokenizer:
                 last_tokens[-1].value.append(token)
 
         i = 0
-        while i < len(line):
-            token = line[i]
+        while i < len(tokens_in_line):
+            token = tokens_in_line[i]
 
             if token.type == DTT.LIST_L:
                 if last_tokens:

{dshellinterpreter-1.2.7.3 → dshellinterpreter-1.2.7.4}/Dshell/full_import.py

@@ -15,3 +15,4 @@ from contextvars import ContextVar
 from re import *
 from random import randint, choice, random
 from enum import Enum, auto, StrEnum
+from collections import ChainMap

{dshellinterpreter-1.2.7.3/dshellInterpreter.egg-info → dshellinterpreter-1.2.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dshellInterpreter
-Version: 1.2.7.3
+Version: 1.2.7.4
 Summary: A Discord bot interpreter for creating custom commands and automations.
 Home-page: https://github.com/BOXERRMD/Dshell_Interpreter
 Author: Chronos

{dshellinterpreter-1.2.7.3 → dshellinterpreter-1.2.7.4/dshellInterpreter.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dshellInterpreter
-Version: 1.2.7.3
+Version: 1.2.7.4
 Summary: A Discord bot interpreter for creating custom commands and automations.
 Home-page: https://github.com/BOXERRMD/Dshell_Interpreter
 Author: Chronos

{dshellinterpreter-1.2.7.3 → dshellinterpreter-1.2.7.4}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setup(
     name="dshellInterpreter",
-    version="1.2.7.3",
+    version="1.2.7.4",
     author="Chronos",
     author_email="vagabonwalybi@gmail.com",
     description="A Discord bot interpreter for creating custom commands and automations.",

dshellinterpreter-1.2.7.3/Dshell/DshellPreProcess/dshell_preprocess.py

@@ -1,41 +0,0 @@
-from ..full_import import search, sub, Optional
-
-class PreProcessorData:
-    def __init__(self, instruction: str, symbol: str, value: str):
-        self.instruction = instruction
-        self.symbol = symbol
-        self.value = value
-
-def define(line: str, pre_processor_data: PreProcessorData) -> str:
-    """
-    Replace all symbol with the current value
-    :param line: the current line before tokenization
-    :param pre_processor_data: all data for the ##define pre-processor
-    :return:
-    """
-    return sub(f"(?<![a-zA-Z]){pre_processor_data.symbol}(?![a-zA-Z])", pre_processor_data.value, line)
-
-
-def preProcessor(line: str) -> Optional[PreProcessorData]:
-    """
-    Execute preprocessor line
-    :param line: the current line before tokenization
-    :return:
-    """
-    pre_process_match = search(r"^\s*##([a-zA-Z]+)\s+([a-zA-Z]+)\s+(.*)$", line)
-    if pre_process_match:
-        return PreProcessorData(pre_process_match.group(1).lower(), pre_process_match.group(2), pre_process_match.group(3))
-    return None
-
-def applyPreProcessor(line: str, pre_processor_data: PreProcessorData):
-    """
-    Apply preprocessor data to code
-    :param line: the current line before tokenization
-    :param pre_processor_data:
-    :return:
-    """
-    match pre_processor_data.instruction:
-        case "define":
-            return define(line, pre_processor_data)
-
-    return None