semantic-kernel 1.8.3__zip

semantic_kernel-1.8.3/semantic_kernel/template_engine/blocks/val_block.py

@@ -0,0 +1,74 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from re import S, compile
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import model_validator
+
+from semantic_kernel.exceptions import ValBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+VAL_BLOCK_REGEX = r"^(?P<quote>[\"'])(?P<value>.*)(?P=quote)$"
+
+VAL_BLOCK_MATCHER = compile(VAL_BLOCK_REGEX, flags=S)
+
+
+class ValBlock(Block):
+    """Create a value block.
+
+    A value block is used to represent a value in a template.
+    It can be used to represent any characters.
+    It needs to start and end with the same quote character,
+    can be both single or double quotes, as long as they are not mixed.
+
+    Examples:
+        'value'
+        "value"
+        'value with "quotes"'
+        "value with 'quotes'"
+
+    Args:
+        content - str : The content of the value block.
+        value - str: The value of the block.
+        quote - str: The quote used to wrap the value.
+
+    Raises:
+        ValBlockSyntaxError: If the content does not match the value block syntax.
+
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.VALUE
+    value: str | None = ""
+    quote: str | None = "'"
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> Any:
+        """Parse the content and extract the value and quote.
+
+        The parsing is based on a regex that returns the value and quote.
+        if the 'value' is already present then the parsing is skipped.
+        """
+        if isinstance(fields, Block) or "value" in fields:
+            return fields
+        content = fields.get("content", "").strip()
+        matches = VAL_BLOCK_MATCHER.match(content)
+        if not matches:
+            raise ValBlockSyntaxError(content=content)
+        if value := matches.groupdict().get("value"):
+            fields["value"] = value
+        if quote := matches.groupdict().get("quote"):
+            fields["quote"] = quote
+        return fields
+
+    def render(self, *_: "Kernel | KernelArguments | None") -> str:
+        """Render the value block."""
+        return self.value or ""

semantic_kernel-1.8.3/semantic_kernel/template_engine/blocks/var_block.py

@@ -0,0 +1,84 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from re import compile
+from typing import TYPE_CHECKING, Any, ClassVar, Optional
+
+from pydantic import model_validator
+
+from semantic_kernel.exceptions import VarBlockRenderError, VarBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.symbols import Symbols
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+VAR_BLOCK_REGEX = r"^[${1}](?P<name>[0-9A-Za-z_]+)$"
+
+VAR_BLOCK_MATCHER = compile(VAR_BLOCK_REGEX)
+
+
+class VarBlock(Block):
+    """Create a variable block.
+
+    A variable block is used to add a variable to a template.
+    It gets rendered from KernelArguments, if the variable is not found
+    a warning is logged and an empty string is returned.
+    The variable must start with $ and be followed by a valid variable name.
+    A valid variable name is a string of letters, numbers and underscores.
+
+    Examples:
+        $var
+        $test_var
+
+    Args:
+        content - str : The content of the variable block, the name of the variable.
+        name - str: The name of the variable.
+
+    Raises:
+        VarBlockSyntaxError: If the content does not match the variable syntax.
+
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.VARIABLE
+    name: str = ""
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> Any:
+        """Parse the content and extract the name.
+
+        The parsing is based on a regex that returns the name.
+        if the 'name' is already present then the parsing is skipped.
+        """
+        if isinstance(fields, Block) or "name" in fields:
+            return fields
+        content = fields.get("content", "").strip()
+        matches = VAR_BLOCK_MATCHER.match(content)
+        if not matches:
+            raise VarBlockSyntaxError(content=content)
+        if name := matches.groupdict().get("name"):
+            fields["name"] = name
+        return fields
+
+    def render(self, _: "Kernel", arguments: Optional["KernelArguments"] = None) -> str:
+        """Render the variable block with the given arguments.
+
+        If the variable is not found in the arguments, return an empty string.
+        """
+        if arguments is None:
+            return ""
+        value = arguments.get(self.name, None)
+        if value is None:
+            logger.warning(f"Variable `{Symbols.VAR_PREFIX}: {self.name}` not found in the KernelArguments")
+            return ""
+        try:
+            return str(value)
+        except Exception as e:
+            raise VarBlockRenderError(
+                f"Block {self.name} failed to be parsed to a string, type is {type(value)}"
+            ) from e

semantic_kernel-1.8.3/semantic_kernel/template_engine/code_tokenizer.py

@@ -0,0 +1,157 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+
+from semantic_kernel.exceptions import CodeBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock
+from semantic_kernel.template_engine.blocks.named_arg_block import NamedArgBlock
+from semantic_kernel.template_engine.blocks.symbols import Symbols
+from semantic_kernel.template_engine.blocks.val_block import ValBlock
+from semantic_kernel.template_engine.blocks.var_block import VarBlock
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+# BNF parsed by CodeTokenizer:
+# [template]       ::= "" | [variable] " " [template]
+#                         | [value] " " [template]
+#                         | [function-call] " " [template]
+# [variable]       ::= "$" [valid-name]
+# [value]          ::= "'" [text] "'" | '"' [text] '"'
+# [function-call]  ::= [function-id] | [function-id] [parameter]
+# [parameter]      ::= [variable] | [value]
+class CodeTokenizer:
+    """Tokenize the code text into blocks."""
+
+    @staticmethod
+    def tokenize(text: str) -> list[Block]:
+        """Tokenize the code text into blocks."""
+        # Remove spaces, which are ignored anyway
+        text = text.strip() if text else ""
+        # Render None/empty to []
+        if not text:
+            return []
+        # 1 char only edge case, var and val blocks are invalid with one char, so it must be a function id block
+        if len(text) == 1:
+            return [FunctionIdBlock(content=text)]
+
+        # Track what type of token we're reading
+        current_token_type = None
+
+        # Track the content of the current token
+        current_token_content: list[str] = []
+
+        # Other state we need to track
+        text_value_delimiter = None
+        space_separator_found = False
+        skip_next_char = False
+        next_char = ""
+        blocks: list[Block] = []
+
+        for index, current_char in enumerate(text[:-1]):
+            next_char = text[index + 1]
+
+            if skip_next_char:
+                skip_next_char = False
+                continue
+
+            # First char is easy
+            if index == 0:
+                if current_char == Symbols.VAR_PREFIX:
+                    current_token_type = BlockTypes.VARIABLE
+                elif current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
+                    current_token_type = BlockTypes.VALUE
+                    text_value_delimiter = current_char
+                else:
+                    current_token_type = BlockTypes.FUNCTION_ID
+
+                current_token_content.append(current_char)
+                continue
+
+            # While reading values between quotes
+            if current_token_type in (BlockTypes.VALUE, BlockTypes.NAMED_ARG):
+                # If the current char is escaping the next special char we:
+                #  - skip the current char (escape char)
+                #  - add the next char (special char)
+                #  - jump to the one after (to handle "\\" properly)
+                if current_char == Symbols.ESCAPE_CHAR and next_char in (
+                    Symbols.DBL_QUOTE,
+                    Symbols.SGL_QUOTE,
+                    Symbols.ESCAPE_CHAR,
+                ):
+                    current_token_content.append(next_char)
+                    skip_next_char = True
+                    continue
+
+                current_token_content.append(current_char)
+
+                # When we reach the end of the value, we add the block
+                if current_char == text_value_delimiter:
+                    blocks.append(ValBlock(content="".join(current_token_content)))
+                    current_token_content.clear()
+                    current_token_type = None
+                    space_separator_found = False
+
+                continue
+
+            # If we're not between quotes, a space signals the end of the current token
+            # Note: there might be multiple consecutive spaces
+            if current_char in (
+                Symbols.SPACE,
+                Symbols.NEW_LINE,
+                Symbols.CARRIAGE_RETURN,
+                Symbols.TAB,
+            ):
+                if current_token_type == BlockTypes.VARIABLE:
+                    blocks.append(VarBlock(content="".join(current_token_content)))
+                    current_token_content.clear()
+                elif current_token_type == BlockTypes.FUNCTION_ID:
+                    if Symbols.NAMED_ARG_BLOCK_SEPARATOR.value in current_token_content:
+                        blocks.append(NamedArgBlock(content="".join(current_token_content)))
+                    else:
+                        blocks.append(FunctionIdBlock(content="".join(current_token_content)))
+                    current_token_content.clear()
+
+                space_separator_found = True
+                current_token_type = None
+
+                continue
+
+            # If we're not inside a quoted value, and we're not processing a space
+            current_token_content.append(current_char)
+
+            if current_token_type is None:
+                if not space_separator_found:
+                    raise CodeBlockSyntaxError("Tokens must be separated by one space least")
+
+                if current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
+                    # A quoted value starts here
+                    current_token_type = BlockTypes.VALUE
+                    text_value_delimiter = current_char
+                elif current_char == Symbols.VAR_PREFIX:
+                    # A variable starts here
+                    current_token_type = BlockTypes.VARIABLE
+                else:
+                    # A function id starts here
+                    current_token_type = BlockTypes.FUNCTION_ID
+
+        # end of main for loop
+
+        # Capture last token
+        current_token_content.append(next_char)
+
+        if current_token_type == BlockTypes.VALUE:
+            blocks.append(ValBlock(content="".join(current_token_content)))
+        elif current_token_type == BlockTypes.VARIABLE:
+            blocks.append(VarBlock(content="".join(current_token_content)))
+        elif current_token_type == BlockTypes.FUNCTION_ID:
+            if Symbols.NAMED_ARG_BLOCK_SEPARATOR.value in current_token_content:
+                blocks.append(NamedArgBlock(content="".join(current_token_content)))
+            else:
+                blocks.append(FunctionIdBlock(content="".join(current_token_content)))
+        else:
+            raise CodeBlockSyntaxError("Tokens must be separated by one space least")
+
+        return blocks

semantic_kernel-1.8.3/semantic_kernel/template_engine/protocols/code_renderer.py

@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from semantic_kernel import Kernel
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+
+
+@runtime_checkable
+class CodeRenderer(Protocol):
+    """Protocol for dynamic code blocks that need async IO to be rendered."""
+
+    async def render_code(self, kernel: "Kernel", arguments: "KernelArguments") -> str:
+        """Render the block using the given context.
+
+        :param context: kernel execution context
+        :return: Rendered content
+        """

semantic_kernel-1.8.3/semantic_kernel/template_engine/protocols/text_renderer.py

@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import TYPE_CHECKING, Optional, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from semantic_kernel import Kernel
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+
+
+@runtime_checkable
+class TextRenderer(Protocol):
+    """Protocol for static (text) blocks that don't need async rendering."""
+
+    def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"] = None) -> str:
+        """Render the block using only the given variables.
+
+        :param variables: Optional variables used to render the block
+        :return: Rendered content
+        """

semantic_kernel-1.8.3/semantic_kernel/template_engine/template_tokenizer.py

@@ -0,0 +1,161 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+
+from semantic_kernel.exceptions import BlockSyntaxError, CodeBlockTokenError, TemplateSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.code_block import CodeBlock
+from semantic_kernel.template_engine.blocks.symbols import Symbols
+from semantic_kernel.template_engine.blocks.text_block import TextBlock
+from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+# BNF parsed by TemplateTokenizer:
+# [template]       ::= "" | [block] | [block] [template]
+# [block]          ::= [sk-block] | [text-block]
+# [sk-block]       ::= "{{" [variable] "}}"
+#                      | "{{" [value] "}}"
+#                      | "{{" [function-call] "}}"
+# [text-block]     ::= [any-char] | [any-char] [text-block]
+# [any-char]       ::= any char
+class TemplateTokenizer:
+    """Tokenize the template text into blocks."""
+
+    @staticmethod
+    def tokenize(text: str) -> list[Block]:
+        """Tokenize the template text into blocks."""
+        code_tokenizer = CodeTokenizer()
+        # An empty block consists of 4 chars: "{{}}"
+        EMPTY_CODE_BLOCK_LENGTH = 4
+        # A block shorter than 5 chars is either empty or
+        # invalid, e.g. "{{ }}" and "{{$}}"
+        MIN_CODE_BLOCK_LENGTH = EMPTY_CODE_BLOCK_LENGTH + 1
+
+        text = text or ""
+
+        # Render None/empty to ""
+        if not text:
+            return [TextBlock.from_text("")]
+
+        # If the template is "empty" return it as a text block
+        if len(text) < MIN_CODE_BLOCK_LENGTH:
+            return [TextBlock.from_text(text)]
+
+        blocks: list[Block] = []
+        end_of_last_block = 0
+        block_start_pos = 0
+        block_start_found = False
+        inside_text_value = False
+        text_value_delimiter = None
+        skip_next_char = False
+
+        for current_char_pos, current_char in enumerate(text[:-1]):
+            next_char_pos = current_char_pos + 1
+            next_char = text[next_char_pos]
+
+            if skip_next_char:
+                skip_next_char = False
+                continue
+
+            # When "{{" is found outside a value
+            # Note: "{{ {{x}}" => ["{{ ", "{{x}}"]
+            if not inside_text_value and current_char == Symbols.BLOCK_STARTER and next_char == Symbols.BLOCK_STARTER:
+                # A block starts at the first "{"
+                block_start_pos = current_char_pos
+                block_start_found = True
+
+            if not block_start_found:
+                continue
+            # After having found "{{"
+            if inside_text_value:
+                # While inside a text value, when the end quote is found
+                # If the current char is escaping the next special char we skip
+                if current_char == Symbols.ESCAPE_CHAR and next_char in (
+                    Symbols.DBL_QUOTE,
+                    Symbols.SGL_QUOTE,
+                    Symbols.ESCAPE_CHAR,
+                ):
+                    skip_next_char = True
+                    continue
+
+                if current_char == text_value_delimiter:
+                    inside_text_value = False
+                continue
+
+            # A value starts here
+            if current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
+                inside_text_value = True
+                text_value_delimiter = current_char
+                continue
+            # If the block ends here
+            if current_char == Symbols.BLOCK_ENDER and next_char == Symbols.BLOCK_ENDER:
+                blocks.extend(
+                    TemplateTokenizer._extract_blocks(
+                        text, code_tokenizer, block_start_pos, end_of_last_block, next_char_pos
+                    )
+                )
+                end_of_last_block = next_char_pos + 1
+                block_start_found = False
+
+        # If there is something left after the last block, capture it as a TextBlock
+        if end_of_last_block < len(text):
+            blocks.append(TextBlock.from_text(text, end_of_last_block, len(text)))
+
+        return blocks
+
+    @staticmethod
+    def _extract_blocks(
+        text: str, code_tokenizer: CodeTokenizer, block_start_pos: int, end_of_last_block: int, next_char_pos: int
+    ) -> list[Block]:
+        """Extract the blocks from the found code.
+
+        If there is text before the current block, create a TextBlock from that.
+
+        If the block is empty, return a TextBlock with the delimiters.
+
+        If the block is not empty, tokenize it and return the result.
+        If there is only a variable or value in the code block,
+        return just that, instead of the CodeBlock.
+        """
+        new_blocks: list[Block] = []
+        if block_start_pos > end_of_last_block:
+            new_blocks.append(
+                TextBlock.from_text(
+                    text,
+                    end_of_last_block,
+                    block_start_pos,
+                )
+            )
+
+        content_with_delimiters = text[block_start_pos : next_char_pos + 1]
+        content_without_delimiters = content_with_delimiters[2:-2].strip()
+
+        if len(content_without_delimiters) == 0:
+            # If what is left is empty (only {{}}), consider the raw block
+            # a TextBlock
+            new_blocks.append(TextBlock.from_text(content_with_delimiters))
+            return new_blocks
+
+        try:
+            code_blocks = code_tokenizer.tokenize(content_without_delimiters)
+        except BlockSyntaxError as e:
+            msg = f"Failed to tokenize code block: {content_without_delimiters}. {e}"
+            logger.warning(msg)
+            raise TemplateSyntaxError(msg) from e
+
+        if code_blocks[0].type in (
+            BlockTypes.VALUE,
+            BlockTypes.VARIABLE,
+        ):
+            new_blocks.append(code_blocks[0])
+            return new_blocks
+        try:
+            new_blocks.append(CodeBlock(content=content_without_delimiters, tokens=code_blocks))
+            return new_blocks
+        except CodeBlockTokenError as e:
+            msg = f"Failed to tokenize code block: {content_without_delimiters}. {e}"
+            logger.warning(msg)
+            raise TemplateSyntaxError(msg) from e

semantic_kernel-1.8.3/semantic_kernel/text/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from semantic_kernel.text.function_extension import aggregate_chunked_results
+from semantic_kernel.text.text_chunker import (
+    split_markdown_lines,
+    split_markdown_paragraph,
+    split_plaintext_lines,
+    split_plaintext_paragraph,
+)
+
+__all__ = [
+    "aggregate_chunked_results",
+    "split_markdown_lines",
+    "split_markdown_paragraph",
+    "split_plaintext_lines",
+    "split_plaintext_paragraph",
+]

semantic_kernel-1.8.3/semantic_kernel/text/function_extension.py

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+
+from semantic_kernel.functions.kernel_arguments import KernelArguments
+from semantic_kernel.functions.kernel_function import KernelFunction
+from semantic_kernel.kernel import Kernel
+
+
+async def aggregate_chunked_results(
+    func: KernelFunction, chunked_results: list[str], kernel: Kernel, arguments: KernelArguments
+) -> str:
+    """Aggregate the results from the chunked results."""
+    results = []
+    for chunk in chunked_results:
+        arguments["input"] = chunk
+        result = await func.invoke(kernel, arguments)
+
+        results.append(str(result))
+
+    return "\n".join(results)