semantic-kernel 0.3.0.dev0__py3-none-any.whl → 0.3.2.dev0__py3-none-any.whl

semantic_kernel/memory/memory_record.py

@@ -13,6 +13,7 @@ class MemoryRecord:
     _id: str
     _description: Optional[str]
     _text: Optional[str]
+    _additional_metadata: Optional[str]
     _embedding: ndarray
 
     def __init__(
@@ -22,7 +23,8 @@ class MemoryRecord:
         id: str,
         description: Optional[str],
         text: Optional[str],
-        embedding: ndarray,
+        additional_metadata: Optional[str],
+        embedding: Optional[ndarray],
         key: Optional[str] = None,
         timestamp: Optional[str] = None,
     ) -> None:
@@ -34,6 +36,7 @@ class MemoryRecord:
             id {str} -- A unique for the record.
             description {Optional[str]} -- The description of the record.
             text {Optional[str]} -- The text of the record.
+            additional_metadata {Optional[str]} -- Custom metadata for the record.
             embedding {ndarray} -- The embedding of the record.
 
         Returns:
@@ -46,6 +49,7 @@ class MemoryRecord:
         self._id = id
         self._description = description
         self._text = text
+        self._additional_metadata = additional_metadata
         self._embedding = embedding
 
     @property
@@ -57,6 +61,7 @@ class MemoryRecord:
         external_id: str,
         source_name: str,
         description: Optional[str],
+        additional_metadata: Optional[str],
         embedding: ndarray,
     ) -> "MemoryRecord":
         """Create a reference record.
@@ -65,6 +70,7 @@ class MemoryRecord:
             external_id {str} -- The external id of the record.
             source_name {str} -- The name of the external source.
             description {Optional[str]} -- The description of the record.
+            additional_metadata {Optional[str]} -- Custom metadata for the record.
             embedding {ndarray} -- The embedding of the record.
 
         Returns:
@@ -76,12 +82,17 @@ class MemoryRecord:
             id=external_id,
             description=description,
             text=None,
+            additional_metadata=additional_metadata,
             embedding=embedding,
         )
 
     @staticmethod
     def local_record(
-        id: str, text: str, description: Optional[str], embedding: ndarray
+        id: str,
+        text: str,
+        description: Optional[str],
+        additional_metadata: Optional[str],
+        embedding: ndarray,
     ) -> "MemoryRecord":
         """Create a local record.
 
@@ -89,6 +100,7 @@ class MemoryRecord:
             id {str} -- A unique for the record.
             text {str} -- The text of the record.
             description {Optional[str]} -- The description of the record.
+            additional_metadata {Optional[str]} -- Custom metadata for the record.
             embedding {ndarray} -- The embedding of the record.
 
         Returns:
@@ -100,5 +112,6 @@ class MemoryRecord:
             id=id,
             description=description,
             text=text,
+            additional_metadata=additional_metadata,
             embedding=embedding,
         )

semantic_kernel/memory/null_memory.py

@@ -8,7 +8,12 @@ from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryB
 
 class NullMemory(SemanticTextMemoryBase):
     async def save_information_async(
-        self, collection: str, text: str, id: str, description: Optional[str] = None
+        self,
+        collection: str,
+        text: str,
+        id: str,
+        description: Optional[str] = None,
+        additional_metadata: Optional[str] = None,
     ) -> None:
         return None
 
@@ -19,6 +24,7 @@ class NullMemory(SemanticTextMemoryBase):
         external_id: str,
         external_source_name: str,
         description: Optional[str] = None,
+        additional_metadata: Optional[str] = None,
     ) -> None:
         return None
 

semantic_kernel/memory/semantic_text_memory.py

@@ -37,6 +37,7 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         text: str,
         id: str,
         description: Optional[str] = None,
+        additional_metadata: Optional[str] = None,
     ) -> None:
         """Save information to the memory (calls the memory store's upsert method).
 
@@ -55,9 +56,15 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         ):
             await self._storage.create_collection_async(collection_name=collection)
 
-        embedding = await self._embeddings_generator.generate_embeddings_async([text])
+        embedding = (
+            await self._embeddings_generator.generate_embeddings_async([text])
+        )[0]
         data = MemoryRecord.local_record(
-            id=id, text=text, description=description, embedding=embedding
+            id=id,
+            text=text,
+            description=description,
+            additional_metadata=additional_metadata,
+            embedding=embedding,
         )
 
         await self._storage.upsert_async(collection_name=collection, record=data)
@@ -69,6 +76,7 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         external_id: str,
         external_source_name: str,
         description: Optional[str] = None,
+        additional_metadata: Optional[str] = None,
     ) -> None:
         """Save a reference to the memory (calls the memory store's upsert method).
 
@@ -88,11 +96,14 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         ):
             await self._storage.create_collection_async(collection_name=collection)
 
-        embedding = await self._embeddings_generator.generate_embeddings_async([text])
+        embedding = (
+            await self._embeddings_generator.generate_embeddings_async([text])
+        )[0]
         data = MemoryRecord.reference_record(
             external_id=external_id,
             source_name=external_source_name,
             description=description,
+            additional_metadata=additional_metadata,
             embedding=embedding,
         )
 
@@ -135,9 +146,9 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         Returns:
             List[MemoryQueryResult] -- The list of MemoryQueryResult found.
         """
-        query_embedding = await self._embeddings_generator.generate_embeddings_async(
-            [query]
-        )
+        query_embedding = (
+            await self._embeddings_generator.generate_embeddings_async([query])
+        )[0]
         results = await self._storage.get_nearest_matches_async(
             collection_name=collection,
             embedding=query_embedding,

semantic_kernel/memory/semantic_text_memory_base.py

@@ -14,6 +14,7 @@ class SemanticTextMemoryBase(ABC):
         text: str,
         id: str,
         description: Optional[str] = None,
+        additional_metadata: Optional[str] = None,
         # TODO: ctoken?
     ) -> None:
         pass
@@ -26,6 +27,7 @@ class SemanticTextMemoryBase(ABC):
         external_id: str,
         external_source_name: str,
         description: Optional[str] = None,
+        additional_metadata: Optional[str] = None,
     ) -> None:
         pass
 

semantic_kernel/orchestration/sk_function.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import asyncio
+import platform
+import sys
 import threading
 from enum import Enum
 from logging import Logger
@@ -36,6 +38,9 @@ from semantic_kernel.skill_definition.read_only_skill_collection_base import (
 )
 from semantic_kernel.utils.null_logger import NullLogger
 
+if platform.system() == "Windows" and sys.version_info >= (3, 8, 0):
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
 
 class SKFunction(SKFunctionBase):
     """

semantic_kernel/planning/basic_planner.py

@@ -3,6 +3,8 @@
 """A basic JSON-based planner for the Python Semantic Kernel"""
 import json
 
+import regex
+
 from semantic_kernel.kernel import Kernel
 from semantic_kernel.orchestration.context_variables import ContextVariables
 from semantic_kernel.planning.plan import Plan
@@ -187,7 +189,13 @@ class BasicPlanner:
         Given a plan, execute each of the functions within the plan
         from start to finish and output the result.
         """
-        generated_plan = json.loads(plan.generated_plan.result)
+
+        # Filter out good JSON from the result in case additional text is present
+        json_regex = r"\{(?:[^{}]|(?R))*\}"
+        generated_plan_string = regex.search(
+            json_regex, plan.generated_plan.result
+        ).group()
+        generated_plan = json.loads(generated_plan_string)
 
         context = ContextVariables()
         context["input"] = generated_plan["input"]

semantic_kernel/semantic_functions/prompt_template_config.py

@@ -13,6 +13,7 @@ class PromptTemplateConfig:
         presence_penalty: float = 0.0
         frequency_penalty: float = 0.0
         max_tokens: int = 256
+        number_of_responses: int = 1
         stop_sequences: List[str] = field(default_factory=list)
 
     @dataclass
@@ -51,6 +52,9 @@ class PromptTemplateConfig:
         config.completion.presence_penalty = completion_dict.get("presence_penalty")
         config.completion.frequency_penalty = completion_dict.get("frequency_penalty")
         config.completion.max_tokens = completion_dict.get("max_tokens")
+        config.completion.number_of_responses = completion_dict.get(
+            "number_of_responses"
+        )
         config.completion.stop_sequences = completion_dict.get("stop_sequences", [])
         config.default_services = data.get("default_services", [])
 
@@ -102,6 +106,7 @@ class PromptTemplateConfig:
         presence_penalty: float = 0.0,
         frequency_penalty: float = 0.0,
         max_tokens: int = 256,
+        number_of_responses: int = 1,
         stop_sequences: List[str] = [],
     ) -> "PromptTemplateConfig":
         config = PromptTemplateConfig()
@@ -110,5 +115,6 @@ class PromptTemplateConfig:
         config.completion.presence_penalty = presence_penalty
         config.completion.frequency_penalty = frequency_penalty
         config.completion.max_tokens = max_tokens
+        config.completion.number_of_responses = number_of_responses
         config.completion.stop_sequences = stop_sequences
         return config

semantic_kernel/skill_definition/functions_view.py

@@ -43,4 +43,17 @@ class FunctionsView:
         return as_sf
 
     def is_native(self, skill_name: str, function_name: str) -> bool:
-        return not self.is_semantic(skill_name, function_name)
+        as_sf = self._semantic_functions.get(skill_name, [])
+        as_sf = any(f.name == function_name for f in as_sf)
+
+        as_nf = self._native_functions.get(skill_name, [])
+        as_nf = any(f.name == function_name for f in as_nf)
+
+        if as_sf and as_nf:
+            raise KernelException(
+                KernelException.ErrorCodes.AmbiguousImplementation,
+                f"There are 2 functions with the same name: {function_name}."
+                f"One is native and the other semantic.",
+            )
+
+        return as_nf

semantic_kernel/text/text_chunker.py

@@ -5,7 +5,8 @@ For plain text, split looking at new lines first, then periods, and so on.
 For markdown, split looking at punctuation first, and so on.
 """
 import os
-from typing import List
+import re
+from typing import Callable, List, Tuple
 
 NEWLINE = os.linesep
 
@@ -36,46 +37,95 @@ MD_SPLIT_OPTIONS = [
 ]
 
 
-def split_plaintext_lines(text: str, max_token_per_line: int) -> List[str]:
+def _token_counter(text: str) -> int:
+    """
+    Count the number of tokens in a string.
+
+    TODO: chunking methods should be configurable to allow for different
+          tokenization strategies depending on the model to be called.
+          For now, we use an extremely rough estimate.
+    """
+    return len(text) // 4
+
+
+def split_plaintext_lines(
+    text: str, max_token_per_line: int, token_counter: Callable = _token_counter
+) -> List[str]:
     """
     Split plain text into lines.
     it will split on new lines first, and then on punctuation.
     """
-    return _split_text_lines(text, max_token_per_line, True)
+    return _split_text_lines(
+        text=text,
+        max_token_per_line=max_token_per_line,
+        trim=True,
+        token_counter=token_counter,
+    )
 
 
-def split_markdown_lines(text: str, max_token_per_line: int) -> List[str]:
+def split_markdown_lines(
+    text: str, max_token_per_line: int, token_counter: Callable = _token_counter
+) -> List[str]:
     """
     Split markdown into lines.
     It will split on punctuation first, and then on space and new lines.
     """
-    return _split_markdown_lines(text, max_token_per_line, True)
+    return _split_markdown_lines(
+        text=text,
+        max_token_per_line=max_token_per_line,
+        trim=True,
+        token_counter=token_counter,
+    )
 
 
-def split_plaintext_paragraph(text: List[str], max_tokens: int) -> List[str]:
+def split_plaintext_paragraph(
+    text: List[str], max_tokens: int, token_counter: Callable = _token_counter
+) -> List[str]:
     """
     Split plain text into paragraphs.
     """
 
     split_lines = []
     for line in text:
-        split_lines.extend(_split_text_lines(line, max_tokens, True))
+        split_lines.extend(
+            _split_text_lines(
+                text=line,
+                max_token_per_line=max_tokens,
+                trim=True,
+                token_counter=token_counter,
+            )
+        )
 
-    return _split_text_paragraph(split_lines, max_tokens)
+    return _split_text_paragraph(
+        text=split_lines, max_tokens=max_tokens, token_counter=token_counter
+    )
 
 
-def split_markdown_paragraph(text: List[str], max_tokens: int) -> List[str]:
+def split_markdown_paragraph(
+    text: List[str], max_tokens: int, token_counter: Callable = _token_counter
+) -> List[str]:
     """
     Split markdown into paragraphs.
     """
     split_lines = []
     for line in text:
-        split_lines.extend(_split_markdown_lines(line, max_tokens, False))
+        split_lines.extend(
+            _split_markdown_lines(
+                text=line,
+                max_token_per_line=max_tokens,
+                trim=False,
+                token_counter=token_counter,
+            )
+        )
 
-    return _split_text_paragraph(split_lines, max_tokens)
+    return _split_text_paragraph(
+        text=split_lines, max_tokens=max_tokens, token_counter=token_counter
+    )
 
 
-def _split_text_paragraph(text: List[str], max_tokens: int) -> List[str]:
+def _split_text_paragraph(
+    text: List[str], max_tokens: int, token_counter: Callable
+) -> List[str]:
     """
     Split text into paragraphs.
     """
@@ -86,8 +136,8 @@ def _split_text_paragraph(text: List[str], max_tokens: int) -> List[str]:
     current_paragraph = []
 
     for line in text:
-        num_tokens_line = _token_count(line)
-        num_tokens_paragraph = _token_count("".join(current_paragraph))
+        num_tokens_line = token_counter(line)
+        num_tokens_paragraph = token_counter("".join(current_paragraph))
 
         if (
             num_tokens_paragraph + num_tokens_line + 1 >= max_tokens
@@ -109,7 +159,7 @@ def _split_text_paragraph(text: List[str], max_tokens: int) -> List[str]:
         last_para = paragraphs[-1]
         sec_last_para = paragraphs[-2]
 
-        if _token_count(last_para) < max_tokens / 4:
+        if token_counter(last_para) < max_tokens / 4:
             last_para_tokens = last_para.split(" ")
             sec_last_para_tokens = sec_last_para.split(" ")
             last_para_token_count = len(last_para_tokens)
@@ -125,27 +175,44 @@ def _split_text_paragraph(text: List[str], max_tokens: int) -> List[str]:
     return paragraphs
 
 
-def _split_markdown_lines(text: str, max_token_per_line: int, trim: bool) -> List[str]:
+def _split_markdown_lines(
+    text: str, max_token_per_line: int, trim: bool, token_counter: Callable
+) -> List[str]:
     """
     Split markdown into lines.
     """
 
-    lines = _split_str_lines(text, max_token_per_line, MD_SPLIT_OPTIONS, trim)
-    return lines
+    return _split_str_lines(
+        text=text,
+        max_tokens=max_token_per_line,
+        separators=MD_SPLIT_OPTIONS,
+        trim=trim,
+        token_counter=token_counter,
+    )
 
 
-def _split_text_lines(text: str, max_token_per_line: int, trim: bool) -> List[str]:
+def _split_text_lines(
+    text: str, max_token_per_line: int, trim: bool, token_counter: Callable
+) -> List[str]:
     """
     Split text into lines.
     """
 
-    lines = _split_str_lines(text, max_token_per_line, TEXT_SPLIT_OPTIONS, trim)
-
-    return lines
+    return _split_str_lines(
+        text=text,
+        max_tokens=max_token_per_line,
+        separators=TEXT_SPLIT_OPTIONS,
+        trim=trim,
+        token_counter=token_counter,
+    )
 
 
 def _split_str_lines(
-    text: str, max_tokens: int, separators: List[List[str]], trim: bool
+    text: str,
+    max_tokens: int,
+    separators: List[List[str]],
+    trim: bool,
+    token_counter: Callable,
 ) -> List[str]:
     if not text:
         return []
@@ -155,65 +222,82 @@ def _split_str_lines(
     was_split = False
     for split_option in separators:
         if not lines:
-            lines, was_split = _split_str(text, max_tokens, split_option, trim)
+            lines, was_split = _split_str(
+                text=text,
+                max_tokens=max_tokens,
+                separators=split_option,
+                trim=trim,
+                token_counter=token_counter,
+            )
         else:
-            lines, was_split = _split_list(lines, max_tokens, split_option, trim)
-        if not was_split:
+            lines, was_split = _split_list(
+                text=lines,
+                max_tokens=max_tokens,
+                separators=split_option,
+                trim=trim,
+                token_counter=token_counter,
+            )
+        if was_split:
             break
 
     return lines
 
 
 def _split_str(
-    text: str, max_tokens: int, separators: List[str], trim: bool
-) -> List[str]:
+    text: str,
+    max_tokens: int,
+    separators: List[str],
+    trim: bool,
+    token_counter: Callable,
+) -> Tuple[List[str], bool]:
     """
     Split text into lines.
     """
+    input_was_split = False
     if not text:
-        return []
+        return [], input_was_split
 
-    input_was_split = False
-    text = text.strip() if trim else text
+    if trim:
+        text = text.strip()
 
     text_as_is = [text]
 
-    if _token_count(text) <= max_tokens:
+    if token_counter(text) <= max_tokens:
         return text_as_is, input_was_split
 
-    input_was_split = True
-
-    half = int(len(text) / 2)
+    half = len(text) // 2
 
     cutpoint = -1
 
     if not separators:
         cutpoint = half
-
     elif set(separators) & set(text) and len(text) > 2:
-        for index, text_char in enumerate(text):
-            if text_char not in separators:
-                continue
-
-            if abs(half - index) < abs(half - cutpoint):
-                cutpoint = index + 1
-
+        regex_separators = re.compile("|".join(re.escape(s) for s in separators))
+        min_dist = half
+        for match in re.finditer(regex_separators, text):
+            end = match.end()
+            dist = abs(half - end)
+            if dist < min_dist:
+                min_dist = dist
+                cutpoint = end
+            elif end > half:
+                # distance is increasing, so we can stop searching
+                break
     else:
         return text_as_is, input_was_split
 
     if 0 < cutpoint < len(text):
         lines = []
-        first_split, has_split1 = _split_str(
-            text[:cutpoint], max_tokens, separators, trim
-        )
-        second_split, has_split2 = _split_str(
-            text[cutpoint:], max_tokens, separators, trim
-        )
-
-        lines.extend(first_split)
-        lines.extend(second_split)
-
-        input_was_split = has_split1 or has_split2
+        for text_part in [text[:cutpoint], text[cutpoint:]]:
+            split, has_split = _split_str(
+                text=text_part,
+                max_tokens=max_tokens,
+                separators=separators,
+                trim=trim,
+                token_counter=token_counter,
+            )
+            lines.extend(split)
+            input_was_split = input_was_split or has_split
     else:
         return text_as_is, input_was_split
 
@@ -221,30 +305,29 @@ def _split_str(
 
 
 def _split_list(
-    text: List[str], max_tokens: int, separators: List[str], trim: bool
-) -> List[str]:
+    text: List[str],
+    max_tokens: int,
+    separators: List[str],
+    trim: bool,
+    token_counter: Callable,
+) -> Tuple[List[str], bool]:
     """
     Split list of string into lines.
     """
     if not text:
-        return []
+        return [], False
 
     lines = []
     input_was_split = False
     for line in text:
-        split_str, was_split = _split_str(line, max_tokens, separators, trim)
+        split_str, was_split = _split_str(
+            text=line,
+            max_tokens=max_tokens,
+            separators=separators,
+            trim=trim,
+            token_counter=token_counter,
+        )
         lines.extend(split_str)
         input_was_split = input_was_split or was_split
 
     return lines, input_was_split
-
-
-def _token_count(text: str) -> int:
-    """
-    Count the number of tokens in a string.
-
-    TODO: chunking methods should be configurable to allow for different
-          tokenization strategies depending on the model to be called.
-          For now, we use an extremely rough estimate.
-    """
-    return int(len(text) / 4)