langroid 0.32.0__py3-none-any.whl → 0.32.2__py3-none-any.whl

langroid/embedding_models/clustering.py

@@ -1,189 +0,0 @@
-import logging
-from collections import Counter
-from typing import Callable, List, Tuple
-
-import faiss
-import numpy as np
-from sklearn.cluster import DBSCAN
-from sklearn.neighbors import NearestNeighbors
-from sklearn.preprocessing import StandardScaler
-
-from langroid.mytypes import Document
-
-logging.getLogger("faiss").setLevel(logging.ERROR)
-logging.getLogger("faiss-cpu").setLevel(logging.ERROR)
-
-
-def find_optimal_clusters(X: np.ndarray, max_clusters: int, threshold=0.1) -> int:
-    """
-    Find the optimal number of clusters for FAISS K-means using the Elbow Method.
-
-    Args:
-        X (np.ndarray): A 2D NumPy array of data points.
-        max_clusters (int): The maximum number of clusters to try.
-        threshold (float): Threshold for the rate of change in inertia values.
-        Defaults to 0.1.
-
-    Returns:
-        int: The optimal number of clusters.
-    """
-    inertias = []
-    max_clusters = min(max_clusters, X.shape[0])
-    cluster_range = range(1, max_clusters + 1)
-
-    for nclusters in cluster_range:
-        kmeans = faiss.Kmeans(X.shape[1], nclusters, niter=20, verbose=False)
-        kmeans.train(X)
-        centroids = kmeans.centroids
-        distances = np.sum(np.square(X[:, None] - centroids), axis=-1)
-        inertia = np.sum(np.min(distances, axis=-1))
-        inertias.append(inertia)
-
-    # Calculate the rate of change in inertia values
-    rate_of_change = [
-        abs((inertias[i + 1] - inertias[i]) / inertias[i])
-        for i in range(len(inertias) - 1)
-    ]
-
-    # Find the optimal number of clusters based on the rate of change threshold
-    optimal_clusters = 1
-    for i, roc in enumerate(rate_of_change):
-        if roc < threshold:
-            optimal_clusters = i + 1
-            break
-
-    return optimal_clusters
-
-
-def densest_clusters(
-    embeddings: List[np.ndarray], k: int = 5
-) -> List[Tuple[np.ndarray, int]]:
-    """
-    Find the top k densest clusters in the given list of embeddings using FAISS K-means.
-    See here:
-    'https://github.com/facebookresearch/faiss/wiki/Faiss-building-blocks%3A-clustering%
-     2C-PCA%2C-quantization'
-
-    Args:
-        embeddings (List[np.ndarray]): A list of embedding vectors.
-        k (int, optional): The number of densest clusters to find. Defaults to 5.
-
-    Returns:
-        List[Tuple[np.ndarray, int]]: A list of representative vectors and their indices
-                                      from the k densest clusters.
-    """
-    # Convert the list of embeddings to a NumPy array
-    X = np.vstack(embeddings)
-
-    # FAISS K-means clustering
-    ncentroids = find_optimal_clusters(X, max_clusters=2 * k, threshold=0.1)
-    k = min(k, ncentroids)
-    niter = 20
-    verbose = True
-    d = X.shape[1]
-    kmeans = faiss.Kmeans(d, k, niter=niter, verbose=verbose)
-    kmeans.train(X)
-
-    # Get the cluster centroids
-    centroids = kmeans.centroids
-
-    # Find the nearest neighbors of the centroids in the original embeddings
-    nbrs = NearestNeighbors(n_neighbors=1, algorithm="auto").fit(X)
-    distances, indices = nbrs.kneighbors(centroids)
-
-    # Sort the centroids by their nearest neighbor distances
-    sorted_centroids_indices = np.argsort(distances, axis=0).flatten()
-
-    # Select the top k densest clusters
-    densest_clusters_indices = sorted_centroids_indices[:k]
-
-    # Get the representative vectors and their indices from the densest clusters
-    representative_vectors = [
-        (idx, embeddings[idx]) for idx in densest_clusters_indices
-    ]
-
-    return representative_vectors
-
-
-def densest_clusters_DBSCAN(
-    embeddings: np.ndarray, k: int = 10
-) -> List[Tuple[int, np.ndarray]]:
-    """
-    Find the representative vector and corresponding index from each of the k densest
-    clusters in the given embeddings.
-
-    Args:
-        embeddings (np.ndarray): A NumPy array of shape (n, d), where n is the number
-                                 of embedding vectors and d is their dimensionality.
-        k (int): Number of densest clusters to find.
-
-    Returns:
-        List[Tuple[int, np.ndarray]]: A list of tuples containing the index and
-                                      representative vector for each of the k densest
-                                      clusters.
-    """
-
-    # Normalize the embeddings if necessary
-    scaler = StandardScaler()
-    embeddings_normalized = scaler.fit_transform(embeddings)
-
-    # Choose a clustering algorithm (DBSCAN in this case)
-    # Tune eps and min_samples for your use case
-    dbscan = DBSCAN(eps=4, min_samples=5)
-
-    # Apply the clustering algorithm
-    cluster_labels = dbscan.fit_predict(embeddings_normalized)
-
-    # Compute the densities of the clusters
-    cluster_density = Counter(cluster_labels)
-
-    # Sort clusters by their density
-    sorted_clusters = sorted(cluster_density.items(), key=lambda x: x[1], reverse=True)
-
-    # Select top-k densest clusters
-    top_k_clusters = sorted_clusters[:k]
-
-    # Find a representative vector for each cluster
-    representatives = []
-    for cluster_id, _ in top_k_clusters:
-        if cluster_id == -1:
-            continue  # Skip the noise cluster (label -1)
-        indices = np.where(cluster_labels == cluster_id)[0]
-        centroid = embeddings[indices].mean(axis=0)
-        closest_index = indices[
-            np.argmin(np.linalg.norm(embeddings[indices] - centroid, axis=1))
-        ]
-        representatives.append((closest_index, embeddings[closest_index]))
-
-    return representatives
-
-
-def densest_doc_clusters(
-    docs: List[Document], k: int, embedding_fn: Callable[[str], np.ndarray]
-) -> List[Document]:
-    """
-    Find the documents corresponding to the representative vectors of the k densest
-    clusters in the given list of documents.
-
-    Args:
-        docs (List[Document]): A list of Document instances, each containing a "content"
-                               field to be embedded and a "metadata" field.
-        k (int): Number of densest clusters to find.
-        embedding_fn (Callable[[str], np.ndarray]): A function that maps a string to an
-                                                    embedding vector.
-
-    Returns:
-        List[Document]: A list of Document instances corresponding to the representative
-                        vectors of the k densest clusters.
-    """
-
-    # Extract embeddings from the documents
-    embeddings = np.array(embedding_fn([doc.content for doc in docs]))
-
-    # Find the densest clusters and their representative indices
-    representative_indices_and_vectors = densest_clusters(embeddings, k)
-
-    # Extract the corresponding documents
-    representative_docs = [docs[i] for i, _ in representative_indices_and_vectors]
-
-    return representative_docs

langroid/experimental/team-save.py

@@ -1,391 +0,0 @@
-import logging
-from abc import ABC, abstractmethod
-from typing import Callable, Dict, List, Optional, Union
-
-import langroid as lr
-from langroid.language_models.mock_lm import MockLMConfig
-
-# Fix logging level type
-logging.basicConfig(level=logging.WARNING)
-logger = logging.getLogger(__name__)
-
-
-def sum_fn(s: str) -> str:
-    """Dummy response for MockLM"""
-    nums = [
-        int(subpart)
-        for part in s.split()
-        for subpart in part.split(",")
-        if subpart.isdigit()
-    ]
-    return str(sum(nums) + 1)
-
-
-def user_message(msg: Union[str, lr.ChatDocument]) -> lr.ChatDocument:
-    """Create a user-role msg from a string or ChatDocument"""
-    if isinstance(msg, lr.ChatDocument):
-        return msg
-    return lr.ChatDocument(
-        content=msg,
-        metadata=lr.ChatDocMetaData(
-            sender=lr.Entity.USER,
-            sender_name="user",
-        ),
-    )
-
-
-class InputContext:
-    """Context for a Component to respond to"""
-
-    def __init__(self) -> None:
-        self.messages: List[lr.ChatDocument] = []
-
-    def add(
-        self, results: Union[str, List[str], lr.ChatDocument, List[lr.ChatDocument]]
-    ) -> None:
-        """
-        Add messages to the input messages list
-        """
-        msgs: List[lr.ChatDocument] = []
-        if isinstance(results, str):
-            msgs = [user_message(results)]
-        elif isinstance(results, lr.ChatDocument):
-            msgs = [results]
-        elif isinstance(results, list):
-            if len(results) == 0:
-                return
-            if isinstance(results[0], str):
-                msgs = [user_message(r) for r in results]
-            else:
-                msgs = [r for r in results if isinstance(r, lr.ChatDocument)]
-        self.messages.extend(msgs)
-
-    def clear(self) -> None:
-        self.messages.clear()
-
-    def get_context(self) -> lr.ChatDocument:
-        """Construct a user-role ChatDocument from the input messages"""
-        if len(self.messages) == 0:
-            return lr.ChatDocument(content="", metadata={"sender": lr.Entity.USER})
-        content = "\n".join(
-            f"{msg.metadata.sender_name}: {msg.content}" for msg in self.messages
-        )
-        return lr.ChatDocument(content=content, metadata={"sender": lr.Entity.USER})
-
-
-class Scheduler(ABC):
-    """Schedule the Components of a Team"""
-
-    def __init__(self) -> None:
-        self.init_state()
-
-    def init_state(self) -> None:
-        self.stepped = False
-        self.responders: List[str] = []
-        self.responder_counts: Dict[str, int] = {}
-        self.current_result: List[lr.ChatDocument] = []
-
-    @abstractmethod
-    def step(self) -> None:
-        pass
-
-    @abstractmethod
-    def done(self) -> bool:
-        pass
-
-    @abstractmethod
-    def result(self) -> List[lr.ChatDocument]:
-        pass
-
-    def run(self) -> List[lr.ChatDocument]:
-        self.init_state()
-        while not self.done():
-            self.step()
-        return self.result()
-
-
-class Component(ABC):
-    """A component of a Team"""
-
-    def __init__(self) -> None:
-        self.input = InputContext()
-        self._listeners: List["Component"] = []
-        self.name: str = ""
-
-    @abstractmethod
-    def run(self) -> List[lr.ChatDocument]:
-        pass
-
-    def listen(self, component: Union["Component", List["Component"]]) -> None:
-        if isinstance(component, list):
-            for comp in component:
-                comp.listeners.append(self)
-        else:
-            component.listeners.append(self)
-
-    @property
-    def listeners(self) -> List["Component"]:
-        return self._listeners
-
-    def _notify(self, results: List[lr.ChatDocument]) -> None:
-        logger.warning(f"{self.name} Notifying listeners...")
-        for listener in self.listeners:
-            logger.warning(f"--> Listener {listener.name} notified")
-            listener.input.add(results)
-
-
-class SimpleScheduler(Scheduler):
-    def __init__(
-        self,
-        components: List[Component],
-    ) -> None:
-        super().__init__()
-        self.components = components  # Get components from team
-        self.stepped: bool = False
-
-    def step(self) -> None:
-        results = []
-        for comp in self.components:
-            result = comp.run()
-            if result:
-                results.extend(result)
-        self.current_result = results
-        self.stepped = True
-
-    def done(self) -> bool:
-        """done after 1 step, i.e. all components have responded"""
-        return self.stepped
-
-    def result(self) -> List[lr.ChatDocument]:
-        return self.current_result
-
-
-class OrElseScheduler(Scheduler):
-    """
-    Implements "OrElse scheduling", i.e. if the components are A, B, C, then
-    in each step, it will try for a valid response from A OrElse B OrElse C,
-    i.e. the first component that gives a valid response is chosen.
-    In the next step, it will start from the next component in the list,
-    cycling back to the first component after the last component.
-    (There may be a better name than OrElseScheduler though.)
-    """
-
-    def __init__(
-        self,
-        components: List[Component],
-    ) -> None:
-        super().__init__()
-        self.components = components
-        self.team: Optional[Team] = None
-        self.current_index: int = 0
-
-    def init_state(self) -> None:
-        super().init_state()
-        self.current_index = 0
-
-    def is_valid(self, result: Optional[List[lr.ChatDocument]]) -> bool:
-        return result is not None and len(result) > 0
-
-    def step(self) -> None:
-        start_index = self.current_index
-        n = len(self.components)
-
-        for i in range(n):
-            idx = (start_index + i) % n
-            comp = self.components[idx]
-            result = comp.run()
-            if self.is_valid(result):
-                self.responders.append(comp.name)
-                self.responder_counts[comp.name] = (
-                    self.responder_counts.get(comp.name, 0) + 1
-                )
-                self.current_result = result
-                # cycle to next component
-                self.current_index = (idx + 1) % n
-                return
-
-    def done(self) -> bool:
-        if self.team is None:
-            return False
-        return self.team.done(self)
-
-    def result(self) -> List[lr.ChatDocument]:
-        return self.current_result
-
-
-class Team(Component):
-    def __init__(
-        self,
-        name: str,
-        done_condition: Optional[Callable[["Team", Scheduler], bool]] = None,
-    ) -> None:
-        super().__init__()
-        self.name = name
-        self.components: List[Component] = []
-        self.scheduler: Optional[Scheduler] = None
-        self.done_condition = done_condition or Team.default_done_condition
-
-    def set_done_condition(
-        self, done_condition: Callable[["Team", Scheduler], bool]
-    ) -> None:
-        self.done_condition = done_condition
-
-    def done(self, scheduler: Scheduler) -> bool:
-        return self.done_condition(self, scheduler)
-
-    def default_done_condition(self, scheduler: Scheduler) -> bool:
-        # Default condition, can be overridden
-        return False
-
-    def add_scheduler(self, scheduler_class: type) -> None:
-        self.scheduler = scheduler_class(self.components)
-        if hasattr(self.scheduler, "team"):
-            setattr(self.scheduler, "team", self)
-
-    def add(self, component: Union[Component, List[Component]]) -> None:
-        if isinstance(component, list):
-            self.components.extend(component)
-        else:
-            self.components.append(component)
-
-    def reset(self) -> None:
-        self.input.clear()
-        if self.scheduler is not None:
-            self.scheduler.init_state()
-
-    def run(self, input: str | lr.ChatDocument | None = None) -> List[lr.ChatDocument]:
-        if input is not None:
-            self.input.add(input)
-        if self.scheduler is None:
-            raise ValueError(
-                f"Team '{self.name}' has no scheduler. Call add_scheduler() first."
-            )
-        input_str = self.input.get_context().content
-        logger.warning(f"Running team {self.name}... on input = {input_str}")
-        # push the input of self to each component that's a listener of self.
-        n_pushed = 0
-        for comp in self.components:
-
-            if comp in self.listeners:
-                comp.input.add(self.input.messages)
-                n_pushed += 1
-        if len(self.input.messages) > 0 and n_pushed == 0:
-            logger.warning(
-                """
-                Warning: Team inputs not pushed to any components!
-                You may not be able to run any components unless they have their
-                own inputs. Make sure to set up component to listen to parent team
-                if needed.
-                """
-            )
-        # clear own input since we've pushed it to internal components
-        self.input.clear()
-
-        result = self.scheduler.run()
-        if len(result) > 0:
-            self._notify(result)
-        result_value = result[0].content if len(result) > 0 else "null"
-        logger.warning(f"Team {self.name} done: {result_value}")
-        return result
-
-
-class DummyAgent:
-    def __init__(self, name: str) -> None:
-        self.name = name
-
-    def process(self, data: str) -> str:
-        return f"{self.name} processed: {data}"
-
-
-class TaskComponent(Component):
-    def __init__(self, task: lr.Task) -> None:
-        super().__init__()
-        self.task = task
-        self.name = task.agent.config.name
-
-    def run(self, input: str | lr.ChatDocument | None = None) -> List[lr.ChatDocument]:
-        if input is not None:
-            self.input.add(input)
-        input_msg = self.input.get_context()
-        if input_msg.content == "":
-            return []
-        logger.warning(f"Running task {self.name} on input = {input_msg.content}")
-        result = self.task.run(input_msg)
-        result_value = result.content if result else "null"
-        logger.warning(f"Task {self.name} done: {result_value}")
-        result_list = [result] if result else []
-        if len(result_list) > 0:
-            self._notify(result_list)
-            self.input.clear()  # clear own input since we just consumed it!
-        return result_list
-
-
-def make_task(name: str, sys: str = "") -> TaskComponent:
-    llm_config = MockLMConfig(response_fn=sum_fn)
-    agent = lr.ChatAgent(
-        lr.ChatAgentConfig(
-            llm=llm_config,
-            name=name,
-        )
-    )
-    # set as single_round since there are no Tools
-    task = lr.Task(agent, interactive=False, single_round=True)
-    return TaskComponent(task)
-
-
-if __name__ == "__main__":
-    # Create agents, tasks
-    t1 = make_task("a1")
-    t2 = make_task("a2")
-    t3 = make_task("a3")
-
-    # done conditions for each time
-    def team1_done_condition(team: Team, scheduler: Scheduler) -> bool:
-        return (
-            scheduler.responder_counts.get("a1", 0) >= 2
-            and scheduler.responder_counts.get("a2", 0) >= 2
-        )
-
-    def team2_done_condition(team: Team, scheduler: Scheduler) -> bool:
-        return "a3" in scheduler.responders
-
-    def general_team_done_condition(team: Team, scheduler: Scheduler) -> bool:
-        # Example: all components have responded at least once
-        return len(set(scheduler.responders)) == len(team.components)
-
-    # Create teams
-    team1 = Team("T1", done_condition=team1_done_condition)
-    team2 = Team("T2", done_condition=team2_done_condition)
-
-    team = Team("Team", done_condition=general_team_done_condition)
-
-    team1.add_scheduler(OrElseScheduler)
-    team2.add_scheduler(OrElseScheduler)
-    team.add_scheduler(OrElseScheduler)
-
-    team.add([team1, team2])
-
-    # Build hierarchy
-    team1.add([t1, t2])
-    team2.add(t3)
-
-    # Set up listening
-    # team2.listen(team1)  # listens to team1 final result
-    team1.listen(team)
-    t1.listen(team1)
-    t2.listen(t1)
-    t1.listen(t2)
-    # TODO should we forbid listening to a component OUTSIDE the team?
-
-    # t3 listens to its parent team2 =>
-    # any input to team2 gets pushed to t3 when t3 runs
-    team2.listen([t1, t2])
-    t3.listen(team2)
-
-    # TODO - we should either define which component of a team gets the teams inputs,
-    # or explicitly add messages to a specific component of the team
-
-    print("Running top-level team...")
-    result = team.run("1")
-
-    ##########

langroid/language_models/.chainlit/config.toml

@@ -1,121 +0,0 @@
-[project]
-# Whether to enable telemetry (default: true). No personal data is collected.
-enable_telemetry = true
-
-
-# List of environment variables to be provided by each user to use the app.
-user_env = []
-
-# Duration (in seconds) during which the session is saved when the connection is lost
-session_timeout = 3600
-
-# Enable third parties caching (e.g LangChain cache)
-cache = false
-
-# Authorized origins
-allow_origins = ["*"]
-
-# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
-# follow_symlink = false
-
-[features]
-# Show the prompt playground
-prompt_playground = true
-
-# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
-unsafe_allow_html = false
-
-# Process and display mathematical expressions. This can clash with "$" characters in messages.
-latex = false
-
-# Automatically tag threads with the current chat profile (if a chat profile is used)
-auto_tag_thread = true
-
-# Authorize users to spontaneously upload files with messages
-[features.spontaneous_file_upload]
-    enabled = true
-    accept = ["*/*"]
-    max_files = 20
-    max_size_mb = 500
-
-[features.audio]
-    # Threshold for audio recording
-    min_decibels = -45
-    # Delay for the user to start speaking in MS
-    initial_silence_timeout = 3000
-    # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop.
-    silence_timeout = 1500
-    # Above this duration (MS), the recording will forcefully stop.
-    max_duration = 15000
-    # Duration of the audio chunks in MS
-    chunk_duration = 1000
-    # Sample rate of the audio
-    sample_rate = 44100
-
-[UI]
-# Name of the app and chatbot.
-name = "Chatbot"
-
-# Show the readme while the thread is empty.
-show_readme_as_default = true
-
-# Description of the app and chatbot. This is used for HTML tags.
-# description = ""
-
-# Large size content are by default collapsed for a cleaner ui
-default_collapse_content = true
-
-# The default value for the expand messages settings.
-default_expand_messages = false
-
-# Hide the chain of thought details from the user in the UI.
-hide_cot = false
-
-# Link to your github repo. This will add a github button in the UI's header.
-# github = ""
-
-# Specify a CSS file that can be used to customize the user interface.
-# The CSS file can be served from the public directory or via an external link.
-# custom_css = "/public/test.css"
-
-# Specify a Javascript file that can be used to customize the user interface.
-# The Javascript file can be served from the public directory.
-# custom_js = "/public/test.js"
-
-# Specify a custom font url.
-# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap"
-
-# Specify a custom meta image url.
-# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
-
-# Specify a custom build directory for the frontend.
-# This can be used to customize the frontend code.
-# Be careful: If this is a relative path, it should not start with a slash.
-# custom_build = "./public/build"
-
-[UI.theme]
-    #layout = "wide"
-    #font_family = "Inter, sans-serif"
-# Override default MUI light theme. (Check theme.ts)
-[UI.theme.light]
-    #background = "#FAFAFA"
-    #paper = "#FFFFFF"
-
-    [UI.theme.light.primary]
-        #main = "#F80061"
-        #dark = "#980039"
-        #light = "#FFE7EB"
-
-# Override default MUI dark theme. (Check theme.ts)
-[UI.theme.dark]
-    #background = "#FAFAFA"
-    #paper = "#FFFFFF"
-
-    [UI.theme.dark.primary]
-        #main = "#F80061"
-        #dark = "#980039"
-        #light = "#FFE7EB"
-
-
-[meta]
-generated_by = "1.1.202"