langroid 0.33.6__py3-none-any.whl → 0.33.8__py3-none-any.whl

langroid/utils/pandas_utils.py

@@ -0,0 +1,30 @@
+from typing import Any
+
+import pandas as pd
+
+
+def stringify(x: Any) -> str:
+    # Convert x to DataFrame if it is not one already
+    if isinstance(x, pd.Series):
+        df = x.to_frame()
+    elif not isinstance(x, pd.DataFrame):
+        return str(x)
+    else:
+        df = x
+
+    # Truncate long text columns to 1000 characters
+    for col in df.columns:
+        if df[col].dtype == object:
+            df[col] = df[col].apply(
+                lambda item: (
+                    (item[:1000] + "...")
+                    if isinstance(item, str) and len(item) > 1000
+                    else item
+                )
+            )
+
+    # Limit to 10 rows
+    df = df.head(10)
+
+    # Convert to string
+    return df.to_string(index=False)  # type: ignore

langroid/utils/pydantic_utils.py

@@ -0,0 +1,602 @@
+import logging
+from collections.abc import MutableMapping
+from contextlib import contextmanager
+from typing import (
+    Any,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    no_type_check,
+)
+
+import numpy as np
+import pandas as pd
+
+from langroid.mytypes import DocMetaData, Document
+from langroid.pydantic_v1 import BaseModel, ValidationError, create_model
+
+logger = logging.getLogger(__name__)
+
+
+def flatten_dict(
+    d: MutableMapping[str, Any], parent_key: str = "", sep: str = "."
+) -> Dict[str, Any]:
+    """Flatten a nested dictionary, using a separator in the keys.
+    Useful for pydantic_v1 models with nested fields -- first use
+        dct = mdl.model_dump()
+    to get a nested dictionary, then use this function to flatten it.
+    """
+    items: List[Tuple[str, Any]] = []
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, MutableMapping):
+            items.extend(flatten_dict(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
+def has_field(model_class: Type[BaseModel], field_name: str) -> bool:
+    """Check if a Pydantic model class has a field with the given name."""
+    return field_name in model_class.__fields__
+
+
+def _recursive_purge_dict_key(d: Dict[str, Any], k: str) -> None:
+    """Remove a key from a dictionary recursively"""
+    if isinstance(d, dict):
+        for key in list(d.keys()):
+            if key == k and "type" in d.keys():
+                del d[key]
+            else:
+                _recursive_purge_dict_key(d[key], k)
+
+
+@no_type_check
+def _flatten_pydantic_model_ignore_defaults(
+    model: Type[BaseModel],
+    base_model: Type[BaseModel] = BaseModel,
+) -> Type[BaseModel]:
+    """
+    Given a possibly nested Pydantic class, return a flattened version of it,
+    by constructing top-level fields, whose names are formed from the path
+    through the nested structure, separated by double underscores.
+
+    This version ignores inherited defaults, so it is incomplete.
+    But retaining it as it is simpler and may be useful in some cases.
+    The full version is `flatten_pydantic_model`, see below.
+
+    Args:
+        model (Type[BaseModel]): The Pydantic model to flatten.
+        base_model (Type[BaseModel], optional): The base model to use for the
+            flattened model. Defaults to BaseModel.
+
+    Returns:
+        Type[BaseModel]: The flattened Pydantic model.
+    """
+
+    flattened_fields: Dict[str, Tuple[Any, ...]] = {}
+    models_to_process = [(model, "")]
+
+    while models_to_process:
+        current_model, current_prefix = models_to_process.pop()
+
+        for name, field in current_model.__annotations__.items():
+            if issubclass(field, BaseModel):
+                new_prefix = (
+                    f"{current_prefix}{name}__" if current_prefix else f"{name}__"
+                )
+                models_to_process.append((field, new_prefix))
+            else:
+                flattened_name = f"{current_prefix}{name}"
+                flattened_fields[flattened_name] = (field, ...)
+
+    return create_model(
+        "FlatModel",
+        __base__=base_model,
+        **flattened_fields,
+    )
+
+
+def flatten_pydantic_model(
+    model: Type[BaseModel],
+    base_model: Type[BaseModel] = BaseModel,
+) -> Type[BaseModel]:
+    """
+    Given a possibly nested Pydantic class, return a flattened version of it,
+    by constructing top-level fields, whose names are formed from the path
+    through the nested structure, separated by double underscores.
+
+    Args:
+        model (Type[BaseModel]): The Pydantic model to flatten.
+        base_model (Type[BaseModel], optional): The base model to use for the
+            flattened model. Defaults to BaseModel.
+
+    Returns:
+        Type[BaseModel]: The flattened Pydantic model.
+    """
+
+    flattened_fields: Dict[str, Any] = {}
+    models_to_process = [(model, "")]
+
+    while models_to_process:
+        current_model, current_prefix = models_to_process.pop()
+
+        for name, field in current_model.__fields__.items():
+            if isinstance(field.outer_type_, type) and issubclass(
+                field.outer_type_, BaseModel
+            ):
+                new_prefix = (
+                    f"{current_prefix}{name}__" if current_prefix else f"{name}__"
+                )
+                models_to_process.append((field.outer_type_, new_prefix))
+            else:
+                flattened_name = f"{current_prefix}{name}"
+
+                if field.default_factory is not field.default_factory:
+                    flattened_fields[flattened_name] = (
+                        field.outer_type_,
+                        field.default_factory,
+                    )
+                elif field.default is not field.default:
+                    flattened_fields[flattened_name] = (
+                        field.outer_type_,
+                        field.default,
+                    )
+                else:
+                    flattened_fields[flattened_name] = (field.outer_type_, ...)
+
+    return create_model("FlatModel", __base__=base_model, **flattened_fields)
+
+
+def get_field_names(model: Type[BaseModel]) -> List[str]:
+    """Get all field names from a possibly nested Pydantic model."""
+    mdl = flatten_pydantic_model(model)
+    fields = list(mdl.__fields__.keys())
+    # fields may be like a__b__c , so we only want the last part
+    return [f.split("__")[-1] for f in fields]
+
+
+def generate_simple_schema(
+    model: Type[BaseModel], exclude: List[str] = []
+) -> Dict[str, Any]:
+    """
+    Generates a JSON schema for a Pydantic model,
+    with options to exclude specific fields.
+
+    This function traverses the Pydantic model's fields, including nested models,
+    to generate a dictionary representing the JSON schema. Fields specified in
+    the exclude list will not be included in the generated schema.
+
+    Args:
+        model (Type[BaseModel]): The Pydantic model class to generate the schema for.
+        exclude (List[str]): A list of string field names to be excluded from the
+                             generated schema. Defaults to an empty list.
+
+    Returns:
+        Dict[str, Any]: A dictionary representing the JSON schema of the provided model,
+                        with specified fields excluded.
+    """
+    if hasattr(model, "__fields__"):
+        output: Dict[str, Any] = {}
+        for field_name, field in model.__fields__.items():
+            if field_name in exclude:
+                continue  # Skip excluded fields
+
+            field_type = field.type_
+            if issubclass(field_type, BaseModel):
+                # Recursively generate schema for nested models
+                output[field_name] = generate_simple_schema(field_type, exclude)
+            else:
+                # Represent the type as a string here
+                output[field_name] = {"type": field_type.__name__}
+        return output
+    else:
+        # Non-model type, return a simplified representation
+        return {"type": model.__name__}
+
+
+def flatten_pydantic_instance(
+    instance: BaseModel,
+    prefix: str = "",
+    force_str: bool = False,
+) -> Dict[str, Any]:
+    """
+    Given a possibly nested Pydantic instance, return a flattened version of it,
+    as a dict where nested traversal paths are translated to keys a__b__c.
+
+    Args:
+        instance (BaseModel): The Pydantic instance to flatten.
+        prefix (str, optional): The prefix to use for the top-level fields.
+        force_str (bool, optional): Whether to force all values to be strings.
+
+    Returns:
+        Dict[str, Any]: The flattened dict.
+
+    """
+    flat_data: Dict[str, Any] = {}
+    for name, value in instance.dict().items():
+        # Assuming nested pydantic model will be a dict here
+        if isinstance(value, dict):
+            nested_flat_data = flatten_pydantic_instance(
+                instance.__fields__[name].type_(**value),
+                prefix=f"{prefix}{name}__",
+                force_str=force_str,
+            )
+            flat_data.update(nested_flat_data)
+        else:
+            flat_data[f"{prefix}{name}"] = str(value) if force_str else value
+    return flat_data
+
+
+def extract_fields(doc: BaseModel, fields: List[str]) -> Dict[str, Any]:
+    """
+    Extract specified fields from a Pydantic object.
+    Supports dotted field names, e.g. "metadata.author".
+    Dotted fields are matched exactly according to the corresponding path.
+    Non-dotted fields are matched against the last part of the path.
+    Clashes ignored.
+    Args:
+        doc (BaseModel): The Pydantic object.
+        fields (List[str]): The list of fields to extract.
+
+    Returns:
+        Dict[str, Any]: A dictionary of field names and values.
+
+    """
+
+    def get_value(obj: BaseModel, path: str) -> Any | None:
+        for part in path.split("."):
+            if hasattr(obj, part):
+                obj = getattr(obj, part)
+            else:
+                return None
+        return obj
+
+    def traverse(obj: BaseModel, result: Dict[str, Any], prefix: str = "") -> None:
+        for k, v in obj.__dict__.items():
+            key = f"{prefix}.{k}" if prefix else k
+            if isinstance(v, BaseModel):
+                traverse(v, result, key)
+            else:
+                result[key] = v
+
+    result: Dict[str, Any] = {}
+
+    # Extract values for dotted field names and use last part as key
+    for field in fields:
+        if "." in field:
+            value = get_value(doc, field)
+            if value is not None:
+                key = field.split(".")[-1]
+                result[key] = value
+
+    # Traverse the object to get non-dotted fields
+    all_fields: Dict[str, Any] = {}
+    traverse(doc, all_fields)
+
+    # Add non-dotted fields to the result,
+    # avoid overwriting if already present from dotted names
+    for field in [f for f in fields if "." not in f]:
+        for key, value in all_fields.items():
+            if key.split(".")[-1] == field and field not in result:
+                result[field] = value
+
+    return result
+
+
+def nested_dict_from_flat(
+    flat_data: Dict[str, Any],
+    sub_dict: str = "",
+) -> Dict[str, Any]:
+    """
+    Given a flattened version of a nested dict, reconstruct the nested dict.
+    Field names in the flattened dict are assumed to be of the form
+    "field1__field2__field3", going from top level down.
+
+    Args:
+        flat_data (Dict[str, Any]): The flattened dict.
+        sub_dict (str, optional): The name of the sub-dict to extract from the
+            flattened dict. Defaults to "" (extract the whole dict).
+
+    Returns:
+        Dict[str, Any]: The nested dict.
+
+    """
+    nested_data: Dict[str, Any] = {}
+    for key, value in flat_data.items():
+        if sub_dict != "" and not key.startswith(sub_dict + "__"):
+            continue
+        keys = key.split("__")
+        d = nested_data
+        for k in keys[:-1]:
+            d = d.setdefault(k, {})
+        d[keys[-1]] = value
+    if sub_dict != "":  # e.g. "payload"
+        nested_data = nested_data[sub_dict]
+    return nested_data
+
+
+def pydantic_obj_from_flat_dict(
+    flat_data: Dict[str, Any],
+    model: Type[BaseModel],
+    sub_dict: str = "",
+) -> BaseModel:
+    """Flattened dict with a__b__c style keys -> nested dict -> pydantic object"""
+    nested_data = nested_dict_from_flat(flat_data, sub_dict)
+    return model(**nested_data)
+
+
+@contextmanager
+def temp_update(
+    pydantic_object: BaseModel, updates: Dict[str, Any]
+) -> Generator[None, None, None]:
+    original_values = {}
+    try:
+        for field, value in updates.items():
+            if hasattr(pydantic_object, field):
+                # Save original value
+                original_values[field] = getattr(pydantic_object, field)
+                setattr(pydantic_object, field, value)
+            else:
+                # Raise error for non-existent field
+                raise AttributeError(
+                    f"The field '{field}' does not exist in the "
+                    f"Pydantic model '{pydantic_object.__class__.__name__}'."
+                )
+        yield
+    except ValidationError as e:
+        # Handle validation error
+        print(f"Validation error: {e}")
+    finally:
+        # Restore original values
+        for field, value in original_values.items():
+            setattr(pydantic_object, field, value)
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+@contextmanager
+def temp_params(config: T, field: str, temp: T) -> Generator[None, None, None]:
+    """Context manager to temporarily override `field` in a `config`"""
+    original_vals = getattr(config, field)
+    try:
+        # Apply temporary settings
+        setattr(config, field, temp)
+        yield
+    finally:
+        # Revert to original settings
+        setattr(config, field, original_vals)
+
+
+def numpy_to_python_type(numpy_type: Type[Any]) -> Type[Any]:
+    """Converts a numpy data type to its Python equivalent."""
+    type_mapping = {
+        np.float64: float,
+        np.float32: float,
+        np.int64: int,
+        np.int32: int,
+        np.bool_: bool,
+        # Add other numpy types as necessary
+    }
+    return type_mapping.get(numpy_type, numpy_type)
+
+
+def dataframe_to_pydantic_model(df: pd.DataFrame) -> Type[BaseModel]:
+    """Make a Pydantic model from a dataframe."""
+    fields = {col: (type(df[col].iloc[0]), ...) for col in df.columns}
+    return create_model("DataFrameModel", __base__=BaseModel, **fields)  # type: ignore
+
+
+def dataframe_to_pydantic_objects(df: pd.DataFrame) -> List[BaseModel]:
+    """Make a list of Pydantic objects from a dataframe."""
+    Model = dataframe_to_pydantic_model(df)
+    return [Model(**row.to_dict()) for index, row in df.iterrows()]
+
+
+def first_non_null(series: pd.Series) -> Any | None:
+    """Find the first non-null item in a pandas Series."""
+    for item in series:
+        if item is not None:
+            return item
+    return None
+
+
+def dataframe_to_document_model(
+    df: pd.DataFrame,
+    content: str = "content",
+    metadata: List[str] = [],
+    exclude: List[str] = [],
+) -> Type[BaseModel]:
+    """
+    Make a subclass of Document from a dataframe.
+
+    Args:
+        df (pd.DataFrame): The dataframe.
+        content (str): The name of the column containing the content,
+            which will map to the Document.content field.
+        metadata (List[str]): A list of column names containing metadata;
+            these will be included in the Document.metadata field.
+        exclude (List[str]): A list of column names to exclude from the model.
+            (e.g. "vector" when lance is used to add an embedding vector to the df)
+
+    Returns:
+        Type[BaseModel]: A pydantic model subclassing Document.
+    """
+
+    # Remove excluded columns
+    df = df.drop(columns=exclude, inplace=False)
+    # Check if metadata_cols is empty
+
+    if metadata:
+        # Define fields for the dynamic subclass of DocMetaData
+        metadata_fields = {
+            col: (
+                Optional[numpy_to_python_type(type(first_non_null(df[col])))],
+                None,  # Optional[numpy_to_python_type(type(first_non_null(df[col])))],
+            )
+            for col in metadata
+        }
+        DynamicMetaData = create_model(  # type: ignore
+            "DynamicMetaData", __base__=DocMetaData, **metadata_fields
+        )
+    else:
+        # Use the base DocMetaData class directly
+        DynamicMetaData = DocMetaData
+
+    # Define additional top-level fields for DynamicDocument
+    additional_fields = {
+        col: (
+            Optional[numpy_to_python_type(type(first_non_null(df[col])))],
+            None,  # Optional[numpy_to_python_type(type(first_non_null(df[col])))],
+        )
+        for col in df.columns
+        if col not in metadata and col != content
+    }
+
+    # Create a dynamic subclass of Document
+    DynamicDocumentFields = {
+        **{"metadata": (DynamicMetaData, ...)},
+        **additional_fields,
+    }
+    DynamicDocument = create_model(  # type: ignore
+        "DynamicDocument", __base__=Document, **DynamicDocumentFields
+    )
+
+    def from_df_row(
+        cls: type[BaseModel],
+        row: pd.Series,
+        content: str = "content",
+        metadata: List[str] = [],
+    ) -> BaseModel | None:
+        content_val = row[content] if (content and content in row) else ""
+        metadata_values = (
+            {col: row[col] for col in metadata if col in row} if metadata else {}
+        )
+        additional_values = {
+            col: row[col] for col in additional_fields if col in row and col != content
+        }
+        metadata = DynamicMetaData(**metadata_values)
+        return cls(content=content_val, metadata=metadata, **additional_values)
+
+    # Bind the method to the class
+    DynamicDocument.from_df_row = classmethod(from_df_row)
+
+    return DynamicDocument  # type: ignore
+
+
+def dataframe_to_documents(
+    df: pd.DataFrame,
+    content: str = "content",
+    metadata: List[str] = [],
+    doc_cls: Type[BaseModel] | None = None,
+) -> List[Document]:
+    """
+    Make a list of Document objects from a dataframe.
+    Args:
+        df (pd.DataFrame): The dataframe.
+        content (str): The name of the column containing the content,
+            which will map to the Document.content field.
+        metadata (List[str]): A list of column names containing metadata;
+            these will be included in the Document.metadata field.
+        doc_cls (Type[BaseModel], optional): A Pydantic model subclassing
+            Document. Defaults to None.
+    Returns:
+        List[Document]: The list of Document objects.
+    """
+    Model = doc_cls or dataframe_to_document_model(df, content, metadata)
+    docs = [
+        Model.from_df_row(row, content, metadata)  # type: ignore
+        for _, row in df.iterrows()
+    ]
+    return [m for m in docs if m is not None]
+
+
+def extra_metadata(document: Document, doc_cls: Type[Document] = Document) -> List[str]:
+    """
+    Checks for extra fields in a document's metadata that are not defined in the
+    original metadata schema.
+
+    Args:
+        document (Document): The document instance to check for extra fields.
+        doc_cls (Type[Document]): The class type derived from Document, used
+            as a reference to identify extra fields in the document's metadata.
+
+    Returns:
+        List[str]: A list of strings representing the keys of the extra fields found
+        in the document's metadata.
+    """
+    # Convert metadata to dict, including extra fields.
+    metadata_fields = set(document.metadata.dict().keys())
+
+    # Get defined fields in the metadata of doc_cls
+    defined_fields = set(doc_cls.__fields__["metadata"].type_.__fields__.keys())
+
+    # Identify extra fields not in defined fields.
+    extra_fields = list(metadata_fields - defined_fields)
+
+    return extra_fields
+
+
+def extend_document_class(d: Document) -> Type[Document]:
+    """Generates a new pydantic class based on a given document instance.
+
+    This function dynamically creates a new pydantic class with additional
+    fields based on the "extra" metadata fields present in the given document
+    instance. The new class is a subclass of the original Document class, with
+    the original metadata fields retained and extra fields added as normal
+    fields to the metadata.
+
+    Args:
+        d: An instance of the Document class.
+
+    Returns:
+        A new subclass of the Document class that includes the additional fields
+        found in the metadata of the given document instance.
+    """
+    # Extract the fields from the original metadata class, including types,
+    # correctly handling special types like List[str].
+    original_metadata_fields = {
+        k: (v.outer_type_ if v.shape != 1 else v.type_, ...)
+        for k, v in DocMetaData.__fields__.items()
+    }
+    # Extract extra fields from the metadata instance with their types
+    extra_fields = {
+        k: (type(v), ...)
+        for k, v in d.metadata.__dict__.items()
+        if k not in DocMetaData.__fields__
+    }
+
+    # Combine original and extra fields for the new metadata class
+    combined_fields = {**original_metadata_fields, **extra_fields}
+
+    # Create a new metadata class with combined fields
+    NewMetadataClass = create_model(  # type: ignore
+        "ExtendedDocMetadata", **combined_fields, __base__=DocMetaData
+    )
+    # NewMetadataClass.__config__.arbitrary_types_allowed = True
+
+    # Create a new document class using the new metadata class
+    NewDocumentClass = create_model(
+        "ExtendedDocument",
+        content=(str, ...),
+        metadata=(NewMetadataClass, ...),
+        __base__=Document,
+    )
+
+    return NewDocumentClass
+
+
+class PydanticWrapper(BaseModel):
+    value: Any
+
+
+def get_pydantic_wrapper(value_type: type) -> type[PydanticWrapper]:
+    class WrappedValue(PydanticWrapper):
+        value: value_type  # type: ignore
+
+    return WrappedValue