beekeeper-ai 0.6.6__py3-none-any.whl → 1.0.1__py3-none-any.whl

beekeeper/document_loaders/docx.py

@@ -1,31 +0,0 @@
-import os
-from pathlib import Path
-from typing import List, Optional
-
-from beekeeper.core.document import Document
-from beekeeper.core.document_loaders import BaseLoader
-
-
-class DocxLoader(BaseLoader):
-    """Microsoft Word (Docx) loader."""
-
-    def load_data(self, input_file: str, extra_info: Optional[dict] = None) -> List[Document]:
-        """Loads data from the specified directory.
-        
-        Args:
-            input_file (str): File path to load.
-        """
-        try:
-            import docx2txt  # noqa: F401
-        except ImportError:
-            raise ImportError("docx2txt package not found, please install it with `pip install docx2txt`")
-        
-        if not os.path.isfile(input_file):
-            raise ValueError(f"File `{input_file}` does not exist")
-        
-        input_file = str(Path(input_file).resolve())
-
-        text = docx2txt.process(input_file)
-        metadata = {"source": input_file}
-
-        return [Document(text=text, metadata=metadata)]

beekeeper/document_loaders/html.py

@@ -1,77 +0,0 @@
-import os
-from pathlib import Path
-from typing import List, Optional
-
-from beekeeper.core.document import Document
-from beekeeper.core.document_loaders import BaseLoader
-
-
-class HTMLLoader(BaseLoader):
-    """Load HTML file and extract text from a specific tag.
-
-    Args:
-        tag (str): HTML tag to extract. Defaults to ``section``.
-    """
-    
-    tag: str = "section"
-
-    def load_data(self, input_file: str, extra_info: Optional[dict] = None) -> List[Document]:
-        """Loads data from the specified directory.
-        
-        Args:
-            input_file (str): File path to load.
-        """
-        try:
-            from bs4 import BeautifulSoup  # noqa: F401
-        except ImportError:
-            raise ImportError("beautifulsoup4 package not found, please install it with `pip install beautifulsoup4`")
-        
-        if not os.path.isfile(input_file):
-            raise ValueError(f"File `{input_file}` does not exist")
-        
-        input_file = str(Path(input_file).resolve())
-
-        with open(input_file, encoding="utf-8") as html_file:
-            soup = BeautifulSoup(html_file, "html.parser")
-
-        tags = soup.find_all(self.tag)
-        documents = []
-
-        for tag in tags:
-            tag_text = self._extract_text_from_tag(tag)
-
-            metadata = {
-                "tag": self.tag,
-                "source": input_file,
-            }
-
-            doc = Document(
-                text=tag_text,
-                metadata=metadata,
-            )
-
-            documents.append(doc)
-
-        return documents
-
-    def _extract_text_from_tag(self, tag) -> str:
-        """Extract the text from an HTML tag, ignoring other nested tags."""
-        try:
-            from bs4 import NavigableString  # noqa: F401
-        except ImportError:
-            raise ImportError("beautifulsoup4 package not found, please install it with `pip install beautifulsoup4`")
-
-        texts = []
-
-        for elem in tag.children:
-            # Check if the element is a text node, not a tag
-            if isinstance(elem, NavigableString):
-                if elem.strip():
-                    texts.append(elem.strip())
-            # Ignore any tag that matches the main tag being processed (to avoid recursion)
-            elif elem.name == self.tag:
-                continue
-            else:
-                texts.append(elem.get_text().strip())
-
-        return "\n".join(texts)

beekeeper/document_loaders/json.py

@@ -1,53 +0,0 @@
-import json
-import os
-from pathlib import Path
-from typing import List, Optional
-
-from beekeeper.core.document import Document
-from beekeeper.core.document_loaders import BaseLoader
-
-
-class JSONLoader(BaseLoader):
-    """JSON loader.
-
-    Args:
-        jq_schema (str, optional): jq schema to use to extract the data from the JSON.
-    """
-
-    jq_schema: Optional[str] = None
-
-    def load_data(self, input_file: str, extra_info: Optional[dict] = None) -> List[Document]:
-        """Loads data from the specified directory.
-        
-        Args:
-            input_file (str): File path to load.
-        """
-        try:
-            import jq  # noqa: F401
-        except ImportError:
-            raise ImportError("jq package not found, please install it with `pip install jq`")
-        
-        if not os.path.isfile(input_file):
-            raise ValueError(f"File `{input_file}` does not exist")
-        
-        documents = []
-        jq_compiler = jq.compile(self.jq_schema)
-        json_file = Path(input_file).resolve().read_text(encoding="utf-8")
-        json_data = jq_compiler.input(json.loads(json_file))
-        
-        
-        for content in json_data:
-
-            if isinstance(content, str):
-                content = content
-            elif isinstance(content, dict):
-                content = json.dumps(content) if content else ""
-            else:
-                content = str(content) if content is not None else ""
-            
-            if content.strip() != "":
-                documents.append(Document(
-                    text=content,
-                    metadata={"source": str(Path(input_file).resolve())}))
-
-        return documents

beekeeper/document_loaders/pdf.py

@@ -1,38 +0,0 @@
-import logging
-import os
-from pathlib import Path
-from typing import List, Optional
-
-from beekeeper.core.document import Document
-from beekeeper.core.document_loaders import BaseLoader
-
-logging.getLogger("pypdf").setLevel(logging.ERROR)
-
-class PDFLoader(BaseLoader):
-    """PDF loader using PyPDF."""
-
-    def load_data(self, input_file: str, extra_info: Optional[dict] = None) -> List[Document]:
-        """Loads data from the specified directory.
-        
-        Args:
-            input_file (str): File path to load.
-        """
-        try:
-            import pypdf  # noqa: F401
-
-        except ImportError:
-            raise ImportError("pypdf package not found, please install it with `pip install pypdf`")
-        
-        if not os.path.isfile(input_file):
-            raise ValueError(f"File `{input_file}` does not exist")
-        
-        input_file = str(Path(input_file).resolve())
-        pdf_loader = pypdf.PdfReader(input_file)
-
-        return [
-            Document(
-                text=page.extract_text().strip(),
-                metadata={"source": input_file, "page": page_number}
-            )
-            for page_number, page in enumerate(pdf_loader.pages)
-        ]

beekeeper/document_loaders/s3.py

@@ -1,72 +0,0 @@
-import os
-import re
-import tempfile
-from typing import List, Optional
-
-from beekeeper.core.document import Document
-from beekeeper.core.document_loaders import BaseLoader
-from beekeeper.document_loaders import DirectoryLoader
-
-
-class S3Loader(BaseLoader):
-    """S3 bucket loader.
-
-    Args:
-        bucket (str): Name of the S3 bucket.
-        ibm_api_key_id (str): IBM Cloud API key.
-        ibm_service_instance_id (str): Service instance ID for the IBM COS.
-        s3_endpoint_url (str): Endpoint URL for the S3 service.
-
-    **Example**
-
-    .. code-block:: python
-
-        from beekeeper.document_loaders import S3Loader
-
-        loader = S3Loader(bucket="your_bucket",
-                     ibm_api_key_id="your_api_key",
-                     ibm_service_instance_id="your_instance_id",
-                     s3_endpoint_url="your_api_url")
-    """
-
-    def __init__(self, bucket: str,
-                 ibm_api_key_id: str = None,
-                 ibm_service_instance_id: str = None,
-                 s3_endpoint_url: str = None
-                 ):
-
-        try:
-            import ibm_boto3
-            from ibm_botocore.client import Config
-
-            self._ibm_boto3 = ibm_boto3
-            self._boto_config = Config
-        except ImportError:
-            raise ImportError("ibm-cos-sdk package not found, please install it with `pip install ibm-cos-sdk`")
-
-        self.bucket = bucket
-        self.ibm_api_key_id = ibm_api_key_id
-        self.ibm_service_instance_id = ibm_service_instance_id
-        self.s3_endpoint_url = s3_endpoint_url
-
-    def load_data(self, extra_info: Optional[dict] = None) -> List[Document]:
-        """Loads data from the specified S3 bucket."""
-        ibm_s3 = self._ibm_boto3.resource(
-            "s3",
-            ibm_api_key_id=self.ibm_api_key_id,
-            ibm_service_instance_id=self.ibm_service_instance_id,
-            config=self._boto_config(signature_version="oauth"),
-            endpoint_url=self.s3_endpoint_url,
-        )
-
-        bucket = ibm_s3.Bucket(self.bucket)
-
-        with tempfile.TemporaryDirectory() as temp_dir:
-            for obj in bucket.objects.filter(Prefix=""):
-                file_path = f"{temp_dir}/{obj.key}"
-                os.makedirs(os.path.dirname(file_path), exist_ok=True)
-                ibm_s3.meta.client.download_file(self.bucket, obj.key, file_path)
-
-            s3_source = re.sub(r"^(https?)://", "", self.s3_endpoint_url)
-
-            return DirectoryLoader(input_dir=temp_dir).load_data(extra_info={"source": f"{s3_source}/{self.bucket}"})

beekeeper/document_loaders/watson_discovery.py

@@ -1,121 +0,0 @@
-from datetime import datetime
-from logging import getLogger
-from typing import List, Optional
-
-from beekeeper.core.document import Document
-from beekeeper.core.document_loaders import BaseLoader
-
-logger = getLogger(__name__)
-
-
-class WatsonDiscoveryLoader(BaseLoader):
-    """Provides functionality to load documents from IBM Watson Discovery.
-
-    See https://cloud.ibm.com/docs/discovery-data?topic=discovery-data-getting-started for more info.
-
-    Args:
-        url (str): Watson Discovery instance url.
-        api_key (str): Watson Discovery API key.
-        project_id (str): Watson Discovery project_id.
-        version (str, optional): Watson Discovery API version. Defaults to ``2023-03-31``.
-        batch_size (int, optional): Batch size for bulk operations. Defaults to ``50``.
-        created_date (str, optional): Load documents created after the date. Expected format ``YYYY-MM-DD``. Defaults to ``datetime.today()``.
-        pre_additional_data_field (str, optional): Additional data field to be added to the beginning of the Document content. Defaults to ``None``.
-
-    **Example**
-
-    .. code-block:: python
-
-        from beekeeper.document_loaders import WatsonDiscoveryLoader
-
-        doc_loader = WatsonDiscoveryLoader(url="your_url",
-                                          api_key="your_api_key",
-                                          project_id="your_project_id")
-    """
-
-    def __init__(self,
-                 url: str,
-                 api_key: str,
-                 project_id: str,
-                 version: str = "2023-03-31",
-                 batch_size: int = 50,
-                 created_date: str = datetime.today().strftime("%Y-%m-%d"),
-                 pre_additional_data_field: str = None
-                 ) -> None:
-        try:
-            from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
-            from ibm_watson import DiscoveryV2
-
-        except ImportError:
-            raise ImportError("ibm-watson package not found, please install it with `pip install ibm-watson`")
-
-        self.project_id = project_id
-        self.batch_size = batch_size
-        self.created_date = created_date
-        self.pre_additional_data_field = pre_additional_data_field
-
-        try:
-            authenticator = IAMAuthenticator(api_key)
-            self._client = DiscoveryV2(authenticator=authenticator,
-                                       version=version)
-
-            self._client.set_service_url(url)
-        except Exception as e:
-            logger.error(f"Error connecting to IBM Watson Discovery: {e}")
-            raise
-
-    def load_data(self, extra_info: Optional[dict] = None) -> List[Document]:
-        """Loads documents from the Watson Discovery.
-
-        **Example**
-
-        .. code-block:: python
-
-            docs = doc_loader.load_data()
-        """
-        from ibm_watson.discovery_v2 import QueryLargePassages
-        last_batch_size = self.batch_size
-        offset_len = 0
-        documents = []
-        return_fields = ["extracted_metadata.filename", "extracted_metadata.file_type", "text"]
-
-        if self.pre_additional_data_field:
-            return_fields.append(self.pre_additional_data_field)
-
-        while last_batch_size == self.batch_size:
-            results = self._client.query(
-                project_id=self.project_id,
-                count=self.batch_size,
-                offset=offset_len,
-                return_=return_fields,
-                filter="extracted_metadata.publicationdate>={}".format(self.created_date),
-                passages=QueryLargePassages(enabled=False)).get_result()
-
-            last_batch_size = len(results["results"])
-            offset_len = offset_len + last_batch_size
-
-            # Make sure all retrieved document 'text' exist
-            results_documents = [doc for doc in results["results"] if "text" in doc]
-
-            if self.pre_additional_data_field:
-                for i, doc in enumerate(results_documents):
-                    doc["text"].insert(0, self._get_nested_value(doc, self.pre_additional_data_field))
-
-            documents.extend([Document(doc_id=doc["document_id"],
-                                       text="\n".join(doc["text"]),
-                                       metadata={"collection_id": doc["result_metadata"]["collection_id"]} | doc[
-                                           "extracted_metadata"])
-                              for doc in results_documents])
-
-        return documents
-
-    @staticmethod
-    def _get_nested_value(d, key_path, separator: Optional[str] = "."):
-        """Accesses a nested value in a dictionary using a string key path."""
-        keys = key_path.split(separator)  # Split the key_path using the separator
-        nested_value = d
-
-        for key in keys:
-            nested_value = nested_value[key]  # Traverse the dictionary by each key
-
-        return nested_value

beekeeper/embeddings/__init__.py

@@ -1,7 +0,0 @@
-from beekeeper.embeddings.huggingface import HuggingFaceEmbedding
-from beekeeper.embeddings.watsonx import WatsonxEmbedding
-
-__all__ = [
-    "HuggingFaceEmbedding",
-    "WatsonxEmbedding"
-]

beekeeper/embeddings/huggingface.py

@@ -1,66 +0,0 @@
-from typing import Any, List, Literal
-
-from pydantic.v1 import BaseModel, PrivateAttr
-
-from beekeeper.core.document import Document
-from beekeeper.core.embeddings import BaseEmbedding, Embedding
-
-
-class HuggingFaceEmbedding(BaseModel, BaseEmbedding):
-    """HuggingFace sentence_transformers embedding models.
-
-    Args:
-        model_name (str): Hugging Face model to be used. Defaults to ``sentence-transformers/all-MiniLM-L6-v2``.
-        device (str, optional): Device to run the model on. Currently supports "cpu" and "cuda". Defaults to ``cpu``.
-
-    **Example**
-
-    .. code-block:: python
-
-        from beekeeper.embeddings import HuggingFaceEmbedding
-
-        embedding = HuggingFaceEmbedding()
-    """
-
-    model_name: str = "sentence-transformers/all-MiniLM-L6-v2"
-    device: Literal["cpu", "cuda"] = "cpu"
-
-    _client: Any = PrivateAttr()
-
-    def __init__(self, **kwargs: Any) -> None:
-        super().__init__(**kwargs)
-        from sentence_transformers import SentenceTransformer
-
-        self._client = SentenceTransformer(self.model_name, device=self.device)
-
-    def get_query_embedding(self, query: str) -> Embedding:
-        """Compute embedding for a text.
-
-        Args:
-            query (str): Input query to compute embedding.
-
-        **Example**
-
-        .. code-block:: python
-
-            embedded_query = embedding.get_query_embedding("Beekeeper is a data framework to load any data in one line of code and connect with AI applications.")
-        """
-        return self.get_texts_embedding([query])[0]
-
-    def get_texts_embedding(self, texts: List[str]) -> List[Embedding]:
-        """Compute embeddings for list of texts.
-
-        Args:
-            texts (List[str]): List of text to compute embeddings.
-        """
-        return self._client.encode(texts).tolist()
-
-    def get_documents_embedding(self, documents: List[Document]) -> List[Embedding]:
-        """Compute embeddings for a list of documents.
-
-        Args:
-            documents (List[Document]): List of `Document` objects to compute embeddings.
-        """
-        texts = [document.get_content() for document in documents]
-
-        return self.get_texts_embedding(texts)

beekeeper/embeddings/watsonx.py

@@ -1,100 +0,0 @@
-from typing import Any, List, Optional
-
-from pydantic.v1 import BaseModel, PrivateAttr
-
-from beekeeper.core.document import Document
-from beekeeper.core.embeddings import BaseEmbedding, Embedding
-
-
-class WatsonxEmbedding(BaseModel, BaseEmbedding):
-    """IBM watsonx embedding models.
-
-    Note:
-            One of these parameters is required: ``project_id`` or ``space_id``. Not both.
-
-    See https://cloud.ibm.com/apidocs/watsonx-ai#endpoint-url for the watsonx.ai API endpoints.
-
-    Args:
-        model_name (str): IBM watsonx.ai model to be used. Defaults to ``ibm/slate-30m-english-rtrvr``.
-        api_key (str): watsonx API key.
-        url (str): watsonx instance url.
-        truncate_input_tokens (str): Maximum number of input tokens accepted. Defaults to ``512``
-        project_id (str, optional): watsonx project_id.
-        space_id (str, optional): watsonx space_id.
-
-    **Example**
-
-    .. code-block:: python
-
-        from beekeeper.embeddings import WatsonxEmbedding
-
-        watsonx_embedding = WatsonxEmbedding(api_key="your_api_key",
-                                         url="your_instance_url",
-                                         project_id="your_project_id")
-    """
-
-    model_name: str = "ibm/slate-30m-english-rtrvr"
-    api_key: str
-    url: str
-    truncate_input_tokens: int = 512
-    project_id: Optional[str] = None
-    space_id: Optional[str] = None
-
-    _client: Any = PrivateAttr()
-
-    def __init__(self, **kwargs: Any) -> None:
-        super().__init__(**kwargs)
-        try:
-            from ibm_watsonx_ai import Credentials
-            from ibm_watsonx_ai.foundation_models import Embeddings as WatsonxEmbeddings
-
-        except ImportError:
-            raise ImportError("ibm-watsonx-ai package not found, please install it with `pip install ibm-watsonx-ai`")
-
-        if (not (self.project_id or self.space_id)) or (self.project_id and self.space_id):
-            raise ValueError("Must provide one of these parameters [`project_id`, `space_id`], not both.")
-
-        kwargs_params = {
-            "model_id": self.model_name,
-            "params": {"truncate_input_tokens": self.truncate_input_tokens, "return_options": {"input_text": False}},
-            "credentials": Credentials(api_key=self.api_key, url=self.url)
-        }
-
-        if self.project_id:
-            kwargs_params["project_id"] = self.project_id
-        else:
-            kwargs_params["space_id"] = self.space_id
-
-        self._client = WatsonxEmbeddings(**kwargs_params)
-
-    def get_query_embedding(self, query: str) -> Embedding:
-        """Compute embedding for a text.
-
-        Args:
-            query (str): Input query to compute embedding.
-
-        **Example**
-
-        .. code-block:: python
-
-            embedded_query = watsonx_embedding.get_query_embedding("Beekeeper is a data framework to load any data in one line of code and connect with AI applications.")
-        """
-        return self.get_texts_embedding([query])[0]
-
-    def get_texts_embedding(self, texts: List[str]) -> List[Embedding]:
-        """Compute embeddings for list of texts.
-
-        Args:
-            texts (List[str]): List of text to compute embeddings.
-        """
-        return self._client.embed_documents(texts)
-
-    def get_documents_embedding(self, documents: List[Document]) -> List[Embedding]:
-        """Compute embeddings for a list of documents.
-
-        Args:
-            documents (List[Document]): List of `Document` objects to compute embeddings.
-        """
-        texts = [document.get_content() for document in documents]
-
-        return self.get_texts_embedding(texts)

beekeeper/evaluation/__init__.py

@@ -1,5 +0,0 @@
-from beekeeper.evaluation.knowledge_base_coverage import KnowledgeBaseCoverage
-
-__all__ = [
-    "KnowledgeBaseCoverage",
-]

beekeeper/evaluation/knowledge_base_coverage.py

@@ -1,62 +0,0 @@
-from typing import Dict, List, Literal
-
-import numpy as np
-from pydantic.v1 import BaseModel
-
-from beekeeper.core.embeddings import BaseEmbedding
-
-
-class KnowledgeBaseCoverage(BaseModel):
-    """Measures how much the knowledge base (context) has contributed to the answer’s coverage.
-    Higher value suggests greater proportion of context are in LLM response.
-
-    Args:
-        embed_model (BaseEmbedding):
-        similarity_mode (str, optional): Similarity strategy. Currently supports "cosine", "dot_product" and "euclidean". Defaults to ``cosine``.
-        similarity_threshold (float, optional): Embedding similarity threshold for "passing". Defaults to ``0.8``.
-
-     **Example**
-
-    .. code-block:: python
-
-        from beekeeper.embeddings import HuggingFaceEmbedding
-        from beekeeper.evaluation import KnowledgeBaseCoverage
-
-        embedding = HuggingFaceEmbedding()
-        eval_coverage = KnowledgeBaseCoverage(embed_model=embedding)
-    """
-
-    embed_model: BaseEmbedding
-    similarity_mode: Literal["cosine", "dot_product", "euclidean"] = "cosine"
-    similarity_threshold: float = 0.8
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    def compute_metric(self, contexts: List[str], candidate: str) -> Dict:
-        """
-        Args:
-            contexts (List[str]): List text used as LLM context.
-            candidate (str): LLM response based on given context.
-
-        **Example**
-
-        .. code-block:: python
-
-            context_coverage = eval_coverage.compute_metric(context=[], candidate="<candidate>")
-        """
-        if not contexts or not candidate:
-            raise ValueError("Must provide these parameters [`contexts`, `candidate`]")
-
-        coverage = {"contexts_score": [], "score": 0}
-        candidate_embedding = self.embed_model.get_query_embedding(candidate)
-
-        for context in contexts:
-            context_embedding = self.embed_model.get_query_embedding(context)
-            coverage["contexts_score"].append(
-                self.embed_model.similarity(candidate_embedding, context_embedding, mode=self.similarity_mode))
-
-        coverage["score"] = np.mean(coverage["contexts_score"])
-        coverage["passing"] = coverage["score"] >= self.similarity_threshold
-
-        return coverage

beekeeper/monitor/__init__.py

@@ -1,11 +0,0 @@
-from beekeeper.monitor.watsonx import (
-    CloudPakforDataCredentials,
-    WatsonxExternalPromptMonitoring,
-    WatsonxPromptMonitoring,
-)
-
-__all__ = [
-    "CloudPakforDataCredentials",
-    "WatsonxExternalPromptMonitoring",
-    "WatsonxPromptMonitoring"
-]