lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/git/gitextractor.py

@@ -0,0 +1,196 @@
+import os
+import shutil
+import tempfile
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+import aiofiles
+import git
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import MessageTextInput, Output
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+class GitExtractorComponent(Component):
+    display_name = "GitExtractor"
+    description = "Analyzes a Git repository and returns file contents and complete repository information"
+    icon = "GitLoader"
+
+    inputs = [
+        MessageTextInput(
+            name="repository_url",
+            display_name="Repository URL",
+            info="URL of the Git repository (e.g., https://github.com/username/repo)",
+            value="",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Text-Based File Contents",
+            name="text_based_file_contents",
+            method="get_text_based_file_contents",
+        ),
+        Output(display_name="Directory Structure", name="directory_structure", method="get_directory_structure"),
+        Output(display_name="Repository Info", name="repository_info", method="get_repository_info"),
+        Output(display_name="Statistics", name="statistics", method="get_statistics"),
+        Output(display_name="Files Content", name="files_content", method="get_files_content"),
+    ]
+
+    @asynccontextmanager
+    async def temp_git_repo(self):
+        """Async context manager for temporary git repository cloning."""
+        temp_dir = tempfile.mkdtemp()
+        try:
+            # Clone is still sync but wrapped in try/finally
+            git.Repo.clone_from(self.repository_url, temp_dir)
+            yield temp_dir
+        finally:
+            shutil.rmtree(temp_dir, ignore_errors=True)
+
+    async def get_repository_info(self) -> list[Data]:
+        try:
+            async with self.temp_git_repo() as temp_dir:
+                repo = git.Repo(temp_dir)
+                repo_info = {
+                    "name": self.repository_url.split("/")[-1],
+                    "url": self.repository_url,
+                    "default_branch": repo.active_branch.name,
+                    "remote_urls": [remote.url for remote in repo.remotes],
+                    "last_commit": {
+                        "hash": repo.head.commit.hexsha,
+                        "author": str(repo.head.commit.author),
+                        "message": repo.head.commit.message.strip(),
+                        "date": str(repo.head.commit.committed_datetime),
+                    },
+                    "branches": [str(branch) for branch in repo.branches],
+                }
+                result = [Data(data=repo_info)]
+                self.status = result
+                return result
+        except git.GitError as e:
+            error_result = [Data(data={"error": f"Error getting repository info: {e!s}"})]
+            self.status = error_result
+            return error_result
+
+    async def get_statistics(self) -> list[Data]:
+        try:
+            async with self.temp_git_repo() as temp_dir:
+                total_files = 0
+                total_size = 0
+                total_lines = 0
+                binary_files = 0
+                directories = 0
+
+                for root, dirs, files in os.walk(temp_dir):
+                    total_files += len(files)
+                    directories += len(dirs)
+                    for file in files:
+                        file_path = Path(root) / file
+                        total_size += file_path.stat().st_size
+                        try:
+                            async with aiofiles.open(file_path, encoding="utf-8") as f:
+                                total_lines += sum(1 for _ in await f.readlines())
+                        except UnicodeDecodeError:
+                            binary_files += 1
+
+                statistics = {
+                    "total_files": total_files,
+                    "total_size_bytes": total_size,
+                    "total_size_kb": round(total_size / 1024, 2),
+                    "total_size_mb": round(total_size / (1024 * 1024), 2),
+                    "total_lines": total_lines,
+                    "binary_files": binary_files,
+                    "directories": directories,
+                }
+                result = [Data(data=statistics)]
+                self.status = result
+                return result
+        except git.GitError as e:
+            error_result = [Data(data={"error": f"Error calculating statistics: {e!s}"})]
+            self.status = error_result
+            return error_result
+
+    async def get_directory_structure(self) -> Message:
+        try:
+            async with self.temp_git_repo() as temp_dir:
+                tree = ["Directory structure:"]
+                for root, _dirs, files in os.walk(temp_dir):
+                    level = root.replace(temp_dir, "").count(os.sep)
+                    indent = "    " * level
+                    if level == 0:
+                        tree.append(f"└── {Path(root).name}")
+                    else:
+                        tree.append(f"{indent}├── {Path(root).name}")
+                    subindent = "    " * (level + 1)
+                    tree.extend(f"{subindent}├── {f}" for f in files)
+                directory_structure = "\n".join(tree)
+                self.status = directory_structure
+                return Message(text=directory_structure)
+        except git.GitError as e:
+            error_message = f"Error getting directory structure: {e!s}"
+            self.status = error_message
+            return Message(text=error_message)
+
+    async def get_files_content(self) -> list[Data]:
+        try:
+            async with self.temp_git_repo() as temp_dir:
+                content_list = []
+                for root, _, files in os.walk(temp_dir):
+                    for file in files:
+                        file_path = Path(root) / file
+                        relative_path = file_path.relative_to(temp_dir)
+                        file_size = file_path.stat().st_size
+                        try:
+                            async with aiofiles.open(file_path, encoding="utf-8") as f:
+                                file_content = await f.read()
+                        except UnicodeDecodeError:
+                            file_content = "[BINARY FILE]"
+                        content_list.append(
+                            Data(data={"path": str(relative_path), "size": file_size, "content": file_content})
+                        )
+                self.status = content_list
+                return content_list
+        except git.GitError as e:
+            error_result = [Data(data={"error": f"Error getting files content: {e!s}"})]
+            self.status = error_result
+            return error_result
+
+    async def get_text_based_file_contents(self) -> Message:
+        try:
+            async with self.temp_git_repo() as temp_dir:
+                content_list = ["(Files content cropped to 300k characters, download full ingest to see more)"]
+                total_chars = 0
+                char_limit = 300000
+
+                for root, _, files in os.walk(temp_dir):
+                    for file in files:
+                        file_path = Path(root) / file
+                        relative_path = file_path.relative_to(temp_dir)
+                        content_list.extend(["=" * 50, f"File: /{relative_path}", "=" * 50])
+
+                        try:
+                            async with aiofiles.open(file_path, encoding="utf-8") as f:
+                                file_content = await f.read()
+                                if total_chars + len(file_content) > char_limit:
+                                    remaining_chars = char_limit - total_chars
+                                    file_content = file_content[:remaining_chars] + "\n... (content truncated)"
+                                content_list.append(file_content)
+                                total_chars += len(file_content)
+                        except UnicodeDecodeError:
+                            content_list.append("[BINARY FILE]")
+
+                        content_list.append("")
+
+                        if total_chars >= char_limit:
+                            break
+
+                text_content = "\n".join(content_list)
+                self.status = text_content
+                return Message(text=text_content)
+        except git.GitError as e:
+            error_message = f"Error getting text-based file contents: {e!s}"
+            self.status = error_message
+            return Message(text=error_message)

lfx/components/glean/__init__.py

@@ -0,0 +1,3 @@
+from .glean_search_api import GleanSearchAPISchema
+
+__all__ = ["GleanSearchAPISchema"]

lfx/components/glean/glean_search_api.py

@@ -0,0 +1,173 @@
+import json
+from typing import Any
+from urllib.parse import urljoin
+
+import httpx
+from langchain_core.tools import StructuredTool, ToolException
+from pydantic import BaseModel
+from pydantic.v1 import Field
+
+from lfx.base.langchain_utilities.model import LCToolComponent
+from lfx.field_typing import Tool
+from lfx.inputs.inputs import IntInput, MultilineInput, NestedDictInput, SecretStrInput, StrInput
+from lfx.io import Output
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+
+
+class GleanSearchAPISchema(BaseModel):
+    query: str = Field(..., description="The search query")
+    page_size: int = Field(10, description="Maximum number of results to return")
+    request_options: dict[str, Any] | None = Field(default_factory=dict, description="Request Options")
+
+
+class GleanAPIWrapper(BaseModel):
+    """Wrapper around Glean API."""
+
+    glean_api_url: str
+    glean_access_token: str
+    act_as: str = "langflow-component@datastax.com"  # TODO: Detect this
+
+    def _prepare_request(
+        self,
+        query: str,
+        page_size: int = 10,
+        request_options: dict[str, Any] | None = None,
+    ) -> dict:
+        # Ensure there's a trailing slash
+        url = self.glean_api_url
+        if not url.endswith("/"):
+            url += "/"
+
+        return {
+            "url": urljoin(url, "search"),
+            "headers": {
+                "Authorization": f"Bearer {self.glean_access_token}",
+                "X-Scio-ActAs": self.act_as,
+            },
+            "payload": {
+                "query": query,
+                "pageSize": page_size,
+                "requestOptions": request_options,
+            },
+        }
+
+    def results(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
+        results = self._search_api_results(query, **kwargs)
+
+        if len(results) == 0:
+            msg = "No good Glean Search Result was found"
+            raise AssertionError(msg)
+
+        return results
+
+    def run(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
+        try:
+            results = self.results(query, **kwargs)
+
+            processed_results = []
+            for result in results:
+                if "title" in result:
+                    result["snippets"] = result.get("snippets", [{"snippet": {"text": result["title"]}}])
+                    if "text" not in result["snippets"][0]:
+                        result["snippets"][0]["text"] = result["title"]
+
+                processed_results.append(result)
+        except Exception as e:
+            error_message = f"Error in Glean Search API: {e!s}"
+            raise ToolException(error_message) from e
+
+        return processed_results
+
+    def _search_api_results(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
+        request_details = self._prepare_request(query, **kwargs)
+
+        response = httpx.post(
+            request_details["url"],
+            json=request_details["payload"],
+            headers=request_details["headers"],
+        )
+
+        response.raise_for_status()
+        response_json = response.json()
+
+        return response_json.get("results", [])
+
+    @staticmethod
+    def _result_as_string(result: dict) -> str:
+        return json.dumps(result, indent=4)
+
+
+class GleanSearchAPIComponent(LCToolComponent):
+    display_name: str = "Glean Search API"
+    description: str = "Search using Glean's API."
+    documentation: str = "https://docs.langflow.org/Components/components-tools#glean-search-api"
+    icon: str = "Glean"
+
+    outputs = [
+        Output(display_name="DataFrame", name="dataframe", method="fetch_content_dataframe"),
+    ]
+
+    inputs = [
+        StrInput(name="glean_api_url", display_name="Glean API URL", required=True),
+        SecretStrInput(name="glean_access_token", display_name="Glean Access Token", required=True),
+        MultilineInput(name="query", display_name="Query", required=True, tool_mode=True),
+        IntInput(name="page_size", display_name="Page Size", value=10),
+        NestedDictInput(name="request_options", display_name="Request Options", required=False),
+    ]
+
+    def build_tool(self) -> Tool:
+        wrapper = self._build_wrapper(
+            glean_api_url=self.glean_api_url,
+            glean_access_token=self.glean_access_token,
+        )
+
+        tool = StructuredTool.from_function(
+            name="glean_search_api",
+            description="Search Glean for relevant results.",
+            func=wrapper.run,
+            args_schema=GleanSearchAPISchema,
+        )
+
+        self.status = "Glean Search API Tool for Langchain"
+
+        return tool
+
+    def run_model(self) -> DataFrame:
+        return self.fetch_content_dataframe()
+
+    def fetch_content(self) -> list[Data]:
+        tool = self.build_tool()
+
+        results = tool.run(
+            {
+                "query": self.query,
+                "page_size": self.page_size,
+                "request_options": self.request_options,
+            }
+        )
+
+        # Build the data
+        data = [Data(data=result, text=result["snippets"][0]["text"]) for result in results]
+        self.status = data  # type: ignore[assignment]
+
+        return data
+
+    def _build_wrapper(
+        self,
+        glean_api_url: str,
+        glean_access_token: str,
+    ):
+        return GleanAPIWrapper(
+            glean_api_url=glean_api_url,
+            glean_access_token=glean_access_token,
+        )
+
+    def fetch_content_dataframe(self) -> DataFrame:
+        """Convert the Glean search results to a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame containing the search results.
+        """
+        data = self.fetch_content()
+        return DataFrame(data)

lfx/components/google/__init__.py

@@ -0,0 +1,17 @@
+from .gmail import GmailLoaderComponent
+from .google_bq_sql_executor import BigQueryExecutorComponent
+from .google_drive import GoogleDriveComponent
+from .google_drive_search import GoogleDriveSearchComponent
+from .google_generative_ai import GoogleGenerativeAIComponent
+from .google_generative_ai_embeddings import GoogleGenerativeAIEmbeddingsComponent
+from .google_oauth_token import GoogleOAuthToken
+
+__all__ = [
+    "BigQueryExecutorComponent",
+    "GmailLoaderComponent",
+    "GoogleDriveComponent",
+    "GoogleDriveSearchComponent",
+    "GoogleGenerativeAIComponent",
+    "GoogleGenerativeAIEmbeddingsComponent",
+    "GoogleOAuthToken",
+]

lfx/components/google/gmail.py

@@ -0,0 +1,192 @@
+import base64
+import json
+import re
+from collections.abc import Iterator
+from json.decoder import JSONDecodeError
+from typing import Any
+
+from google.auth.exceptions import RefreshError
+from google.oauth2.credentials import Credentials
+from googleapiclient.discovery import build
+from langchain_core.chat_sessions import ChatSession
+from langchain_core.messages import HumanMessage
+from langchain_google_community.gmail.loader import GMailLoader
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import MessageTextInput
+from lfx.io import SecretStrInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.template.field.base import Output
+
+
+class GmailLoaderComponent(Component):
+    display_name = "Gmail Loader"
+    description = "Loads emails from Gmail using provided credentials."
+    icon = "Google"
+    legacy: bool = True
+
+    inputs = [
+        SecretStrInput(
+            name="json_string",
+            display_name="JSON String of the Service Account Token",
+            info="JSON string containing OAuth 2.0 access token information for service account access",
+            required=True,
+            value="""{
+                "account": "",
+                "client_id": "",
+                "client_secret": "",
+                "expiry": "",
+                "refresh_token": "",
+                "scopes": [
+                    "https://www.googleapis.com/auth/gmail.readonly",
+                ],
+                "token": "",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "universe_domain": "googleapis.com"
+            }""",
+        ),
+        MessageTextInput(
+            name="label_ids",
+            display_name="Label IDs",
+            info="Comma-separated list of label IDs to filter emails.",
+            required=True,
+            value="INBOX,SENT,UNREAD,IMPORTANT",
+        ),
+        MessageTextInput(
+            name="max_results",
+            display_name="Max Results",
+            info="Maximum number of emails to load.",
+            required=True,
+            value="10",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="load_emails"),
+    ]
+
+    def load_emails(self) -> Data:
+        class CustomGMailLoader(GMailLoader):
+            def __init__(
+                self, creds: Any, *, n: int = 100, label_ids: list[str] | None = None, raise_error: bool = False
+            ) -> None:
+                super().__init__(creds, n, raise_error)
+                self.label_ids = label_ids if label_ids is not None else ["SENT"]
+
+            def clean_message_content(self, message):
+                # Remove URLs
+                message = re.sub(r"http\S+|www\S+|https\S+", "", message, flags=re.MULTILINE)
+
+                # Remove email addresses
+                message = re.sub(r"\S+@\S+", "", message)
+
+                # Remove special characters and excessive whitespace
+                message = re.sub(r"[^A-Za-z0-9\s]+", " ", message)
+                message = re.sub(r"\s{2,}", " ", message)
+
+                # Trim leading and trailing whitespace
+                return message.strip()
+
+            def _extract_email_content(self, msg: Any) -> HumanMessage:
+                from_email = None
+                for values in msg["payload"]["headers"]:
+                    name = values["name"]
+                    if name == "From":
+                        from_email = values["value"]
+                if from_email is None:
+                    msg = "From email not found."
+                    raise ValueError(msg)
+
+                parts = msg["payload"]["parts"] if "parts" in msg["payload"] else [msg["payload"]]
+
+                for part in parts:
+                    if part["mimeType"] == "text/plain":
+                        data = part["body"]["data"]
+                        data = base64.urlsafe_b64decode(data).decode("utf-8")
+                        pattern = re.compile(r"\r\nOn .+(\r\n)*wrote:\r\n")
+                        newest_response = re.split(pattern, data)[0]
+                        return HumanMessage(
+                            content=self.clean_message_content(newest_response),
+                            additional_kwargs={"sender": from_email},
+                        )
+                msg = "No plain text part found in the email."
+                raise ValueError(msg)
+
+            def _get_message_data(self, service: Any, message: Any) -> ChatSession:
+                msg = service.users().messages().get(userId="me", id=message["id"]).execute()
+                message_content = self._extract_email_content(msg)
+
+                in_reply_to = None
+                email_data = msg["payload"]["headers"]
+                for values in email_data:
+                    name = values["name"]
+                    if name == "In-Reply-To":
+                        in_reply_to = values["value"]
+
+                thread_id = msg["threadId"]
+
+                if in_reply_to:
+                    thread = service.users().threads().get(userId="me", id=thread_id).execute()
+                    messages = thread["messages"]
+
+                    response_email = None
+                    for _message in messages:
+                        email_data = _message["payload"]["headers"]
+                        for values in email_data:
+                            if values["name"] == "Message-ID":
+                                message_id = values["value"]
+                                if message_id == in_reply_to:
+                                    response_email = _message
+                    if response_email is None:
+                        msg = "Response email not found in the thread."
+                        raise ValueError(msg)
+                    starter_content = self._extract_email_content(response_email)
+                    return ChatSession(messages=[starter_content, message_content])
+                return ChatSession(messages=[message_content])
+
+            def lazy_load(self) -> Iterator[ChatSession]:
+                service = build("gmail", "v1", credentials=self.creds)
+                results = (
+                    service.users().messages().list(userId="me", labelIds=self.label_ids, maxResults=self.n).execute()
+                )
+                messages = results.get("messages", [])
+                if not messages:
+                    logger.warning("No messages found with the specified labels.")
+                for message in messages:
+                    try:
+                        yield self._get_message_data(service, message)
+                    except Exception:
+                        if self.raise_error:
+                            raise
+                        else:
+                            logger.exception(f"Error processing message {message['id']}")
+
+        json_string = self.json_string
+        label_ids = self.label_ids.split(",") if self.label_ids else ["INBOX"]
+        max_results = int(self.max_results) if self.max_results else 100
+
+        # Load the token information from the JSON string
+        try:
+            token_info = json.loads(json_string)
+        except JSONDecodeError as e:
+            msg = "Invalid JSON string"
+            raise ValueError(msg) from e
+
+        creds = Credentials.from_authorized_user_info(token_info)
+
+        # Initialize the custom loader with the provided credentials
+        loader = CustomGMailLoader(creds=creds, n=max_results, label_ids=label_ids)
+
+        try:
+            docs = loader.load()
+        except RefreshError as e:
+            msg = "Authentication error: Unable to refresh authentication token. Please try to reauthenticate."
+            raise ValueError(msg) from e
+        except Exception as e:
+            msg = f"Error loading documents: {e}"
+            raise ValueError(msg) from e
+
+        # Return the loaded documents
+        self.status = docs
+        return Data(data={"text": docs})