langflow-base-nightly 0.5.1.dev3__py3-none-any.whl → 0.5.1.dev5__py3-none-any.whl

langflow/components/git/git.py

@@ -1,262 +0,0 @@
-import re
-import tempfile
-from contextlib import asynccontextmanager
-from fnmatch import fnmatch
-from pathlib import Path
-
-import anyio
-from langchain_community.document_loaders.git import GitLoader
-
-from langflow.custom.custom_component.component import Component
-from langflow.io import DropdownInput, MessageTextInput, Output
-from langflow.schema.data import Data
-
-
-class GitLoaderComponent(Component):
-    display_name = "Git"
-    description = (
-        "Load and filter documents from a local or remote Git repository. "
-        "Use a local repo path or clone from a remote URL."
-    )
-    trace_type = "tool"
-    icon = "GitLoader"
-
-    inputs = [
-        DropdownInput(
-            name="repo_source",
-            display_name="Repository Source",
-            options=["Local", "Remote"],
-            required=True,
-            info="Select whether to use a local repo path or clone from a remote URL.",
-            real_time_refresh=True,
-        ),
-        MessageTextInput(
-            name="repo_path",
-            display_name="Local Repository Path",
-            required=False,
-            info="The local path to the existing Git repository (used if 'Local' is selected).",
-            dynamic=True,
-            show=False,
-        ),
-        MessageTextInput(
-            name="clone_url",
-            display_name="Clone URL",
-            required=False,
-            info="The URL of the Git repository to clone (used if 'Clone' is selected).",
-            dynamic=True,
-            show=False,
-        ),
-        MessageTextInput(
-            name="branch",
-            display_name="Branch",
-            required=False,
-            value="main",
-            info="The branch to load files from. Defaults to 'main'.",
-        ),
-        MessageTextInput(
-            name="file_filter",
-            display_name="File Filter",
-            required=False,
-            advanced=True,
-            info=(
-                "Patterns to filter files. For example:\n"
-                "Include only .py files: '*.py'\n"
-                "Exclude .py files: '!*.py'\n"
-                "Multiple patterns can be separated by commas."
-            ),
-        ),
-        MessageTextInput(
-            name="content_filter",
-            display_name="Content Filter",
-            required=False,
-            advanced=True,
-            info="A regex pattern to filter files based on their content.",
-        ),
-    ]
-
-    outputs = [
-        Output(name="data", display_name="Data", method="load_documents"),
-    ]
-
-    @staticmethod
-    def is_binary(file_path: str | Path) -> bool:
-        """Check if a file is binary by looking for null bytes."""
-        try:
-            with Path(file_path).open("rb") as file:
-                content = file.read(1024)
-                return b"\x00" in content
-        except Exception:  # noqa: BLE001
-            return True
-
-    @staticmethod
-    def check_file_patterns(file_path: str | Path, patterns: str) -> bool:
-        """Check if a file matches the given patterns.
-
-        Args:
-            file_path: Path to the file to check
-            patterns: Comma-separated list of glob patterns
-
-        Returns:
-            bool: True if file should be included, False if excluded
-        """
-        # Handle empty or whitespace-only patterns
-        if not patterns or patterns.isspace():
-            return True
-
-        path_str = str(file_path)
-        file_name = Path(path_str).name
-        pattern_list: list[str] = [pattern.strip() for pattern in patterns.split(",") if pattern.strip()]
-
-        # If no valid patterns after stripping, treat as include all
-        if not pattern_list:
-            return True
-
-        # Process exclusion patterns first
-        for pattern in pattern_list:
-            if pattern.startswith("!"):
-                # For exclusions, match against both full path and filename
-                exclude_pattern = pattern[1:]
-                if fnmatch(path_str, exclude_pattern) or fnmatch(file_name, exclude_pattern):
-                    return False
-
-        # Then check inclusion patterns
-        include_patterns = [p for p in pattern_list if not p.startswith("!")]
-        # If no include patterns, treat as include all
-        if not include_patterns:
-            return True
-
-        # For inclusions, match against both full path and filename
-        return any(fnmatch(path_str, pattern) or fnmatch(file_name, pattern) for pattern in include_patterns)
-
-    @staticmethod
-    def check_content_pattern(file_path: str | Path, pattern: str) -> bool:
-        """Check if file content matches the given regex pattern.
-
-        Args:
-            file_path: Path to the file to check
-            pattern: Regex pattern to match against content
-
-        Returns:
-            bool: True if content matches, False otherwise
-        """
-        try:
-            # Check if file is binary
-            with Path(file_path).open("rb") as file:
-                content = file.read(1024)
-                if b"\x00" in content:
-                    return False
-
-            # Try to compile the regex pattern first
-            try:
-                # Use the MULTILINE flag to better handle text content
-                content_regex = re.compile(pattern, re.MULTILINE)
-                # Test the pattern with a simple string to catch syntax errors
-                test_str = "test\nstring"
-                if not content_regex.search(test_str):
-                    # Pattern is valid but doesn't match test string
-                    pass
-            except (re.error, TypeError, ValueError):
-                return False
-
-            # If not binary and regex is valid, check content
-            with Path(file_path).open(encoding="utf-8") as file:
-                file_content = file.read()
-            return bool(content_regex.search(file_content))
-        except (OSError, UnicodeDecodeError):
-            return False
-
-    def build_combined_filter(self, file_filter_patterns: str | None = None, content_filter_pattern: str | None = None):
-        """Build a combined filter function from file and content patterns.
-
-        Args:
-            file_filter_patterns: Comma-separated glob patterns
-            content_filter_pattern: Regex pattern for content
-
-        Returns:
-            callable: Filter function that takes a file path and returns bool
-        """
-
-        def combined_filter(file_path: str) -> bool:
-            try:
-                path = Path(file_path)
-
-                # Check if file exists and is readable
-                if not path.exists():
-                    return False
-
-                # Check if file is binary
-                if self.is_binary(path):
-                    return False
-
-                # Apply file pattern filters
-                if file_filter_patterns and not self.check_file_patterns(path, file_filter_patterns):
-                    return False
-
-                # Apply content filter
-                return not (content_filter_pattern and not self.check_content_pattern(path, content_filter_pattern))
-            except Exception:  # noqa: BLE001
-                return False
-
-        return combined_filter
-
-    @asynccontextmanager
-    async def temp_clone_dir(self):
-        """Context manager for handling temporary clone directory."""
-        temp_dir = None
-        try:
-            temp_dir = tempfile.mkdtemp(prefix="langflow_clone_")
-            yield temp_dir
-        finally:
-            if temp_dir:
-                await anyio.Path(temp_dir).rmdir()
-
-    def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:
-        # Hide fields by default
-        build_config["repo_path"]["show"] = False
-        build_config["clone_url"]["show"] = False
-
-        if field_name == "repo_source":
-            if field_value == "Local":
-                build_config["repo_path"]["show"] = True
-                build_config["repo_path"]["required"] = True
-                build_config["clone_url"]["required"] = False
-            elif field_value == "Remote":
-                build_config["clone_url"]["show"] = True
-                build_config["clone_url"]["required"] = True
-                build_config["repo_path"]["required"] = False
-
-        return build_config
-
-    async def build_gitloader(self) -> GitLoader:
-        file_filter_patterns = getattr(self, "file_filter", None)
-        content_filter_pattern = getattr(self, "content_filter", None)
-
-        combined_filter = self.build_combined_filter(file_filter_patterns, content_filter_pattern)
-
-        repo_source = getattr(self, "repo_source", None)
-        if repo_source == "Local":
-            repo_path = self.repo_path
-            clone_url = None
-        else:
-            # Clone source
-            clone_url = self.clone_url
-            async with self.temp_clone_dir() as temp_dir:
-                repo_path = temp_dir
-
-        # Only pass branch if it's explicitly set
-        branch = getattr(self, "branch", None)
-        if not branch:
-            branch = None
-
-        return GitLoader(
-            repo_path=repo_path,
-            clone_url=clone_url if repo_source == "Remote" else None,
-            branch=branch,
-            file_filter=combined_filter,
-        )
-
-    async def load_documents(self) -> list[Data]:
-        gitloader = await self.build_gitloader()
-        data = [Data.from_document(doc) async for doc in gitloader.alazy_load()]
-        self.status = data
-        return data

langflow/components/git/gitextractor.py

@@ -1,196 +0,0 @@
-import os
-import shutil
-import tempfile
-from contextlib import asynccontextmanager
-from pathlib import Path
-
-import aiofiles
-import git
-
-from langflow.custom.custom_component.component import Component
-from langflow.io import MessageTextInput, Output
-from langflow.schema.data import Data
-from langflow.schema.message import Message
-
-
-class GitExtractorComponent(Component):
-    display_name = "GitExtractor"
-    description = "Analyzes a Git repository and returns file contents and complete repository information"
-    icon = "GitLoader"
-
-    inputs = [
-        MessageTextInput(
-            name="repository_url",
-            display_name="Repository URL",
-            info="URL of the Git repository (e.g., https://github.com/username/repo)",
-            value="",
-        ),
-    ]
-
-    outputs = [
-        Output(
-            display_name="Text-Based File Contents",
-            name="text_based_file_contents",
-            method="get_text_based_file_contents",
-        ),
-        Output(display_name="Directory Structure", name="directory_structure", method="get_directory_structure"),
-        Output(display_name="Repository Info", name="repository_info", method="get_repository_info"),
-        Output(display_name="Statistics", name="statistics", method="get_statistics"),
-        Output(display_name="Files Content", name="files_content", method="get_files_content"),
-    ]
-
-    @asynccontextmanager
-    async def temp_git_repo(self):
-        """Async context manager for temporary git repository cloning."""
-        temp_dir = tempfile.mkdtemp()
-        try:
-            # Clone is still sync but wrapped in try/finally
-            git.Repo.clone_from(self.repository_url, temp_dir)
-            yield temp_dir
-        finally:
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-    async def get_repository_info(self) -> list[Data]:
-        try:
-            async with self.temp_git_repo() as temp_dir:
-                repo = git.Repo(temp_dir)
-                repo_info = {
-                    "name": self.repository_url.split("/")[-1],
-                    "url": self.repository_url,
-                    "default_branch": repo.active_branch.name,
-                    "remote_urls": [remote.url for remote in repo.remotes],
-                    "last_commit": {
-                        "hash": repo.head.commit.hexsha,
-                        "author": str(repo.head.commit.author),
-                        "message": repo.head.commit.message.strip(),
-                        "date": str(repo.head.commit.committed_datetime),
-                    },
-                    "branches": [str(branch) for branch in repo.branches],
-                }
-                result = [Data(data=repo_info)]
-                self.status = result
-                return result
-        except git.GitError as e:
-            error_result = [Data(data={"error": f"Error getting repository info: {e!s}"})]
-            self.status = error_result
-            return error_result
-
-    async def get_statistics(self) -> list[Data]:
-        try:
-            async with self.temp_git_repo() as temp_dir:
-                total_files = 0
-                total_size = 0
-                total_lines = 0
-                binary_files = 0
-                directories = 0
-
-                for root, dirs, files in os.walk(temp_dir):
-                    total_files += len(files)
-                    directories += len(dirs)
-                    for file in files:
-                        file_path = Path(root) / file
-                        total_size += file_path.stat().st_size
-                        try:
-                            async with aiofiles.open(file_path, encoding="utf-8") as f:
-                                total_lines += sum(1 for _ in await f.readlines())
-                        except UnicodeDecodeError:
-                            binary_files += 1
-
-                statistics = {
-                    "total_files": total_files,
-                    "total_size_bytes": total_size,
-                    "total_size_kb": round(total_size / 1024, 2),
-                    "total_size_mb": round(total_size / (1024 * 1024), 2),
-                    "total_lines": total_lines,
-                    "binary_files": binary_files,
-                    "directories": directories,
-                }
-                result = [Data(data=statistics)]
-                self.status = result
-                return result
-        except git.GitError as e:
-            error_result = [Data(data={"error": f"Error calculating statistics: {e!s}"})]
-            self.status = error_result
-            return error_result
-
-    async def get_directory_structure(self) -> Message:
-        try:
-            async with self.temp_git_repo() as temp_dir:
-                tree = ["Directory structure:"]
-                for root, _dirs, files in os.walk(temp_dir):
-                    level = root.replace(temp_dir, "").count(os.sep)
-                    indent = "    " * level
-                    if level == 0:
-                        tree.append(f"└── {Path(root).name}")
-                    else:
-                        tree.append(f"{indent}├── {Path(root).name}")
-                    subindent = "    " * (level + 1)
-                    tree.extend(f"{subindent}├── {f}" for f in files)
-                directory_structure = "\n".join(tree)
-                self.status = directory_structure
-                return Message(text=directory_structure)
-        except git.GitError as e:
-            error_message = f"Error getting directory structure: {e!s}"
-            self.status = error_message
-            return Message(text=error_message)
-
-    async def get_files_content(self) -> list[Data]:
-        try:
-            async with self.temp_git_repo() as temp_dir:
-                content_list = []
-                for root, _, files in os.walk(temp_dir):
-                    for file in files:
-                        file_path = Path(root) / file
-                        relative_path = file_path.relative_to(temp_dir)
-                        file_size = file_path.stat().st_size
-                        try:
-                            async with aiofiles.open(file_path, encoding="utf-8") as f:
-                                file_content = await f.read()
-                        except UnicodeDecodeError:
-                            file_content = "[BINARY FILE]"
-                        content_list.append(
-                            Data(data={"path": str(relative_path), "size": file_size, "content": file_content})
-                        )
-                self.status = content_list
-                return content_list
-        except git.GitError as e:
-            error_result = [Data(data={"error": f"Error getting files content: {e!s}"})]
-            self.status = error_result
-            return error_result
-
-    async def get_text_based_file_contents(self) -> Message:
-        try:
-            async with self.temp_git_repo() as temp_dir:
-                content_list = ["(Files content cropped to 300k characters, download full ingest to see more)"]
-                total_chars = 0
-                char_limit = 300000
-
-                for root, _, files in os.walk(temp_dir):
-                    for file in files:
-                        file_path = Path(root) / file
-                        relative_path = file_path.relative_to(temp_dir)
-                        content_list.extend(["=" * 50, f"File: /{relative_path}", "=" * 50])
-
-                        try:
-                            async with aiofiles.open(file_path, encoding="utf-8") as f:
-                                file_content = await f.read()
-                                if total_chars + len(file_content) > char_limit:
-                                    remaining_chars = char_limit - total_chars
-                                    file_content = file_content[:remaining_chars] + "\n... (content truncated)"
-                                content_list.append(file_content)
-                                total_chars += len(file_content)
-                        except UnicodeDecodeError:
-                            content_list.append("[BINARY FILE]")
-
-                        content_list.append("")
-
-                        if total_chars >= char_limit:
-                            break
-
-                text_content = "\n".join(content_list)
-                self.status = text_content
-                return Message(text=text_content)
-        except git.GitError as e:
-            error_message = f"Error getting text-based file contents: {e!s}"
-            self.status = error_message
-            return Message(text=error_message)

langflow/components/glean/__init__.py

@@ -1,3 +0,0 @@
-from .glean_search_api import GleanSearchAPISchema
-
-__all__ = ["GleanSearchAPISchema"]

langflow/components/glean/glean_search_api.py

@@ -1,173 +0,0 @@
-import json
-from typing import Any
-from urllib.parse import urljoin
-
-import httpx
-from langchain_core.tools import StructuredTool, ToolException
-from pydantic import BaseModel
-from pydantic.v1 import Field
-
-from langflow.base.langchain_utilities.model import LCToolComponent
-from langflow.field_typing import Tool
-from langflow.inputs.inputs import IntInput, MultilineInput, NestedDictInput, SecretStrInput, StrInput
-from langflow.io import Output
-from langflow.schema.data import Data
-from langflow.schema.dataframe import DataFrame
-
-
-class GleanSearchAPISchema(BaseModel):
-    query: str = Field(..., description="The search query")
-    page_size: int = Field(10, description="Maximum number of results to return")
-    request_options: dict[str, Any] | None = Field(default_factory=dict, description="Request Options")
-
-
-class GleanAPIWrapper(BaseModel):
-    """Wrapper around Glean API."""
-
-    glean_api_url: str
-    glean_access_token: str
-    act_as: str = "langflow-component@datastax.com"  # TODO: Detect this
-
-    def _prepare_request(
-        self,
-        query: str,
-        page_size: int = 10,
-        request_options: dict[str, Any] | None = None,
-    ) -> dict:
-        # Ensure there's a trailing slash
-        url = self.glean_api_url
-        if not url.endswith("/"):
-            url += "/"
-
-        return {
-            "url": urljoin(url, "search"),
-            "headers": {
-                "Authorization": f"Bearer {self.glean_access_token}",
-                "X-Scio-ActAs": self.act_as,
-            },
-            "payload": {
-                "query": query,
-                "pageSize": page_size,
-                "requestOptions": request_options,
-            },
-        }
-
-    def results(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
-        results = self._search_api_results(query, **kwargs)
-
-        if len(results) == 0:
-            msg = "No good Glean Search Result was found"
-            raise AssertionError(msg)
-
-        return results
-
-    def run(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
-        try:
-            results = self.results(query, **kwargs)
-
-            processed_results = []
-            for result in results:
-                if "title" in result:
-                    result["snippets"] = result.get("snippets", [{"snippet": {"text": result["title"]}}])
-                    if "text" not in result["snippets"][0]:
-                        result["snippets"][0]["text"] = result["title"]
-
-                processed_results.append(result)
-        except Exception as e:
-            error_message = f"Error in Glean Search API: {e!s}"
-            raise ToolException(error_message) from e
-
-        return processed_results
-
-    def _search_api_results(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
-        request_details = self._prepare_request(query, **kwargs)
-
-        response = httpx.post(
-            request_details["url"],
-            json=request_details["payload"],
-            headers=request_details["headers"],
-        )
-
-        response.raise_for_status()
-        response_json = response.json()
-
-        return response_json.get("results", [])
-
-    @staticmethod
-    def _result_as_string(result: dict) -> str:
-        return json.dumps(result, indent=4)
-
-
-class GleanSearchAPIComponent(LCToolComponent):
-    display_name: str = "Glean Search API"
-    description: str = "Search using Glean's API."
-    documentation: str = "https://docs.langflow.org/Components/components-tools#glean-search-api"
-    icon: str = "Glean"
-
-    outputs = [
-        Output(display_name="DataFrame", name="dataframe", method="fetch_content_dataframe"),
-    ]
-
-    inputs = [
-        StrInput(name="glean_api_url", display_name="Glean API URL", required=True),
-        SecretStrInput(name="glean_access_token", display_name="Glean Access Token", required=True),
-        MultilineInput(name="query", display_name="Query", required=True, tool_mode=True),
-        IntInput(name="page_size", display_name="Page Size", value=10),
-        NestedDictInput(name="request_options", display_name="Request Options", required=False),
-    ]
-
-    def build_tool(self) -> Tool:
-        wrapper = self._build_wrapper(
-            glean_api_url=self.glean_api_url,
-            glean_access_token=self.glean_access_token,
-        )
-
-        tool = StructuredTool.from_function(
-            name="glean_search_api",
-            description="Search Glean for relevant results.",
-            func=wrapper.run,
-            args_schema=GleanSearchAPISchema,
-        )
-
-        self.status = "Glean Search API Tool for Langchain"
-
-        return tool
-
-    def run_model(self) -> DataFrame:
-        return self.fetch_content_dataframe()
-
-    def fetch_content(self) -> list[Data]:
-        tool = self.build_tool()
-
-        results = tool.run(
-            {
-                "query": self.query,
-                "page_size": self.page_size,
-                "request_options": self.request_options,
-            }
-        )
-
-        # Build the data
-        data = [Data(data=result, text=result["snippets"][0]["text"]) for result in results]
-        self.status = data  # type: ignore[assignment]
-
-        return data
-
-    def _build_wrapper(
-        self,
-        glean_api_url: str,
-        glean_access_token: str,
-    ):
-        return GleanAPIWrapper(
-            glean_api_url=glean_api_url,
-            glean_access_token=glean_access_token,
-        )
-
-    def fetch_content_dataframe(self) -> DataFrame:
-        """Convert the Glean search results to a DataFrame.
-
-        Returns:
-            DataFrame: A DataFrame containing the search results.
-        """
-        data = self.fetch_content()
-        return DataFrame(data)

langflow/components/google/__init__.py

@@ -1,17 +0,0 @@
-from .gmail import GmailLoaderComponent
-from .google_bq_sql_executor import BigQueryExecutorComponent
-from .google_drive import GoogleDriveComponent
-from .google_drive_search import GoogleDriveSearchComponent
-from .google_generative_ai import GoogleGenerativeAIComponent
-from .google_generative_ai_embeddings import GoogleGenerativeAIEmbeddingsComponent
-from .google_oauth_token import GoogleOAuthToken
-
-__all__ = [
-    "BigQueryExecutorComponent",
-    "GmailLoaderComponent",
-    "GoogleDriveComponent",
-    "GoogleDriveSearchComponent",
-    "GoogleGenerativeAIComponent",
-    "GoogleGenerativeAIEmbeddingsComponent",
-    "GoogleOAuthToken",
-]