lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/apify/apify_actor.py

@@ -0,0 +1,325 @@
+import json
+import string
+from typing import Any, cast
+
+from apify_client import ApifyClient
+from langchain_community.document_loaders.apify_dataset import ApifyDatasetLoader
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field, field_serializer
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing import Tool
+from lfx.inputs.inputs import BoolInput
+from lfx.io import MultilineInput, Output, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+MAX_DESCRIPTION_LEN = 250
+
+
+class ApifyActorsComponent(Component):
+    display_name = "Apify Actors"
+    description = (
+        "Use Apify Actors to extract data from hundreds of places fast. "
+        "This component can be used in a flow to retrieve data or as a tool with an agent."
+    )
+    documentation: str = "https://docs.langflow.org/bundles-apify"
+    icon = "Apify"
+    name = "ApifyActors"
+
+    inputs = [
+        SecretStrInput(
+            name="apify_token",
+            display_name="Apify Token",
+            info="The API token for the Apify account.",
+            required=True,
+            password=True,
+        ),
+        StrInput(
+            name="actor_id",
+            display_name="Actor",
+            info=(
+                "Actor name from Apify store to run. For example 'apify/website-content-crawler' "
+                "to use the Website Content Crawler Actor."
+            ),
+            value="apify/website-content-crawler",
+            required=True,
+        ),
+        # multiline input is more pleasant to use than the nested dict input
+        MultilineInput(
+            name="run_input",
+            display_name="Run input",
+            info=(
+                'The JSON input for the Actor run. For example for the "apify/website-content-crawler" Actor: '
+                '{"startUrls":[{"url":"https://docs.apify.com/academy/web-scraping-for-beginners"}],"maxCrawlDepth":0}'
+            ),
+            value='{"startUrls":[{"url":"https://docs.apify.com/academy/web-scraping-for-beginners"}],"maxCrawlDepth":0}',
+            required=True,
+        ),
+        MultilineInput(
+            name="dataset_fields",
+            display_name="Output fields",
+            info=(
+                "Fields to extract from the dataset, split by commas. "
+                "Other fields will be ignored. Dots in nested structures will be replaced by underscores. "
+                "Sample input: 'text, metadata.title'. "
+                "Sample output: {'text': 'page content here', 'metadata_title': 'page title here'}. "
+                "For example, for the 'apify/website-content-crawler' Actor, you can extract the 'markdown' field, "
+                "which is the content of the website in markdown format."
+            ),
+        ),
+        BoolInput(
+            name="flatten_dataset",
+            display_name="Flatten output",
+            info=(
+                "The output dataset will be converted from a nested format to a flat structure. "
+                "Dots in nested structure will be replaced by underscores. "
+                "This is useful for further processing of the Data object. "
+                "For example, {'a': {'b': 1}} will be flattened to {'a_b': 1}."
+            ),
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Output", name="output", type_=list[Data], method="run_model"),
+        Output(display_name="Tool", name="tool", type_=Tool, method="build_tool"),
+    ]
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._apify_client: ApifyClient | None = None
+
+    def run_model(self) -> list[Data]:
+        """Run the Actor and return node output."""
+        input_ = json.loads(self.run_input)
+        fields = ApifyActorsComponent.parse_dataset_fields(self.dataset_fields) if self.dataset_fields else None
+        res = self.run_actor(self.actor_id, input_, fields=fields)
+        if self.flatten_dataset:
+            res = [ApifyActorsComponent.flatten(item) for item in res]
+        data = [Data(data=item) for item in res]
+
+        self.status = data
+        return data
+
+    def build_tool(self) -> Tool:
+        """Build a tool for an agent that runs the Apify Actor."""
+        actor_id = self.actor_id
+
+        build = self._get_actor_latest_build(actor_id)
+        readme = build.get("readme", "")[:250] + "..."
+        if not (input_schema_str := build.get("inputSchema")):
+            msg = "Input schema not found"
+            raise ValueError(msg)
+        input_schema = json.loads(input_schema_str)
+        properties, required = ApifyActorsComponent.get_actor_input_schema_from_build(input_schema)
+        properties = {"run_input": properties}
+
+        # works from input schema
+        info_ = [
+            (
+                "JSON encoded as a string with input schema (STRICTLY FOLLOW JSON FORMAT AND SCHEMA):\n\n"
+                f"{json.dumps(properties, separators=(',', ':'))}"
+            )
+        ]
+        if required:
+            info_.append("\n\nRequired fields:\n" + "\n".join(required))
+
+        info = "".join(info_)
+
+        input_model_cls = ApifyActorsComponent.create_input_model_class(info)
+        tool_cls = ApifyActorsComponent.create_tool_class(self, readme, input_model_cls, actor_id)
+
+        return cast("Tool", tool_cls())
+
+    @staticmethod
+    def create_tool_class(
+        parent: "ApifyActorsComponent", readme: str, input_model: type[BaseModel], actor_id: str
+    ) -> type[BaseTool]:
+        """Create a tool class that runs an Apify Actor."""
+
+        class ApifyActorRun(BaseTool):
+            """Tool that runs Apify Actors."""
+
+            name: str = f"apify_actor_{ApifyActorsComponent.actor_id_to_tool_name(actor_id)}"
+            description: str = (
+                "Run an Apify Actor with the given input. "
+                "Here is a part of the currently loaded Actor README:\n\n"
+                f"{readme}\n\n"
+            )
+
+            args_schema: type[BaseModel] = input_model
+
+            @field_serializer("args_schema")
+            def serialize_args_schema(self, args_schema):
+                return args_schema.schema()
+
+            def _run(self, run_input: str | dict) -> str:
+                """Use the Apify Actor."""
+                input_dict = json.loads(run_input) if isinstance(run_input, str) else run_input
+
+                # retrieve if nested, just in case
+                input_dict = input_dict.get("run_input", input_dict)
+
+                res = parent.run_actor(actor_id, input_dict)
+                return "\n\n".join([ApifyActorsComponent.dict_to_json_str(item) for item in res])
+
+        return ApifyActorRun
+
+    @staticmethod
+    def create_input_model_class(description: str) -> type[BaseModel]:
+        """Create a Pydantic model class for the Actor input."""
+
+        class ActorInput(BaseModel):
+            """Input for the Apify Actor tool."""
+
+            run_input: str = Field(..., description=description)
+
+        return ActorInput
+
+    def _get_apify_client(self) -> ApifyClient:
+        """Get the Apify client.
+
+        Is created if not exists or token changes.
+        """
+        if not self.apify_token:
+            msg = "API token is required."
+            raise ValueError(msg)
+        # when token changes, create a new client
+        if self._apify_client is None or self._apify_client.token != self.apify_token:
+            self._apify_client = ApifyClient(self.apify_token)
+            if httpx_client := self._apify_client.http_client.httpx_client:
+                httpx_client.headers["user-agent"] += "; Origin/langflow"
+        return self._apify_client
+
+    def _get_actor_latest_build(self, actor_id: str) -> dict:
+        """Get the latest build of an Actor from the default build tag."""
+        client = self._get_apify_client()
+        actor = client.actor(actor_id=actor_id)
+        if not (actor_info := actor.get()):
+            msg = f"Actor {actor_id} not found."
+            raise ValueError(msg)
+
+        default_build_tag = actor_info.get("defaultRunOptions", {}).get("build")
+        latest_build_id = actor_info.get("taggedBuilds", {}).get(default_build_tag, {}).get("buildId")
+
+        if (build := client.build(latest_build_id).get()) is None:
+            msg = f"Build {latest_build_id} not found."
+            raise ValueError(msg)
+
+        return build
+
+    @staticmethod
+    def get_actor_input_schema_from_build(input_schema: dict) -> tuple[dict, list[str]]:
+        """Get the input schema from the Actor build.
+
+        Trim the description to 250 characters.
+        """
+        properties = input_schema.get("properties", {})
+        required = input_schema.get("required", [])
+
+        properties_out: dict = {}
+        for item, meta in properties.items():
+            properties_out[item] = {}
+            if desc := meta.get("description"):
+                properties_out[item]["description"] = (
+                    desc[:MAX_DESCRIPTION_LEN] + "..." if len(desc) > MAX_DESCRIPTION_LEN else desc
+                )
+            for key_name in ("type", "default", "prefill", "enum"):
+                if value := meta.get(key_name):
+                    properties_out[item][key_name] = value
+
+        return properties_out, required
+
+    def _get_run_dataset_id(self, run_id: str) -> str:
+        """Get the dataset id from the run id."""
+        client = self._get_apify_client()
+        run = client.run(run_id=run_id)
+        if (dataset := run.dataset().get()) is None:
+            msg = "Dataset not found"
+            raise ValueError(msg)
+        if (did := dataset.get("id")) is None:
+            msg = "Dataset id not found"
+            raise ValueError(msg)
+        return did
+
+    @staticmethod
+    def dict_to_json_str(d: dict) -> str:
+        """Convert a dictionary to a JSON string."""
+        return json.dumps(d, separators=(",", ":"), default=lambda _: "<n/a>")
+
+    @staticmethod
+    def actor_id_to_tool_name(actor_id: str) -> str:
+        """Turn actor_id into a valid tool name.
+
+        Tool name must only contain letters, numbers, underscores, dashes,
+            and cannot contain spaces.
+        """
+        valid_chars = string.ascii_letters + string.digits + "_-"
+        return "".join(char if char in valid_chars else "_" for char in actor_id)
+
+    def run_actor(self, actor_id: str, run_input: dict, fields: list[str] | None = None) -> list[dict]:
+        """Run an Apify Actor and return the output dataset.
+
+        Args:
+            actor_id: Actor name from Apify store to run.
+            run_input: JSON input for the Actor.
+            fields: List of fields to extract from the dataset. Other fields will be ignored.
+        """
+        client = self._get_apify_client()
+        if (details := client.actor(actor_id=actor_id).call(run_input=run_input, wait_secs=1)) is None:
+            msg = "Actor run details not found"
+            raise ValueError(msg)
+        if (run_id := details.get("id")) is None:
+            msg = "Run id not found"
+            raise ValueError(msg)
+
+        if (run_client := client.run(run_id)) is None:
+            msg = "Run client not found"
+            raise ValueError(msg)
+
+        # stream logs
+        with run_client.log().stream() as response:
+            if response:
+                for line in response.iter_lines():
+                    self.log(line)
+        run_client.wait_for_finish()
+
+        dataset_id = self._get_run_dataset_id(run_id)
+
+        loader = ApifyDatasetLoader(
+            dataset_id=dataset_id,
+            dataset_mapping_function=lambda item: item
+            if not fields
+            else {k.replace(".", "_"): ApifyActorsComponent.get_nested_value(item, k) for k in fields},
+        )
+        return loader.load()
+
+    @staticmethod
+    def get_nested_value(data: dict[str, Any], key: str) -> Any:
+        """Get a nested value from a dictionary."""
+        keys = key.split(".")
+        value = data
+        for k in keys:
+            if not isinstance(value, dict) or k not in value:
+                return None
+            value = value[k]
+        return value
+
+    @staticmethod
+    def parse_dataset_fields(dataset_fields: str) -> list[str]:
+        """Convert a string of comma-separated fields into a list of fields."""
+        dataset_fields = dataset_fields.replace("'", "").replace('"', "").replace("`", "")
+        return [field.strip() for field in dataset_fields.split(",")]
+
+    @staticmethod
+    def flatten(d: dict) -> dict:
+        """Flatten a nested dictionary."""
+
+        def items():
+            for key, value in d.items():
+                if isinstance(value, dict):
+                    for subkey, subvalue in ApifyActorsComponent.flatten(value).items():
+                        yield key + "_" + subkey, subvalue
+                else:
+                    yield key, value
+
+        return dict(items())

lfx/components/arxiv/__init__.py

@@ -0,0 +1,3 @@
+from .arxiv import ArXivComponent
+
+__all__ = ["ArXivComponent"]

lfx/components/arxiv/arxiv.py

@@ -0,0 +1,169 @@
+import urllib.request
+from urllib.parse import urlparse
+from xml.etree.ElementTree import Element
+
+from defusedxml.ElementTree import fromstring
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DropdownInput, IntInput, MessageTextInput, Output
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+
+
+class ArXivComponent(Component):
+    display_name = "arXiv"
+    description = "Search and retrieve papers from arXiv.org"
+    icon = "arXiv"
+
+    inputs = [
+        MessageTextInput(
+            name="search_query",
+            display_name="Search Query",
+            info="The search query for arXiv papers (e.g., 'quantum computing')",
+            tool_mode=True,
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Field",
+            info="The field to search in",
+            options=["all", "title", "abstract", "author", "cat"],  # cat is for category
+            value="all",
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results",
+            info="Maximum number of results to return",
+            value=10,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="DataFrame", name="dataframe", method="search_papers_dataframe"),
+    ]
+
+    def build_query_url(self) -> str:
+        """Build the arXiv API query URL."""
+        base_url = "http://export.arxiv.org/api/query?"
+
+        # Build the search query based on search type
+        if self.search_type == "all":
+            search_query = self.search_query  # No prefix for all fields
+        else:
+            # Map dropdown values to ArXiv API prefixes
+            prefix_map = {"title": "ti", "abstract": "abs", "author": "au", "cat": "cat"}
+            prefix = prefix_map.get(self.search_type, "")
+            search_query = f"{prefix}:{self.search_query}"
+
+        # URL parameters
+        params = {
+            "search_query": search_query,
+            "max_results": str(self.max_results),
+        }
+
+        # Convert params to URL query string
+        query_string = "&".join([f"{k}={urllib.parse.quote(str(v))}" for k, v in params.items()])
+
+        return base_url + query_string
+
+    def parse_atom_response(self, response_text: str) -> list[dict]:
+        """Parse the Atom XML response from arXiv."""
+        # Parse XML safely using defusedxml
+        root = fromstring(response_text)
+
+        # Define namespace dictionary for XML parsing
+        ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
+
+        papers = []
+        # Process each entry (paper)
+        for entry in root.findall("atom:entry", ns):
+            paper = {
+                "id": self._get_text(entry, "atom:id", ns),
+                "title": self._get_text(entry, "atom:title", ns),
+                "summary": self._get_text(entry, "atom:summary", ns),
+                "published": self._get_text(entry, "atom:published", ns),
+                "updated": self._get_text(entry, "atom:updated", ns),
+                "authors": [author.find("atom:name", ns).text for author in entry.findall("atom:author", ns)],
+                "arxiv_url": self._get_link(entry, "alternate", ns),
+                "pdf_url": self._get_link(entry, "related", ns),
+                "comment": self._get_text(entry, "arxiv:comment", ns),
+                "journal_ref": self._get_text(entry, "arxiv:journal_ref", ns),
+                "primary_category": self._get_category(entry, ns),
+                "categories": [cat.get("term") for cat in entry.findall("atom:category", ns)],
+            }
+            papers.append(paper)
+
+        return papers
+
+    def _get_text(self, element: Element, path: str, ns: dict) -> str | None:
+        """Safely extract text from an XML element."""
+        el = element.find(path, ns)
+        return el.text.strip() if el is not None and el.text else None
+
+    def _get_link(self, element: Element, rel: str, ns: dict) -> str | None:
+        """Get link URL based on relation type."""
+        for link in element.findall("atom:link", ns):
+            if link.get("rel") == rel:
+                return link.get("href")
+        return None
+
+    def _get_category(self, element: Element, ns: dict) -> str | None:
+        """Get primary category."""
+        cat = element.find("arxiv:primary_category", ns)
+        return cat.get("term") if cat is not None else None
+
+    def run_model(self) -> DataFrame:
+        return self.search_papers_dataframe()
+
+    def search_papers(self) -> list[Data]:
+        """Search arXiv and return results."""
+        try:
+            # Build the query URL
+            url = self.build_query_url()
+
+            # Validate URL scheme and host
+            parsed_url = urlparse(url)
+            if parsed_url.scheme not in {"http", "https"}:
+                error_msg = f"Invalid URL scheme: {parsed_url.scheme}"
+                raise ValueError(error_msg)
+            if parsed_url.hostname != "export.arxiv.org":
+                error_msg = f"Invalid host: {parsed_url.hostname}"
+                raise ValueError(error_msg)
+
+            # Create a custom opener that only allows http/https schemes
+            class RestrictedHTTPHandler(urllib.request.HTTPHandler):
+                def http_open(self, req):
+                    return super().http_open(req)
+
+            class RestrictedHTTPSHandler(urllib.request.HTTPSHandler):
+                def https_open(self, req):
+                    return super().https_open(req)
+
+            # Build opener with restricted handlers
+            opener = urllib.request.build_opener(RestrictedHTTPHandler, RestrictedHTTPSHandler)
+            urllib.request.install_opener(opener)
+
+            # Make the request with validated URL using restricted opener
+            response = opener.open(url)
+            response_text = response.read().decode("utf-8")
+
+            # Parse the response
+            papers = self.parse_atom_response(response_text)
+
+            # Convert to Data objects
+            results = [Data(data=paper) for paper in papers]
+            self.status = results
+        except (urllib.error.URLError, ValueError) as e:
+            error_data = Data(data={"error": f"Request error: {e!s}"})
+            self.status = error_data
+            return [error_data]
+        else:
+            return results
+
+    def search_papers_dataframe(self) -> DataFrame:
+        """Convert the Arxiv search results to a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame containing the search results.
+        """
+        data = self.search_papers()
+        return DataFrame(data)

lfx/components/assemblyai/__init__.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .assemblyai_get_subtitles import AssemblyAIGetSubtitles
+    from .assemblyai_lemur import AssemblyAILeMUR
+    from .assemblyai_list_transcripts import AssemblyAIListTranscripts
+    from .assemblyai_poll_transcript import AssemblyAITranscriptionJobPoller
+    from .assemblyai_start_transcript import AssemblyAITranscriptionJobCreator
+
+_dynamic_imports = {
+    "AssemblyAIGetSubtitles": "assemblyai_get_subtitles",
+    "AssemblyAILeMUR": "assemblyai_lemur",
+    "AssemblyAIListTranscripts": "assemblyai_list_transcripts",
+    "AssemblyAITranscriptionJobCreator": "assemblyai_start_transcript",
+    "AssemblyAITranscriptionJobPoller": "assemblyai_poll_transcript",
+}
+
+__all__ = [
+    "AssemblyAIGetSubtitles",
+    "AssemblyAILeMUR",
+    "AssemblyAIListTranscripts",
+    "AssemblyAITranscriptionJobCreator",
+    "AssemblyAITranscriptionJobPoller",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import assemblyai components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/assemblyai/assemblyai_get_subtitles.py

@@ -0,0 +1,83 @@
+import assemblyai as aai
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, DropdownInput, IntInput, Output, SecretStrInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+
+
+class AssemblyAIGetSubtitles(Component):
+    display_name = "AssemblyAI Get Subtitles"
+    description = "Export your transcript in SRT or VTT format for subtitles and closed captions"
+    documentation = "https://www.assemblyai.com/docs"
+    icon = "AssemblyAI"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Assembly API Key",
+            info="Your AssemblyAI API key. You can get one from https://www.assemblyai.com/",
+            required=True,
+        ),
+        DataInput(
+            name="transcription_result",
+            display_name="Transcription Result",
+            info="The transcription result from AssemblyAI",
+            required=True,
+        ),
+        DropdownInput(
+            name="subtitle_format",
+            display_name="Subtitle Format",
+            options=["srt", "vtt"],
+            value="srt",
+            info="The format of the captions (SRT or VTT)",
+        ),
+        IntInput(
+            name="chars_per_caption",
+            display_name="Characters per Caption",
+            info="The maximum number of characters per caption (0 for no limit)",
+            value=0,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Subtitles", name="subtitles", method="get_subtitles"),
+    ]
+
+    def get_subtitles(self) -> Data:
+        aai.settings.api_key = self.api_key
+
+        # check if it's an error message from the previous step
+        if self.transcription_result.data.get("error"):
+            self.status = self.transcription_result.data["error"]
+            return self.transcription_result
+
+        try:
+            transcript_id = self.transcription_result.data["id"]
+            transcript = aai.Transcript.get_by_id(transcript_id)
+        except Exception as e:  # noqa: BLE001
+            error = f"Getting transcription failed: {e}"
+            logger.debug(error, exc_info=True)
+            self.status = error
+            return Data(data={"error": error})
+
+        if transcript.status == aai.TranscriptStatus.completed:
+            subtitles = None
+            chars_per_caption = self.chars_per_caption if self.chars_per_caption > 0 else None
+            if self.subtitle_format == "srt":
+                subtitles = transcript.export_subtitles_srt(chars_per_caption)
+            else:
+                subtitles = transcript.export_subtitles_vtt(chars_per_caption)
+
+            result = Data(
+                subtitles=subtitles,
+                format=self.subtitle_format,
+                transcript_id=transcript_id,
+                chars_per_caption=chars_per_caption,
+            )
+
+            self.status = result
+            return result
+        self.status = transcript.error
+        return Data(data={"error": transcript.error})