apify-haystack 0.0.1a2__tar.gz → 0.0.1a4__tar.gz

{apify_haystack-0.0.1a2 → apify_haystack-0.0.1a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify_haystack
-Version: 0.0.1a2
+Version: 0.0.1a4
 Summary: Apify-haystack integration
 Home-page: https://apify.com
 License: Apache-2.0
@@ -18,6 +18,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: apify-client (>=1.7.1,<2.0.0)
+Requires-Dist: haystack-ai (>=2.3.0,<3.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Project-URL: Changelog, https://github.com/apify/apify-haystack/blob/master/CHANGELOG.md
 Project-URL: Documentation, https://github.com/apify/apify-haystack
 Project-URL: Issue Tracker, https://github.com/apify/apify-haystack/issues

{apify_haystack-0.0.1a2 → apify_haystack-0.0.1a4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify_haystack"
-version = "0.0.1a2"
+version = "0.0.1a4"
 description = "Apify-haystack integration"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 homepage = "https://apify.com"
@@ -43,6 +43,9 @@ keywords = [
 # https://github.com/apify/apify-sdk-python/pull/154.
 [tool.poetry.dependencies]
 python = "^3.9"
+apify-client = "^1.7.1"
+haystack-ai = "^2.3.0"
+python-dotenv = "^1.0.1"
 
 [tool.poetry.group.dev.dependencies]
 build = "~1.2.0"
@@ -54,7 +57,7 @@ ruff = "~0.5.0"
 setuptools = "~70.3.0"  # setuptools are used by pytest, but not explicitly required
 
 [tool.ruff]
-line-length = 150
+line-length = 120
 
 [tool.ruff.lint]
 select = ["ALL"]
@@ -110,6 +113,12 @@ indent-style = "space"
     "T20",     # flake8-print
     "TRY301",  # Abstract `raise` to an inner function
 ]
+"**/{examples}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "T20",     # flake8-print
+]
+
 
 [tool.ruff.lint.flake8-quotes]
 docstring-quotes = "double"
@@ -139,6 +148,10 @@ warn_return_any = true
 warn_unreachable = true
 warn_unused_ignores = true
 
+[[tool.mypy.overrides]]
+module = "haystack.*"
+ignore_missing_imports = true
+
 [tool.mypy-sortedcollections]
 ignore_missing_imports = true
 

apify_haystack-0.0.1a4/src/apify_haystack/__init__.py

@@ -0,0 +1,7 @@
+from importlib import metadata
+
+__version__ = metadata.version("apify-haystack")
+
+from apify_haystack.apify_dataset import ApifyDatasetFromActorCall, ApifyDatasetLoader
+
+__all__ = ["ApifyDatasetLoader", "ApifyDatasetFromActorCall", "ApifyDatasetFromActorCall"]

apify_haystack-0.0.1a4/src/apify_haystack/apify_dataset.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import os
+from typing import Callable
+
+from apify_client import ApifyClient
+from haystack import component
+from haystack.dataclasses import Document
+
+from apify_haystack.constants import HAYSTACK_ATTRIBUTE_USER_AGENT
+
+
+@component
+class ApifyDatasetLoader:
+    """Load datasets produced by the `Apify Actors`.
+
+    Actors are serverless functions that can be used to scrape, crawl, and extract data from the web.
+    For details, see https://docs.apify.com/platform/integrations/haystack
+
+    Example:
+        .. code-block:: python
+
+            from haystack import Document
+
+            from apify_haystack import ApifyDatasetLoader
+
+            def dataset_mapping_function(dataset_item):
+                return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+            loader = ApifyDatasetLoader(dataset_id=dataset_id, dataset_mapping_function=dataset_mapping_function)
+            dataset = loader.run()
+    """
+
+    def __init__(
+        self,
+        dataset_id: str,
+        dataset_mapping_function: Callable[[dict], Document],
+    ) -> None:
+        """Initialize the Apify dataset loader.
+
+        Args:
+            dataset_id (str): The ID of the dataset on the Apify platform.
+            dataset_mapping_function (Callable): A function that takes a single (an Apify dataset item) and converts
+                it to an instance of the Document class.
+        """
+        self.dataset_id = dataset_id
+        self.dataset_mapping_function = dataset_mapping_function
+
+        # Add a custom user-agent to the httpx client for attribution purposes
+        self.client = ApifyClient()
+        if httpx_client := getattr(self.client.http_client, "httpx_client", None):
+            httpx_client.headers["user-agent"] += f"; {HAYSTACK_ATTRIBUTE_USER_AGENT}"
+
+    @component.output_types(documents=list[Document])
+    def run(self):  # type: ignore[no-untyped-def] # noqa: ANN201
+        """Load datasets produced by the `Apify Actors`.
+
+        Type-hint note: the output is not type-hinted, otherwise linting complains
+        that `run` method is not haystack a component
+        """
+        dataset_items = self.client.dataset(self.dataset_id).list_items(clean=True).items
+        return {"documents": list(map(self.dataset_mapping_function, dataset_items))}
+
+
+@component
+class ApifyDatasetFromActorCall:
+    """Get Apify dataset by calling Apify Actor.
+
+    Perform call of Apify actor and load produced dataset.
+    For details, see https://docs.apify.com/platform/integrations/haystack
+
+    To use, you should have the ``apify-client`` python package installed,
+    and the environment variable ``APIFY_API_TOKEN`` set with your API key, or pass
+    `apify_api_token` as a named parameter to the constructor.
+    """
+
+    def __init__(
+        self,
+        actor_id: str,
+        dataset_mapping_function: Callable[[dict], Document],
+        run_input: dict,
+        *,
+        build: str | None = None,
+        memory_mbytes: int | None = None,
+        timeout_secs: int | None = None,
+        apify_api_token: str | None = None,
+    ) -> None:
+        """Initialize the Apify Actor loader.
+
+        Args:
+            actor_id (str): The ID or name of the Actor on the Apify platform.
+            run_input (Dict): The input object of the Actor that you're trying to run.
+            dataset_mapping_function (Callable): A function that takes a single
+                dictionary (an Apify dataset item) and converts it to an
+                instance of the Document class.
+            build (str, optional): Optionally specifies the actor build to run.
+                It can be either a build tag or build number.
+            memory_mbytes (int, optional): Optional memory limit for the run, in megabytes.
+            timeout_secs (int, optional): Optional timeout for the run, in seconds.
+            apify_api_token (str, optional): The API token for the Apify platform.
+                If not provided, it will be read from the environment
+        """
+        self.actor_id = actor_id
+        self.dataset_mapping_function = dataset_mapping_function
+        self.run_input = run_input
+        self.build = build
+        self.memory_mbytes = memory_mbytes
+        self.timeout_secs = timeout_secs
+
+        if not (apify_api_token := apify_api_token or os.getenv("APIFY_API_TOKEN")):
+            raise ValueError(
+                "Apify API token not found. Please provide Apify API token when initializing "
+                "ApifyDatasetCall, or set the environment variable APIFY_API_TOKEN."
+            )
+        # Add a custom user-agent to the httpx client for attribution purposes
+        self.client = ApifyClient(apify_api_token)
+
+        if httpx_client := getattr(self.client.http_client, "httpx_client", None):
+            httpx_client.headers["user-agent"] += f"; {HAYSTACK_ATTRIBUTE_USER_AGENT}"
+
+    @component.output_types(documents=list[Document])
+    def run(self):  # type: ignore[no-untyped-def] # noqa: ANN201
+        """Run an Actor on the Apify platform and wait for results to be ready.
+
+        Type-hint note: the output is not type-hinted, otherwise linting complains
+        that `run` method is not haystack a component
+        """
+        if not (
+            actor_call := self.client.actor(self.actor_id).call(
+                run_input=self.run_input,
+                build=self.build,
+                memory_mbytes=self.memory_mbytes,
+                timeout_secs=self.timeout_secs,
+            )
+        ):
+            raise ValueError(f"No response found in the actor {self.actor_id} call")
+
+        if not (dataset_id := actor_call.get("defaultDatasetId")):
+            raise ValueError(f"No dataset found in the actor {self.actor_id} call response.")
+
+        loader = ApifyDatasetLoader(
+            dataset_id=dataset_id,
+            dataset_mapping_function=self.dataset_mapping_function,
+        )
+        return loader.run()
+
+
+@component
+class ApifyDatasetFromTaskCall:
+    """Get Apify dataset by calling Apify Task.
+
+    Perform call of Apify task and load produced dataset.
+    """
+
+    def __init__(
+        self,
+        task_id: str,
+        dataset_mapping_function: Callable[[dict], Document],
+        task_input: dict,
+        *,
+        build: str | None = None,
+        memory_mbytes: int | None = None,
+        timeout_secs: int | None = None,
+        apify_api_token: str | None = None,
+    ) -> None:
+        """Initialize the Apify Actor loader.
+
+        Args:
+            task_id (str): The ID or name of the Actor on the Apify platform.
+            task_input (Dict): The input object of the Actor that you're trying to run.
+            dataset_mapping_function (Callable): A function that takes a single
+                dictionary (an Apify dataset item) and converts it to an
+                instance of the Document class.
+            build (str, optional): Optionally specifies the actor build to run.
+                It can be either a build tag or build number.
+            memory_mbytes (int, optional): Optional memory limit for the run, in megabytes.
+            timeout_secs (int, optional): Optional timeout for the run, in seconds.
+            apify_api_token (str, optional): The API token for the Apify platform.
+                If not provided, it will be read from the environment
+        """
+        self.task_id = task_id
+        self.dataset_mapping_function = dataset_mapping_function
+        self.task_input = task_input
+        self.build = build
+        self.memory_mbytes = memory_mbytes
+        self.timeout_secs = timeout_secs
+
+        if not (apify_api_token := apify_api_token or os.getenv("APIFY_API_TOKEN")):
+            raise ValueError(
+                "Apify API token not found. Please provide Apify API token when initializing "
+                "ApifyDatasetCall, or set the environment variable APIFY_API_TOKEN."
+            )
+        # Add a custom user-agent to the httpx client for attribution purposes
+        self.client = ApifyClient(apify_api_token)
+
+        if httpx_client := getattr(self.client.http_client, "httpx_client", None):
+            httpx_client.headers["user-agent"] += f"; {HAYSTACK_ATTRIBUTE_USER_AGENT}"
+
+    @component.output_types(documents=list[Document])
+    def run(self):  # type: ignore[no-untyped-def] # noqa: ANN201
+        """Run an Actor on the Apify platform and wait for results to be ready.
+
+        Type-hint note: the output is not type-hinted, otherwise linting complains
+        that `run` method is not haystack a component
+        """
+        if not (
+            actor_call := self.client.task(self.task_id).call(
+                task_input=self.task_input,
+                build=self.build,
+                memory_mbytes=self.memory_mbytes,
+                timeout_secs=self.timeout_secs,
+            )
+        ):
+            raise ValueError(f"No response found in the actor {self.task_id} call")
+
+        if not (dataset_id := actor_call.get("defaultDatasetId")):
+            raise ValueError(f"No dataset found in the actor {self.task_id} call response.")
+
+        loader = ApifyDatasetLoader(
+            dataset_id=dataset_id,
+            dataset_mapping_function=self.dataset_mapping_function,
+        )
+        return loader.run()

apify_haystack-0.0.1a4/src/apify_haystack/constants.py

@@ -0,0 +1,2 @@
+# A custom user-agent for attribution and monitoring purposes
+HAYSTACK_ATTRIBUTE_USER_AGENT = "Origin/haystack"

apify_haystack-0.0.1a4/src/apify_haystack/examples/apify_actor_call.py

@@ -0,0 +1,44 @@
+"""
+Call Apify Actor and load a dataset to convert it to Haystack Documents.
+
+This script demonstrates how to use Apify's Website Content Crawler to scrape text content from a specified website
+and convert the scraped data into Haystack Documents.
+
+For example, if you want to crawl and get text content from https://haystack.deepset.ai/, you can use this script
+to call the Apify actor and retrieve the data in a structured format.
+
+The script should produce the following output (an example of a single Document):
+......
+Document(id=a617d376*****, content: 'Introduction to Haystack 2.x)
+Haystack is an open-source framework fo...', meta: {'url': 'https://docs.haystack.deepset.ai/docs/intro'}
+.....
+"""
+
+from dotenv import load_dotenv
+from haystack import Document
+
+from apify_haystack import ApifyDatasetFromActorCall
+
+# Sey APIFY-API-TOKEN here or load it from .env file
+apify_token = "" or load_dotenv()
+
+actor_id = "apify/website-content-crawler"
+run_input = {
+    "maxCrawlPages": 3,  # limit the number of pages to crawl
+    "startUrls": [{"url": "https://haystack.deepset.ai/"}],
+}
+
+
+def dataset_mapping_function(dataset_item: dict) -> Document:
+    return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+
+actor = ApifyDatasetFromActorCall(
+    actor_id=actor_id, run_input=run_input, dataset_mapping_function=dataset_mapping_function
+)
+print(f"Calling Apify actor {actor_id} ... crawling will take some time ...")
+dataset = actor.run().get("documents")
+
+print(f"Loaded {len(dataset)} documents from the Apify Actor {actor_id}:")
+for d in dataset:
+    print(d)

apify_haystack-0.0.1a4/src/apify_haystack/examples/apify_dataset_load.py

@@ -0,0 +1,31 @@
+"""
+Load a dataset from Apify and convert it to Haystack Documents.
+
+This script demonstrates how to load a dataset from Apify and convert it into Haystack Documents.
+Assuming you have run an Apify actor that scraped web pages from https://docs.haystack.deepset.ai/,
+you can use the dataset ID to load the dataset and transform it into a structured format suitable for Haystack.
+
+The script should produce the following output (an example of a single Document):
+......
+Document(id=a617d376*****, content: 'Introduction to Haystack 2.x)
+Haystack is an open-source framework fo...', meta: {'url': 'https://docs.haystack.deepset.ai/docs/intro'}
+.....
+"""
+
+from haystack import Document
+
+from apify_haystack import ApifyDatasetLoader
+
+dataset_id = "YOUR-DATASET-ID"
+
+
+def dataset_mapping_function(dataset_item: dict) -> Document:
+    return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+
+loader = ApifyDatasetLoader(dataset_id=dataset_id, dataset_mapping_function=dataset_mapping_function)
+dataset = loader.run().get("documents")
+
+print(f"Loaded {len(dataset)} documents from Apify dataset {dataset_id}:")
+for d in dataset:
+    print(d)

apify_haystack-0.0.1a4/src/apify_haystack/examples/crawl_and_process_data.py

@@ -0,0 +1,84 @@
+"""
+This script demonstrates how to extract content from a website using the Apify actor: Website Content Crawler.
+The content is then cleaned, split into smaller chunks, embedded, and stored in the InMemoryDocumentStore.
+
+After the pipeline is executed, the documents are retrieved from the document store using BM25 retrieval.
+
+The script should produce the following output (an example of a single Document):
+......
+Document(id=15a9f391a9f14871675826af8767bd3d2c81d7de1892fd9288b6be0dac26ea89, content: 'Get Started | Haystack
+Haystack is an open-source Python framework that helps developers build LLM-p...',
+meta: {'url': 'https://haystack.deepset.ai/overview/quick-start',
+'source_id': '2a7263cb668b8353749cc990f487a3feb7a7d7f3abe3e3af3a69772cdecc8200', 'page_number': 1,
+ 'split_id': 0, 'split_idx_start': 0, '_split_overlap': ......
+.....
+"""
+
+import os
+
+from dotenv import load_dotenv
+from haystack import Document, Pipeline
+from haystack.components.embedders import OpenAIDocumentEmbedder
+from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils.auth import Secret
+
+from apify_haystack import ApifyDatasetFromActorCall
+
+load_dotenv()
+
+# Set APIFY-API-TOKEN here or use it from .env file
+apify_token = "" or os.getenv("APIFY_API_TOKEN")
+openai_api_key = "" or os.getenv("OPENAI_API_KEY")
+
+actor_id = "apify/website-content-crawler"
+run_input = {
+    "maxCrawlPages": 3,  # limit the number of pages to crawl
+    "startUrls": [{"url": "https://haystack.deepset.ai/"}],
+}
+
+
+def dataset_mapping_function(dataset_item: dict) -> Document:
+    return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+
+document_loader = ApifyDatasetFromActorCall(
+    actor_id=actor_id, run_input=run_input, dataset_mapping_function=dataset_mapping_function
+)
+
+document_store = InMemoryDocumentStore()
+print(f"Initialized InMemoryDocumentStore with {document_store.count_documents()} documents")
+
+document_cleaner = DocumentCleaner()
+document_splitter = DocumentSplitter(split_by="word", split_length=150, split_overlap=50)
+document_embedder = OpenAIDocumentEmbedder(api_key=Secret.from_token(openai_api_key))
+document_writer = DocumentWriter(document_store)
+
+pipe = Pipeline()
+pipe.add_component("document_loader", document_loader)
+pipe.add_component("document_cleaner", document_cleaner)
+pipe.add_component("document_splitter", document_splitter)
+pipe.add_component("document_embedder", document_embedder)
+pipe.add_component("document_writer", document_writer)
+
+pipe.connect("document_loader", "document_cleaner")
+pipe.connect("document_cleaner", "document_splitter")
+pipe.connect("document_splitter", "document_embedder")
+pipe.connect("document_embedder", "document_writer")
+
+print(
+    "Running pipeline the Apify document_loader -> document_cleaner -> document_splitter "
+    "-> document_embedder -> document_writer"
+)
+print("Crawling will take some time ...")
+print("You can visit https://console.apify.com/actors/runs to monitor the progress")
+
+pipe.run({"document_loader": {}})
+
+print(f"Added {document_store.count_documents()} to vector from Website Content Crawler")
+
+print("\nRetrieving documents from the document store: query='Haystack'")
+print("query='Haystack'\n")
+for doc in document_store.bm25_retrieval("Haystack", top_k=1):
+    print(doc)

apify_haystack-0.0.1a4/src/apify_haystack/examples/rag_with_crawled_website.py

@@ -0,0 +1,111 @@
+"""
+Retrieval-Augmented Generation (RAG)
+
+This script demonstrates how to extract text content from a website, store it in a vector database, and use
+it for question answering.
+
+Steps involved:
+- Load environment variables, including API keys.
+- Define the Apify actor and its input parameters for web scraping.
+- Initialize the Apify dataset loader to fetch and process the dataset.
+- Set up the components for the RAG pipeline:
+- Load documents from Apify and store their embeddings in the document store.
+- Define a prompt template for the question-answering task.
+- Build and connect the components in the pipeline.
+- Run the pipeline to answer a sample question and print the response.
+
+Expected output:
+question: "What is haystack?"
+......
+Haystack is an open-source framework for building production-ready LLM applications
+......
+"""
+
+import os
+
+from dotenv import load_dotenv
+from haystack import Document, Pipeline
+from haystack.components.builders import PromptBuilder
+from haystack.components.embedders import OpenAIDocumentEmbedder, OpenAITextEmbedder
+from haystack.components.generators import OpenAIGenerator
+from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils.auth import Secret
+
+from apify_haystack import ApifyDatasetFromActorCall
+
+# Set APIFY-API-TOKEN here or use it from .env file
+apify_token = "" or os.getenv("APIFY_API_TOKEN")
+openai_api_key = "" or os.getenv("OPENAI_API_KEY")
+load_dotenv()
+
+actor_id = "apify/website-content-crawler"
+run_input = {
+    "maxCrawlPages": 1,  # limit the number of pages to crawl
+    "startUrls": [{"url": "https://haystack.deepset.ai/"}],
+}
+
+
+def dataset_mapping_function(dataset_item: dict) -> Document:
+    return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+
+apify_dataset_loader = ApifyDatasetFromActorCall(
+    actor_id=actor_id, run_input=run_input, dataset_mapping_function=dataset_mapping_function
+)
+
+# Components
+document_store = InMemoryDocumentStore()
+
+docs_embedder = OpenAIDocumentEmbedder(api_key=Secret.from_token(openai_api_key))
+text_embedder = OpenAITextEmbedder(api_key=Secret.from_token(openai_api_key))
+retriever = InMemoryEmbeddingRetriever(document_store)
+generator = OpenAIGenerator(model="gpt-3.5-turbo")
+
+# Load documents from Apify
+docs = apify_dataset_loader.run()
+embeddings = docs_embedder.run(docs.get("documents"))
+document_store.write_documents(embeddings["documents"])
+
+template = """
+Given the following information, answer the question.
+
+Context:
+{% for document in documents %}
+    {{ document.content }}
+{% endfor %}
+
+Question: {{question}}
+Answer:
+"""
+
+prompt_builder = PromptBuilder(template=template)
+
+# Add components to your pipeline
+pipe = Pipeline()
+pipe.add_component("embedder", text_embedder)
+pipe.add_component("retriever", retriever)
+pipe.add_component("prompt_builder", prompt_builder)
+pipe.add_component("llm", generator)
+
+# Now, connect the components to each other
+pipe.connect("embedder.embedding", "retriever.query_embedding")
+pipe.connect("retriever", "prompt_builder.documents")
+pipe.connect("prompt_builder", "llm")
+
+question = "What is haystack?"
+
+response = pipe.run({"embedder": {"text": question}, "prompt_builder": {"question": question}})
+
+print(f"question: {question}")
+print(f"answer: {response['llm']['replies'][0]}")
+
+# Other questions
+examples = [
+    "Who created Haystack AI?",
+    "Are there any upcoming events or community talks?",
+]
+
+for example in examples:
+    response = pipe.run({"embedder": {"text": example}, "prompt_builder": {"question": example}})
+    print(response["llm"]["replies"][0])

apify_haystack-0.0.1a2/src/apify_haystack/__init__.py

@@ -1,3 +0,0 @@
-from importlib import metadata
-
-__version__ = metadata.version("apify-haystack")