apify-haystack 0.0.1a3__tar.gz → 0.0.1a5__tar.gz

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify_haystack
-Version: 0.0.1a3
+Version: 0.0.1a5
 Summary: Apify-haystack integration
 Home-page: https://apify.com
 License: Apache-2.0
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
 Requires-Dist: apify-client (>=1.7.1,<2.0.0)
-Requires-Dist: haystack-ai (>=2.3.0,<3.0.0)
+Requires-Dist: haystack-ai (>=2.3.1,<3.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Project-URL: Changelog, https://github.com/apify/apify-haystack/blob/master/CHANGELOG.md
 Project-URL: Documentation, https://github.com/apify/apify-haystack
@@ -29,11 +29,12 @@ Description-Content-Type: text/markdown
 
 # Apify-Haystack integration
 
-Apify-Haystack integration allows to quickly fetch results from Apify Actors.
+The Apify-Haystack integration allows easy interaction between the Apify platform and Haystack.
 
-🚧 **Work In Progress** 🚧
+Apify is a platform for web scraping, data extraction, and web automation tasks.
+It provides serverless applications called Actors for different tasks, like crawling websites, and scraping Facebook, Instagram, and Google results, etc.
 
-This project is currently under active development.
+Haystack offers an ecosystem of tools for building, managing, and deploying search engines and LLM applications.
 
 ## Installation
 
@@ -45,9 +46,10 @@ pip install apify-haystack
 
 ## Examples
 
-Here are some practical examples to help you get started ...
-
-## Features
+- Load a dataset from Apify and convert it to Haystack Documents: [apify_dataset_load.py](src/apify_haystack/examples/apify_dataset_load.py)
+- Call Apify Actor and load a dataset to convert it to Haystack Documents: [apify_actor_call.py](src/apify_haystack/examples/apify_actor_call.py)
+- Crawl website, scrape text content, and store it in the InMemoryDocumentStore: [crawl_and_process_data.py](src/apify_haystack/examples/crawl_and_process_data.py)
+- Retrieval-Augmented Generation (RAG): Extracting text content from a website and using it for question answering [rag_with_crawled_website.py](src/apify_haystack/examples/rag_with_crawled_website.py)
 
 ## Support
 

apify_haystack-0.0.1a5/README.md

@@ -0,0 +1,38 @@
+# Apify-Haystack integration
+
+The Apify-Haystack integration allows easy interaction between the Apify platform and Haystack.
+
+Apify is a platform for web scraping, data extraction, and web automation tasks.
+It provides serverless applications called Actors for different tasks, like crawling websites, and scraping Facebook, Instagram, and Google results, etc.
+
+Haystack offers an ecosystem of tools for building, managing, and deploying search engines and LLM applications.
+
+## Installation
+
+Apify-haystack is available as the [`apify-haystack`](https://pypi.org/project/apify-haystack/) PyPI package.
+
+```sh
+pip install apify-haystack
+```
+
+## Examples
+
+- Load a dataset from Apify and convert it to Haystack Documents: [apify_dataset_load.py](src/apify_haystack/examples/apify_dataset_load.py)
+- Call Apify Actor and load a dataset to convert it to Haystack Documents: [apify_actor_call.py](src/apify_haystack/examples/apify_actor_call.py)
+- Crawl website, scrape text content, and store it in the InMemoryDocumentStore: [crawl_and_process_data.py](src/apify_haystack/examples/crawl_and_process_data.py)
+- Retrieval-Augmented Generation (RAG): Extracting text content from a website and using it for question answering [rag_with_crawled_website.py](src/apify_haystack/examples/rag_with_crawled_website.py)
+
+## Support
+
+If you find any bug or issue, please [submit an issue on GitHub](https://github.com/apify/apify-haystack/issues).
+For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questions/tagged/apify), in GitHub Discussions or you can join our [Discord server](https://discord.com/invite/jyEM2PRvMU).
+
+## Contributing
+
+Your code contributions are welcome, and you'll be praised for eternity!
+If you have any ideas for improvements, either submit an issue or create a pull request.
+For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/apify-haystack/blob/master/CONTRIBUTING.md).
+
+## License
+
+This project is licensed under the Apache License 2.0 - see the [LICENSE](https://github.com/apify/apify-haystack/blob/master/LICENSE) file for details.

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify_haystack"
-version = "0.0.1a3"
+version = "0.0.1a5"
 description = "Apify-haystack integration"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 homepage = "https://apify.com"
@@ -44,7 +44,7 @@ keywords = [
 [tool.poetry.dependencies]
 python = "^3.9"
 apify-client = "^1.7.1"
-haystack-ai = "^2.3.0"
+haystack-ai = "^2.3.1"
 python-dotenv = "^1.0.1"
 
 [tool.poetry.group.dev.dependencies]

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a5}/src/apify_haystack/examples/crawl_and_process_data.py

@@ -1,4 +1,6 @@
 """
+Crawl website, scrape text content, and store it in the InMemoryDocumentStore.
+
 This script demonstrates how to extract content from a website using the Apify actor: Website Content Crawler.
 The content is then cleaned, split into smaller chunks, embedded, and stored in the InMemoryDocumentStore.
 
@@ -14,17 +16,23 @@ meta: {'url': 'https://haystack.deepset.ai/overview/quick-start',
 .....
 """
 
+import os
+
 from dotenv import load_dotenv
 from haystack import Document, Pipeline
-from haystack.components.embedders import SentenceTransformersDocumentEmbedder
+from haystack.components.embedders import OpenAIDocumentEmbedder
 from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
 from haystack.components.writers import DocumentWriter
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils.auth import Secret
 
 from apify_haystack import ApifyDatasetFromActorCall
 
-# Sey APIFY-API-TOKEN here or load it from .env file
-apify_token = "" or load_dotenv()
+load_dotenv()
+
+# Set APIFY-API-TOKEN here or use it from .env file
+apify_token = "" or os.getenv("APIFY_API_TOKEN")
+openai_api_key = "" or os.getenv("OPENAI_API_KEY")
 
 actor_id = "apify/website-content-crawler"
 run_input = {
@@ -46,7 +54,7 @@ print(f"Initialized InMemoryDocumentStore with {document_store.count_documents()
 
 document_cleaner = DocumentCleaner()
 document_splitter = DocumentSplitter(split_by="word", split_length=150, split_overlap=50)
-document_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
+document_embedder = OpenAIDocumentEmbedder(api_key=Secret.from_token(openai_api_key))
 document_writer = DocumentWriter(document_store)
 
 pipe = Pipeline()
@@ -73,5 +81,6 @@ pipe.run({"document_loader": {}})
 print(f"Added {document_store.count_documents()} to vector from Website Content Crawler")
 
 print("Retrieving documents from the document store: query='Haystack'")
+print("query='Haystack'")
 for doc in document_store.bm25_retrieval("Haystack", top_k=1):
     print(doc)

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a5}/src/apify_haystack/examples/rag_with_crawled_website.py

@@ -1,8 +1,5 @@
 """
-Retrieval-Augmented Generation (RAG)
-
-This script demonstrates how to extract text content from a website, store it in a vector database, and use
-it for question answering.
+Retrieval-Augmented Generation (RAG): Extracting text content from a website and using it for question answering.
 
 Steps involved:
 - Load environment variables, including API keys.
@@ -21,18 +18,23 @@ Haystack is an open-source framework for building production-ready LLM applicati
 ......
 """
 
+import os
+
 from dotenv import load_dotenv
 from haystack import Document, Pipeline
 from haystack.components.builders import PromptBuilder
-from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
+from haystack.components.embedders import OpenAIDocumentEmbedder, OpenAITextEmbedder
 from haystack.components.generators import OpenAIGenerator
 from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils.auth import Secret
 
 from apify_haystack import ApifyDatasetFromActorCall
 
-# Load OPENAI_API_KEY and APIFY_API_TOKEN from .env file
+# Set APIFY-API-TOKEN here or use it from .env file
 load_dotenv()
+apify_token = "" or os.getenv("APIFY_API_TOKEN")
+openai_api_key = "" or os.getenv("OPENAI_API_KEY")
 
 actor_id = "apify/website-content-crawler"
 run_input = {
@@ -50,12 +52,11 @@ apify_dataset_loader = ApifyDatasetFromActorCall(
 )
 
 # Components
+print("Initializing components...")
 document_store = InMemoryDocumentStore()
 
-docs_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
-docs_embedder.warm_up()
-text_embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
-text_embedder.warm_up()
+docs_embedder = OpenAIDocumentEmbedder(api_key=Secret.from_token(openai_api_key))
+text_embedder = OpenAITextEmbedder(api_key=Secret.from_token(openai_api_key))
 retriever = InMemoryEmbeddingRetriever(document_store)
 generator = OpenAIGenerator(model="gpt-3.5-turbo")
 
@@ -79,6 +80,7 @@ Answer:
 prompt_builder = PromptBuilder(template=template)
 
 # Add components to your pipeline
+print("Initializing pipeline...")
 pipe = Pipeline()
 pipe.add_component("embedder", text_embedder)
 pipe.add_component("retriever", retriever)
@@ -92,6 +94,9 @@ pipe.connect("prompt_builder", "llm")
 
 question = "What is haystack?"
 
+print("Running pipeline ... crawling will take some time ...")
+print("You can visit https://console.apify.com/actors/runs to monitor the progress")
+
 response = pipe.run({"embedder": {"text": question}, "prompt_builder": {"question": question}})
 
 print(f"question: {question}")
@@ -105,4 +110,5 @@ examples = [
 
 for example in examples:
     response = pipe.run({"embedder": {"text": example}, "prompt_builder": {"question": example}})
-    print(response["llm"]["replies"][0])
+    print(f"question: {question}")
+    print(f"answer: {response['llm']['replies'][0]}")

apify_haystack-0.0.1a3/README.md

@@ -1,36 +0,0 @@
-# Apify-Haystack integration
-
-Apify-Haystack integration allows to quickly fetch results from Apify Actors.
-
-🚧 **Work In Progress** 🚧
-
-This project is currently under active development.
-
-## Installation
-
-Apify-haystack is available as the [`apify-haystack`](https://pypi.org/project/apify-haystack/) PyPI package.
-
-```sh
-pip install apify-haystack
-```
-
-## Examples
-
-Here are some practical examples to help you get started ...
-
-## Features
-
-## Support
-
-If you find any bug or issue, please [submit an issue on GitHub](https://github.com/apify/apify-haystack/issues).
-For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questions/tagged/apify), in GitHub Discussions or you can join our [Discord server](https://discord.com/invite/jyEM2PRvMU).
-
-## Contributing
-
-Your code contributions are welcome, and you'll be praised for eternity!
-If you have any ideas for improvements, either submit an issue or create a pull request.
-For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/apify-haystack/blob/master/CONTRIBUTING.md).
-
-## License
-
-This project is licensed under the Apache License 2.0 - see the [LICENSE](https://github.com/apify/apify-haystack/blob/master/LICENSE) file for details.