apify-haystack 0.0.1a5__tar.gz → 0.1.0a1__tar.gz

{apify_haystack-0.0.1a5 → apify_haystack-0.1.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify_haystack
-Version: 0.0.1a5
+Version: 0.1.0a1
 Summary: Apify-haystack integration
 Home-page: https://apify.com
 License: Apache-2.0
@@ -46,10 +46,47 @@ pip install apify-haystack
 
 ## Examples
 
-- Load a dataset from Apify and convert it to Haystack Documents: [apify_dataset_load.py](src/apify_haystack/examples/apify_dataset_load.py)
-- Call Apify Actor and load a dataset to convert it to Haystack Documents: [apify_actor_call.py](src/apify_haystack/examples/apify_actor_call.py)
-- Crawl website, scrape text content, and store it in the InMemoryDocumentStore: [crawl_and_process_data.py](src/apify_haystack/examples/crawl_and_process_data.py)
-- Retrieval-Augmented Generation (RAG): Extracting text content from a website and using it for question answering [rag_with_crawled_website.py](src/apify_haystack/examples/rag_with_crawled_website.py)
+Crawl website using Apify Website Content Crawler and convert it to Haystack Documents
+
+```python
+from dotenv import load_dotenv
+from haystack import Document
+
+from apify_haystack import ApifyDatasetFromActorCall
+
+# Sey APIFY-API-TOKEN here or load it from .env file
+apify_token = "" or load_dotenv()
+
+actor_id = "apify/website-content-crawler"
+run_input = {
+    "maxCrawlPages": 3,  # limit the number of pages to crawl
+    "startUrls": [{"url": "https://haystack.deepset.ai/"}],
+}
+
+
+def dataset_mapping_function(dataset_item: dict) -> Document:
+    return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+
+actor = ApifyDatasetFromActorCall(
+    actor_id=actor_id, run_input=run_input, dataset_mapping_function=dataset_mapping_function
+)
+print(f"Calling the Apify actor {actor_id} ... crawling will take some time ...")
+print("You can monitor the progress at: https://console.apify.com/actors/runs")
+
+dataset = actor.run().get("documents")
+
+print(f"Loaded {len(dataset)} documents from the Apify Actor {actor_id}:")
+for d in dataset:
+    print(d)
+```
+
+See other examples in the [examples directory](https://github.com/apify/apify-haystack/blob/master/src/apify_haystack/examples) for more examples, here is a list of few of them
+
+- Load a dataset from Apify and convert it to Haystack Documents
+- Call Apify Actor and load a dataset to convert it to Haystack Documents
+- Crawl website, scrape text content, and store it in the InMemoryDocumentStore
+- Retrieval-Augmented Generation (RAG): Extracting text from a website & question answering
 
 ## Support
 
@@ -58,7 +95,7 @@ For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questio
 
 ## Contributing
 
-Your code contributions are welcome, and you'll be praised for eternity!
+Your code contributions are welcome.
 If you have any ideas for improvements, either submit an issue or create a pull request.
 For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/apify-haystack/blob/master/CONTRIBUTING.md).
 

{apify_haystack-0.0.1a5 → apify_haystack-0.1.0a1}/README.md

@@ -17,10 +17,47 @@ pip install apify-haystack
 
 ## Examples
 
-- Load a dataset from Apify and convert it to Haystack Documents: [apify_dataset_load.py](src/apify_haystack/examples/apify_dataset_load.py)
-- Call Apify Actor and load a dataset to convert it to Haystack Documents: [apify_actor_call.py](src/apify_haystack/examples/apify_actor_call.py)
-- Crawl website, scrape text content, and store it in the InMemoryDocumentStore: [crawl_and_process_data.py](src/apify_haystack/examples/crawl_and_process_data.py)
-- Retrieval-Augmented Generation (RAG): Extracting text content from a website and using it for question answering [rag_with_crawled_website.py](src/apify_haystack/examples/rag_with_crawled_website.py)
+Crawl website using Apify Website Content Crawler and convert it to Haystack Documents
+
+```python
+from dotenv import load_dotenv
+from haystack import Document
+
+from apify_haystack import ApifyDatasetFromActorCall
+
+# Sey APIFY-API-TOKEN here or load it from .env file
+apify_token = "" or load_dotenv()
+
+actor_id = "apify/website-content-crawler"
+run_input = {
+    "maxCrawlPages": 3,  # limit the number of pages to crawl
+    "startUrls": [{"url": "https://haystack.deepset.ai/"}],
+}
+
+
+def dataset_mapping_function(dataset_item: dict) -> Document:
+    return Document(content=dataset_item.get("text"), meta={"url": dataset_item.get("url")})
+
+
+actor = ApifyDatasetFromActorCall(
+    actor_id=actor_id, run_input=run_input, dataset_mapping_function=dataset_mapping_function
+)
+print(f"Calling the Apify actor {actor_id} ... crawling will take some time ...")
+print("You can monitor the progress at: https://console.apify.com/actors/runs")
+
+dataset = actor.run().get("documents")
+
+print(f"Loaded {len(dataset)} documents from the Apify Actor {actor_id}:")
+for d in dataset:
+    print(d)
+```
+
+See other examples in the [examples directory](https://github.com/apify/apify-haystack/blob/master/src/apify_haystack/examples) for more examples, here is a list of few of them
+
+- Load a dataset from Apify and convert it to Haystack Documents
+- Call Apify Actor and load a dataset to convert it to Haystack Documents
+- Crawl website, scrape text content, and store it in the InMemoryDocumentStore
+- Retrieval-Augmented Generation (RAG): Extracting text from a website & question answering
 
 ## Support
 
@@ -29,7 +66,7 @@ For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questio
 
 ## Contributing
 
-Your code contributions are welcome, and you'll be praised for eternity!
+Your code contributions are welcome.
 If you have any ideas for improvements, either submit an issue or create a pull request.
 For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/apify-haystack/blob/master/CONTRIBUTING.md).
 

{apify_haystack-0.0.1a5 → apify_haystack-0.1.0a1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify_haystack"
-version = "0.0.1a5"
+version = "0.1.0a1"
 description = "Apify-haystack integration"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 homepage = "https://apify.com"

{apify_haystack-0.0.1a5 → apify_haystack-0.1.0a1}/src/apify_haystack/examples/apify_actor_call.py

@@ -36,7 +36,9 @@ def dataset_mapping_function(dataset_item: dict) -> Document:
 actor = ApifyDatasetFromActorCall(
     actor_id=actor_id, run_input=run_input, dataset_mapping_function=dataset_mapping_function
 )
-print(f"Calling Apify actor {actor_id} ... crawling will take some time ...")
+print(f"Calling the Apify actor {actor_id} ... crawling will take some time ...")
+print("You can monitor the progress at: https://console.apify.com/actors/runs")
+
 dataset = actor.run().get("documents")
 
 print(f"Loaded {len(dataset)} documents from the Apify Actor {actor_id}:")

apify_haystack-0.0.1a5/src/apify_haystack/main.py

@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-
-def main() -> None:
-    """Main function."""