apify-haystack 0.0.1a3__tar.gz → 0.0.1a4__tar.gz

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify_haystack
-Version: 0.0.1a3
+Version: 0.0.1a4
 Summary: Apify-haystack integration
 Home-page: https://apify.com
 License: Apache-2.0

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify_haystack"
-version = "0.0.1a3"
+version = "0.0.1a4"
 description = "Apify-haystack integration"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 homepage = "https://apify.com"

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a4}/src/apify_haystack/examples/crawl_and_process_data.py

@@ -14,17 +14,23 @@ meta: {'url': 'https://haystack.deepset.ai/overview/quick-start',
 .....
 """
 
+import os
+
 from dotenv import load_dotenv
 from haystack import Document, Pipeline
-from haystack.components.embedders import SentenceTransformersDocumentEmbedder
+from haystack.components.embedders import OpenAIDocumentEmbedder
 from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
 from haystack.components.writers import DocumentWriter
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils.auth import Secret
 
 from apify_haystack import ApifyDatasetFromActorCall
 
-# Sey APIFY-API-TOKEN here or load it from .env file
-apify_token = "" or load_dotenv()
+load_dotenv()
+
+# Set APIFY-API-TOKEN here or use it from .env file
+apify_token = "" or os.getenv("APIFY_API_TOKEN")
+openai_api_key = "" or os.getenv("OPENAI_API_KEY")
 
 actor_id = "apify/website-content-crawler"
 run_input = {
@@ -46,7 +52,7 @@ print(f"Initialized InMemoryDocumentStore with {document_store.count_documents()
 
 document_cleaner = DocumentCleaner()
 document_splitter = DocumentSplitter(split_by="word", split_length=150, split_overlap=50)
-document_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
+document_embedder = OpenAIDocumentEmbedder(api_key=Secret.from_token(openai_api_key))
 document_writer = DocumentWriter(document_store)
 
 pipe = Pipeline()
@@ -72,6 +78,7 @@ pipe.run({"document_loader": {}})
 
 print(f"Added {document_store.count_documents()} to vector from Website Content Crawler")
 
-print("Retrieving documents from the document store: query='Haystack'")
+print("\nRetrieving documents from the document store: query='Haystack'")
+print("query='Haystack'\n")
 for doc in document_store.bm25_retrieval("Haystack", top_k=1):
     print(doc)

{apify_haystack-0.0.1a3 → apify_haystack-0.0.1a4}/src/apify_haystack/examples/rag_with_crawled_website.py

@@ -21,17 +21,22 @@ Haystack is an open-source framework for building production-ready LLM applicati
 ......
 """
 
+import os
+
 from dotenv import load_dotenv
 from haystack import Document, Pipeline
 from haystack.components.builders import PromptBuilder
-from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
+from haystack.components.embedders import OpenAIDocumentEmbedder, OpenAITextEmbedder
 from haystack.components.generators import OpenAIGenerator
 from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils.auth import Secret
 
 from apify_haystack import ApifyDatasetFromActorCall
 
-# Load OPENAI_API_KEY and APIFY_API_TOKEN from .env file
+# Set APIFY-API-TOKEN here or use it from .env file
+apify_token = "" or os.getenv("APIFY_API_TOKEN")
+openai_api_key = "" or os.getenv("OPENAI_API_KEY")
 load_dotenv()
 
 actor_id = "apify/website-content-crawler"
@@ -52,10 +57,8 @@ apify_dataset_loader = ApifyDatasetFromActorCall(
 # Components
 document_store = InMemoryDocumentStore()
 
-docs_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
-docs_embedder.warm_up()
-text_embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
-text_embedder.warm_up()
+docs_embedder = OpenAIDocumentEmbedder(api_key=Secret.from_token(openai_api_key))
+text_embedder = OpenAITextEmbedder(api_key=Secret.from_token(openai_api_key))
 retriever = InMemoryEmbeddingRetriever(document_store)
 generator = OpenAIGenerator(model="gpt-3.5-turbo")