apify-haystack 0.1.8b1__tar.gz → 0.1.8b2__tar.gz

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/CHANGELOG.md

@@ -28,6 +28,7 @@ All notable changes to this project will be documented in this file.
 ### 🚜 Refactor
 
 - [**breaking**] Make the constants module private ([#68](https://github.com/apify/apify-haystack/pull/68)) ([8f5dbbd](https://github.com/apify/apify-haystack/commit/8f5dbbd7a4fa0e8c3c7c8036799d40744ee47624)) by [@vdusek](https://github.com/vdusek)
+- [**breaking**] Make all modules private and export the public API from the package root ([#77](https://github.com/apify/apify-haystack/pull/77)) ([4e6ccca](https://github.com/apify/apify-haystack/commit/4e6cccae287999f9ceb72fe9b67c2305b1206113)) by [@vdusek](https://github.com/vdusek)
 
 ### ⚙️ Miscellaneous Tasks
 

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/CONTRIBUTING.md

@@ -32,6 +32,8 @@ We use [ruff](https://docs.astral.sh/ruff/) for linting and formatting, [ty](htt
 
 Unit tests (`tests/unit`) are isolated from the Apify API. Run them with `uv run poe unit-tests`, or a single test with `uv run pytest tests/unit/<path>.py::<test_name>`.
 
+Integration tests (`tests/integration`) run the components against the Apify platform, so they need an Apify API token in the `APIFY_TOKEN` environment variable and are skipped without one. Run them with `uv run poe integration-tests`. They run real Actors on that account, which costs a little. CI runs them on pull requests from this repository and on pushes to `main`, with the test user's token.
+
 ## Documentation
 
 We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify_haystack
-Version: 0.1.8b1
+Version: 0.1.8b2
 Summary: Apify-haystack integration
 Keywords: apify,crawler,haystack,rag,scraper,scraping
 Author: Apify Technologies s.r.o.

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify_haystack"
-version = "0.1.8b1"
+version = "0.1.8b2"
 description = "Apify-haystack integration"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -191,6 +191,14 @@ help = "Run unit tests"
 cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify_haystack --cov-report=xml:coverage-unit.xml tests/unit"
 help = "Run unit tests with an XML coverage report"
 
+[tool.poe.tasks.integration-tests]
+cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/integration"
+help = "Run integration tests against the Apify platform"
+
+[tool.poe.tasks.integration-tests-cov]
+cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify_haystack --cov-report=xml:coverage-integration.xml tests/integration"
+help = "Run integration tests with an XML coverage report"
+
 [tool.poe.tasks.check-code]
 sequence = [
     "lint",

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify_haystack"
-version = "0.1.8b1"
+version = "0.1.8b2"
 description = "Apify-haystack integration"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = "Apache-2.0"
@@ -156,6 +156,8 @@ build = { cmd = "uv build", help = "Build the package" }
 type-check = { cmd = "uv run ty check", help = "Run the type checker" }
 unit-tests = { cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit", help = "Run unit tests" }
 unit-tests-cov = { cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify_haystack --cov-report=xml:coverage-unit.xml tests/unit", help = "Run unit tests with an XML coverage report" }
+integration-tests = { cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/integration", help = "Run integration tests against the Apify platform" }
+integration-tests-cov = { cmd = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify_haystack --cov-report=xml:coverage-integration.xml tests/integration", help = "Run integration tests with an XML coverage report" }
 check-code = { sequence = ["lint", "type-check", "unit-tests"], help = "Run lint, type check, and unit tests" }
 
 [tool.poe.tasks.install-dev]

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/src/apify_haystack/__init__.py

@@ -2,14 +2,14 @@ from importlib import metadata
 
 __version__ = metadata.version('apify-haystack')
 
-from apify_haystack.actor_tool import ApifyActorTool
-from apify_haystack.apify_dataset import (
+from apify_haystack._actor_tool import ApifyActorTool
+from apify_haystack._dataset import (
     ApifyDatasetFromActorCall,
     ApifyDatasetFromTaskCall,
     ApifyDatasetLoader,
     default_dataset_mapping_function,
 )
-from apify_haystack.web_search import ApifyWebSearch
+from apify_haystack._web_search import ApifyWebSearch
 
 __all__ = [
     'ApifyActorTool',

apify_haystack-0.1.8b1/src/apify_haystack/actor_tool.py → apify_haystack-0.1.8b2/src/apify_haystack/_actor_tool.py

@@ -10,11 +10,11 @@ from haystack.tools import Tool
 from haystack.utils import Secret, deserialize_secrets_inplace
 
 from apify_haystack._client import (
-    _ApifyClientComponent,
-    _create_client,
-    _get_dataset_id,
-    _to_decimal,
-    _to_timedelta,
+    ApifyClientComponent,
+    create_client,
+    get_dataset_id,
+    to_decimal,
+    to_timedelta,
 )
 
 # OpenAI limits a function name to 64 characters and a description to 1024.
@@ -30,15 +30,16 @@ UNTRUSTED_CONTENT_END = '<<<END_EXTERNAL_UNTRUSTED_CONTENT>>>'
 _PROPERTY_SCHEMA_KEYS = ('type', 'description', 'enum', 'default', 'items', 'minimum', 'maximum')
 
 
-class ApifyActorTool(_ApifyClientComponent, Tool):
+class ApifyActorTool(ApifyClientComponent, Tool):
     """A Haystack tool that runs an Apify Actor and returns the items of its default dataset.
 
     Use it to give a Haystack `Agent` access to any Actor from Apify Store. The tool takes its parameters from the
     Actor's input schema and its description from the Actor's definition, so the LLM sees the same inputs as a user
     of the Actor. Reading them needs one Apify API request when the tool is created without `parameters`.
 
-    The LLM gets the dataset items as JSON between the `UNTRUSTED_CONTENT_START` and `UNTRUSTED_CONTENT_END` markers,
-    since scraped content can carry prompt injections. `invoke` and `invoke_async` return the items unwrapped.
+    The LLM gets the dataset items as JSON between the `<<<EXTERNAL_UNTRUSTED_CONTENT>>>` and
+    `<<<END_EXTERNAL_UNTRUSTED_CONTENT>>>` markers, since scraped content can carry prompt injections. `invoke` and
+    `invoke_async` return the items unwrapped.
     For details, see https://docs.apify.com/platform/integrations/haystack
 
     Example:
@@ -53,6 +54,9 @@ class ApifyActorTool(_ApifyClientComponent, Tool):
             agent = Agent(chat_generator=OpenAIChatGenerator(), tools=[tool])
     """
 
+    # Serialized pipelines store the class by this public path, so it does not depend on the module layout.
+    __module__ = 'apify_haystack'
+
     def __init__(
         self,
         actor_id: str,
@@ -152,10 +156,10 @@ class ApifyActorTool(_ApifyClientComponent, Tool):
             run_input=run_input,
             build=self.build,
             memory_mbytes=self.memory_mbytes,
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'Actor {self.actor_id}')
+        dataset_id = get_dataset_id(run, f'Actor {self.actor_id}')
         return client.dataset(dataset_id).list_items(clean=True, limit=self.dataset_items_limit).items
 
     async def _run_actor_async(self, **run_input: Any) -> list[dict]:
@@ -164,15 +168,15 @@ class ApifyActorTool(_ApifyClientComponent, Tool):
             run_input=run_input,
             build=self.build,
             memory_mbytes=self.memory_mbytes,
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'Actor {self.actor_id}')
+        dataset_id = get_dataset_id(run, f'Actor {self.actor_id}')
         return (await client.dataset(dataset_id).list_items(clean=True, limit=self.dataset_items_limit)).items
 
 
 def _get_actor_definition(apify_token: Secret, actor_id: str) -> dict[str, Any]:
-    client = _create_client(ApifyClient, apify_token.resolve_value())
+    client = create_client(ApifyClient, apify_token.resolve_value())
     try:
         build = client.actor(actor_id).default_build().get()
     finally:

{apify_haystack-0.1.8b1 → apify_haystack-0.1.8b2}/src/apify_haystack/_client.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
 _ClientT = TypeVar('_ClientT', ApifyClient, ApifyClientAsync)
 
 
-class _ApifyClientComponent:
+class ApifyClientComponent:
     """Base for the components that call the Apify API.
 
     The sync client lives between `warm_up` and `close`, the async one between `warm_up_async` and `close_async`.
@@ -47,16 +47,16 @@ class _ApifyClientComponent:
 
     def _get_client(self) -> ApifyClient:
         if self.client is None:
-            self.client = _create_client(ApifyClient, self.apify_token.resolve_value())
+            self.client = create_client(ApifyClient, self.apify_token.resolve_value())
         return self.client
 
     def _get_async_client(self) -> ApifyClientAsync:
         if self.async_client is None:
-            self.async_client = _create_client(ApifyClientAsync, self.apify_token.resolve_value())
+            self.async_client = create_client(ApifyClientAsync, self.apify_token.resolve_value())
         return self.async_client
 
 
-def _create_client(client_class: type[_ClientT], token: str | None) -> _ClientT:
+def create_client(client_class: type[_ClientT], token: str | None) -> _ClientT:
     """Create an Apify client that identifies its requests as coming from this integration."""
     client = client_class(token, headers={'x-apify-integration-platform': HAYSTACK_INTEGRATION_PLATFORM})
 
@@ -66,7 +66,7 @@ def _create_client(client_class: type[_ClientT], token: str | None) -> _ClientT:
     return client
 
 
-def _get_dataset_id(run: Run | None, label: str) -> str:
+def get_dataset_id(run: Run | None, label: str) -> str:
     """Return the default dataset of a finished run, raising unless the run succeeded."""
     if run is None:
         raise ValueError(f'No response found in the {label} call')
@@ -75,10 +75,10 @@ def _get_dataset_id(run: Run | None, label: str) -> str:
     return run.default_dataset_id
 
 
-def _to_timedelta(seconds: int | None) -> timedelta | None:
+def to_timedelta(seconds: int | None) -> timedelta | None:
     return None if seconds is None else timedelta(seconds=seconds)
 
 
-def _to_decimal(value: float | None) -> Decimal | None:
+def to_decimal(value: float | None) -> Decimal | None:
     # `str` keeps the amount as written; `Decimal(0.1)` would carry the binary float error.
     return None if value is None else Decimal(str(value))

apify_haystack-0.1.8b1/src/apify_haystack/apify_dataset.py → apify_haystack-0.1.8b2/src/apify_haystack/_dataset.py

@@ -7,7 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.utils import Secret, deserialize_callable, serialize_callable
 
-from apify_haystack._client import _ApifyClientComponent, _get_dataset_id, _to_decimal, _to_timedelta
+from apify_haystack._client import ApifyClientComponent, get_dataset_id, to_decimal, to_timedelta
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
     from apify_client import ApifyClient, ApifyClientAsync
 
 
-class _ApifyDatasetComponent(_ApifyClientComponent):
+class ApifyDatasetComponent(ApifyClientComponent):
     """Base for the components that convert dataset items to Documents with a mapping function."""
 
     @classmethod
@@ -29,7 +29,7 @@ class _ApifyDatasetComponent(_ApifyClientComponent):
 
 
 @component
-class ApifyDatasetLoader(_ApifyDatasetComponent):
+class ApifyDatasetLoader(ApifyDatasetComponent):
     """Load datasets produced by Apify Actors.
 
     Actors are serverless functions that can be used to scrape, crawl, and extract data from the web.
@@ -49,6 +49,9 @@ class ApifyDatasetLoader(_ApifyDatasetComponent):
             dataset = loader.run()
     """
 
+    # Serialized pipelines store the class by this public path, so it does not depend on the module layout.
+    __module__ = 'apify_haystack'
+
     def __init__(
         self,
         dataset_id: str,
@@ -95,7 +98,7 @@ class ApifyDatasetLoader(_ApifyDatasetComponent):
 
 
 @component
-class ApifyDatasetFromActorCall(_ApifyDatasetComponent):
+class ApifyDatasetFromActorCall(ApifyDatasetComponent):
     """Get an Apify dataset by calling an Apify Actor.
 
     Perform a call of an Apify Actor and load the produced dataset.
@@ -105,6 +108,9 @@ class ApifyDatasetFromActorCall(_ApifyDatasetComponent):
     `apify_token`.
     """
 
+    # Serialized pipelines store the class by this public path, so it does not depend on the module layout.
+    __module__ = 'apify_haystack'
+
     def __init__(
         self,
         actor_id: str,
@@ -174,10 +180,10 @@ class ApifyDatasetFromActorCall(_ApifyDatasetComponent):
             run_input=run_input,
             build=self.build,
             memory_mbytes=self.memory_mbytes,
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'Actor {self.actor_id}')
+        dataset_id = get_dataset_id(run, f'Actor {self.actor_id}')
         return {'documents': _load_documents(client, dataset_id, self.dataset_mapping_function)}
 
     @component.output_types(documents=list[Document])
@@ -197,10 +203,10 @@ class ApifyDatasetFromActorCall(_ApifyDatasetComponent):
             run_input=run_input,
             build=self.build,
             memory_mbytes=self.memory_mbytes,
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'Actor {self.actor_id}')
+        dataset_id = get_dataset_id(run, f'Actor {self.actor_id}')
         documents = await _load_documents_async(client, dataset_id, self.dataset_mapping_function)
         return {'documents': documents}
 
@@ -220,7 +226,7 @@ class ApifyDatasetFromActorCall(_ApifyDatasetComponent):
 
 
 @component
-class ApifyDatasetFromTaskCall(_ApifyDatasetComponent):
+class ApifyDatasetFromTaskCall(ApifyDatasetComponent):
     """Get an Apify dataset by calling an Apify task.
 
     Perform a call of an Apify task and load the produced dataset.
@@ -230,6 +236,9 @@ class ApifyDatasetFromTaskCall(_ApifyDatasetComponent):
     `apify_token`.
     """
 
+    # Serialized pipelines store the class by this public path, so it does not depend on the module layout.
+    __module__ = 'apify_haystack'
+
     def __init__(
         self,
         task_id: str,
@@ -289,10 +298,10 @@ class ApifyDatasetFromTaskCall(_ApifyDatasetComponent):
             task_input=task_input,
             build=self.build,
             memory_mbytes=self.memory_mbytes,
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'task {self.task_id}')
+        dataset_id = get_dataset_id(run, f'task {self.task_id}')
         return {'documents': _load_documents(client, dataset_id, self.dataset_mapping_function)}
 
     @component.output_types(documents=list[Document])
@@ -312,10 +321,10 @@ class ApifyDatasetFromTaskCall(_ApifyDatasetComponent):
             task_input=task_input,
             build=self.build,
             memory_mbytes=self.memory_mbytes,
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'task {self.task_id}')
+        dataset_id = get_dataset_id(run, f'task {self.task_id}')
         documents = await _load_documents_async(client, dataset_id, self.dataset_mapping_function)
         return {'documents': documents}
 

apify_haystack-0.1.8b1/src/apify_haystack/web_search.py → apify_haystack-0.1.8b2/src/apify_haystack/_web_search.py

@@ -6,8 +6,8 @@ from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.utils import Secret
 
-from apify_haystack._client import _ApifyClientComponent, _get_dataset_id, _to_decimal, _to_timedelta
-from apify_haystack.apify_dataset import default_dataset_mapping_function
+from apify_haystack._client import ApifyClientComponent, get_dataset_id, to_decimal, to_timedelta
+from apify_haystack._dataset import default_dataset_mapping_function
 
 RAG_WEB_BROWSER_ACTOR_ID = 'apify/rag-web-browser'
 
@@ -16,7 +16,7 @@ _MAX_TOP_K = 100
 
 
 @component
-class ApifyWebSearch(_ApifyClientComponent):
+class ApifyWebSearch(ApifyClientComponent):
     """Search the web with the Apify RAG Web Browser Actor and return the result pages as Documents.
 
     The Actor searches Google, scrapes the top results, and returns their content as Markdown. Each result becomes
@@ -33,6 +33,9 @@ class ApifyWebSearch(_ApifyClientComponent):
             documents, links = result['documents'], result['links']
     """
 
+    # Serialized pipelines store the class by this public path, so it does not depend on the module layout.
+    __module__ = 'apify_haystack'
+
     def __init__(
         self,
         top_k: int = 5,
@@ -75,10 +78,10 @@ class ApifyWebSearch(_ApifyClientComponent):
         client = self._get_client()
         run = client.actor(RAG_WEB_BROWSER_ACTOR_ID).call(
             run_input=self._run_input(query),
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'Actor {RAG_WEB_BROWSER_ACTOR_ID}')
+        dataset_id = get_dataset_id(run, f'Actor {RAG_WEB_BROWSER_ACTOR_ID}')
         items = client.dataset(dataset_id).list_items(clean=True).items
         return _to_output(items)
 
@@ -92,10 +95,10 @@ class ApifyWebSearch(_ApifyClientComponent):
         client = self._get_async_client()
         run = await client.actor(RAG_WEB_BROWSER_ACTOR_ID).call(
             run_input=self._run_input(query),
-            run_timeout=_to_timedelta(self.timeout_secs),
-            max_total_charge_usd=_to_decimal(self.max_total_charge_usd),
+            run_timeout=to_timedelta(self.timeout_secs),
+            max_total_charge_usd=to_decimal(self.max_total_charge_usd),
         )
-        dataset_id = _get_dataset_id(run, f'Actor {RAG_WEB_BROWSER_ACTOR_ID}')
+        dataset_id = get_dataset_id(run, f'Actor {RAG_WEB_BROWSER_ACTOR_ID}')
         items = (await client.dataset(dataset_id).list_items(clean=True)).items
         return _to_output(items)