gooddata-flexconnect 1.41.1.dev1__tar.gz → 1.41.1.dev3__tar.gz

{gooddata_flexconnect-1.41.1.dev1 → gooddata_flexconnect-1.41.1.dev3}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: gooddata-flexconnect
-Version: 1.41.1.dev1
+Version: 1.41.1.dev3
 Summary: Build your own data source for GoodData Cloud and GoodData Cloud Native.
 Author: GoodData
 Author-email: support@gooddata.com
 License: MIT
-Project-URL: Documentation, https://gooddata-flexconnect.readthedocs.io/en/v1.41.1.dev1
+Project-URL: Documentation, https://gooddata-flexconnect.readthedocs.io/en/v1.41.1.dev3
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Keywords: gooddata,flight,rpc,flight rpc,custom functions,analytics,headless,business,intelligence,headless-bi,cloud,native,semantic,layer,sql,metrics
 Classifier: Development Status :: 4 - Beta
@@ -24,8 +24,8 @@ Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: dynaconf<4.0.0,>=3.1.11
-Requires-Dist: gooddata-flight-server~=1.41.1.dev1
-Requires-Dist: gooddata-sdk~=1.41.1.dev1
+Requires-Dist: gooddata-flight-server~=1.41.1.dev3
+Requires-Dist: gooddata-sdk~=1.41.1.dev3
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pyarrow>=16.1.0
 Requires-Dist: structlog<25.0.0,>=24.0.0

{gooddata_flexconnect-1.41.1.dev1 → gooddata_flexconnect-1.41.1.dev3}/gooddata_flexconnect/__init__.py

@@ -1,5 +1,10 @@
 # (C) 2024 GoodData Corporation
 
+from gooddata_flexconnect.function.data_source_messages import (
+    DataSourceMessage,
+    add_data_source_messages_metadata,
+    with_data_source_messages,
+)
 from gooddata_flexconnect.function.execution_context import (
     ExecutionContext,
     ExecutionContextAbsoluteDateFilter,

gooddata_flexconnect-1.41.1.dev3/gooddata_flexconnect/function/data_source_messages.py

@@ -0,0 +1,101 @@
+# (C) 2025 GoodData Corporation
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import orjson
+import pyarrow
+
+_DATA_SOURCE_MESSAGES_KEY = b"x-gdc-data-source-messages"
+"""
+Key used in the PyArrow metadata to carry the data source messages.
+"""
+
+
+@dataclass(frozen=True)
+class DataSourceMessage:
+    """
+    A message sent by the data source. This will be included in the execution results' metadata.
+    """
+
+    source: str
+    """
+    Identification of the source of the message, typically the id or other identifier of the data source.
+    """
+
+    correlation_id: str
+    """
+    Unique id of the operation, meant to correlate different messages together.
+    """
+
+    type: str
+    """
+    Type of the message, currently free-form, we might define some enum for these in the future.
+    """
+
+    data: Optional[Any] = None
+    """
+    Optional message-specific data. This can be anything that can be JSON-serialized.
+    Try to keep this as small as possible: the backend has a quite strict size limit on the messages.
+    """
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Converts this instance to its dictionary representation.
+        """
+        return {"source": self.source, "correlation_id": self.correlation_id, "type": self.type, "data": self.data}
+
+
+def add_data_source_messages_metadata(
+    data_source_messages: Iterable[DataSourceMessage], original_metadata: Optional[dict] = None
+) -> dict[bytes, bytes]:
+    """
+    Given a list of DataSourceMessages, creates a PyArrow-compatible metadata dictionary.
+
+    This is useful when creating a PyArrow Table directly:
+
+    >>> t = pyarrow.table(
+    ...     [1, 2, 3],
+    ...     ["test"],
+    ...     metadata=add_data_source_messages_metadata(
+    ...         [DataSourceMessage(correlation_id="123", source="test_source", type="info")]
+    ...     ),
+    ... )
+
+    You can also add other metadata as needed:
+    >>> t2 = pyarrow.table(
+    ...     [1, 2, 3],
+    ...     ["test"],
+    ...     metadata=add_data_source_messages_metadata(
+    ...         [DataSourceMessage(correlation_id="123", source="test_source", type="info")],
+    ...         original_metadata={b"some": b"extra metadata"},
+    ...     ),
+    ... )
+
+    :param data_source_messages: list of DataSourceMessages to include
+    :param original_metadata: original medata to add the DataSourceMessages to.
+    :return: a new metadata dictionary
+    """
+    if original_metadata is None:
+        original_metadata = {}
+    return {**original_metadata, _DATA_SOURCE_MESSAGES_KEY: orjson.dumps(data_source_messages)}
+
+
+def with_data_source_messages(
+    data_source_messages: Iterable[DataSourceMessage], schema: pyarrow.Schema
+) -> pyarrow.Schema:
+    """
+    Given a schema and a list of DataSourceMessages, returns a new schema with the DataSourceMessages included.
+
+    This is useful when creating PyArrow RecordBatchReaders:
+
+    >>> s = with_data_source_messages(
+    ...     [DataSourceMessage(correlation_id="123", source="test_source", type="info")],
+    ...     pyarrow.schema(...),  # the original schema for your RecordBatchReader
+    ... )
+    >>> rbr = pyarrow.RecordBatchReader.from_batches(schema=s, batches=...)
+
+    :param schema: the schema to enrich with the DataSourceMessages
+    :param data_source_messages: list of DataSourceMessages to include
+    """
+    return schema.with_metadata(add_data_source_messages_metadata(data_source_messages, schema.metadata))

{gooddata_flexconnect-1.41.1.dev1 → gooddata_flexconnect-1.41.1.dev3}/gooddata_flexconnect.egg-info/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: gooddata-flexconnect
-Version: 1.41.1.dev1
+Version: 1.41.1.dev3
 Summary: Build your own data source for GoodData Cloud and GoodData Cloud Native.
 Author: GoodData
 Author-email: support@gooddata.com
 License: MIT
-Project-URL: Documentation, https://gooddata-flexconnect.readthedocs.io/en/v1.41.1.dev1
+Project-URL: Documentation, https://gooddata-flexconnect.readthedocs.io/en/v1.41.1.dev3
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Keywords: gooddata,flight,rpc,flight rpc,custom functions,analytics,headless,business,intelligence,headless-bi,cloud,native,semantic,layer,sql,metrics
 Classifier: Development Status :: 4 - Beta
@@ -24,8 +24,8 @@ Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: dynaconf<4.0.0,>=3.1.11
-Requires-Dist: gooddata-flight-server~=1.41.1.dev1
-Requires-Dist: gooddata-sdk~=1.41.1.dev1
+Requires-Dist: gooddata-flight-server~=1.41.1.dev3
+Requires-Dist: gooddata-sdk~=1.41.1.dev3
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pyarrow>=16.1.0
 Requires-Dist: structlog<25.0.0,>=24.0.0

{gooddata_flexconnect-1.41.1.dev1 → gooddata_flexconnect-1.41.1.dev3}/gooddata_flexconnect.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ gooddata_flexconnect.egg-info/dependency_links.txt
 gooddata_flexconnect.egg-info/requires.txt
 gooddata_flexconnect.egg-info/top_level.txt
 gooddata_flexconnect/function/__init__.py
+gooddata_flexconnect/function/data_source_messages.py
 gooddata_flexconnect/function/execution_context.py
 gooddata_flexconnect/function/flight_methods.py
 gooddata_flexconnect/function/function.py

{gooddata_flexconnect-1.41.1.dev1 → gooddata_flexconnect-1.41.1.dev3}/gooddata_flexconnect.egg-info/requires.txt

@@ -1,6 +1,6 @@
 dynaconf<4.0.0,>=3.1.11
-gooddata-flight-server~=1.41.1.dev1
-gooddata-sdk~=1.41.1.dev1
+gooddata-flight-server~=1.41.1.dev3
+gooddata-sdk~=1.41.1.dev3
 orjson<4.0.0,>=3.9.15
 pyarrow>=16.1.0
 structlog<25.0.0,>=24.0.0

{gooddata_flexconnect-1.41.1.dev1 → gooddata_flexconnect-1.41.1.dev3}/setup.py

@@ -8,8 +8,8 @@ long_description = (this_directory / "README.md").read_text(encoding="utf-8")
 
 REQUIRES = [
     "dynaconf>=3.1.11,<4.0.0",
-    "gooddata-flight-server~=1.41.1.dev1",
-    "gooddata-sdk~=1.41.1.dev1",
+    "gooddata-flight-server~=1.41.1.dev3",
+    "gooddata-sdk~=1.41.1.dev3",
     "orjson>=3.9.15,<4.0.0",
     "pyarrow>=16.1.0",
     "structlog>=24.0.0,<25.0.0",
@@ -20,7 +20,7 @@ setup(
     description="Build your own data source for GoodData Cloud and GoodData Cloud Native.",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    version="1.41.1.dev1",
+    version="1.41.1.dev3",
     author="GoodData",
     author_email="support@gooddata.com",
     license="MIT",
@@ -31,7 +31,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9.0",
     project_urls={
-        "Documentation": "https://gooddata-flexconnect.readthedocs.io/en/v1.41.1.dev1",
+        "Documentation": "https://gooddata-flexconnect.readthedocs.io/en/v1.41.1.dev3",
         "Source": "https://github.com/gooddata/gooddata-python-sdk",
     },
     classifiers=[