another-ayx-python-sdk 2.4.2__py3-none-any.whl → 2.4.22__py3-none-any.whl

build/lib/build/lib/another_ayx_python_sdk/core/record_packet_base.py

@@ -0,0 +1,123 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Record Packet Base class definition."""
+import hashlib
+from abc import ABC, abstractmethod
+from typing import Any, TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from another_ayx_python_sdk.core.metadata import Metadata
+    import pandas as pd
+
+
+class RecordPacketBase(ABC):
+    """Abstract class that describes a record packet."""
+
+    @property
+    @abstractmethod
+    def metadata(self) -> "Metadata":
+        """
+        Get the packet metadata.
+
+        Returns
+        -------
+        Metadata
+            The metadata for records contained in the packet.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def to_dataframe(self) -> "pd.DataFrame":
+        """
+        Get the packet data as a dataframe.
+
+        Returns
+        -------
+        pd.DataFrame
+            The dataframe that contains all records in the packet.
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    @abstractmethod
+    def from_dataframe(
+        cls, metadata: "Metadata", df: "pd.DataFrame"
+    ) -> "RecordPacketBase":
+        """
+        Set the packet data from a dataframe.
+
+        Parameters
+        ----------
+        metadata
+            The metadata for the records.
+        df
+            The dataframe to generate records from.
+        """
+        raise NotImplementedError()
+
+    def __eq__(self, other: Any) -> bool:
+        """
+        Check equality between 2 packets.
+
+        Parameters
+        ----------
+        other
+            Other object to compare with this object.
+
+        Returns
+        -------
+        bool
+            True if metadata and all dataframes of this record packet have the same values as the other record packet.
+        """
+        if not isinstance(other, RecordPacketBase):
+            return NotImplemented
+
+        if self.metadata != other.metadata:
+            return False
+
+        import pandas as pd
+
+        try:
+            pd.testing.assert_frame_equal(self.to_dataframe(), other.to_dataframe())
+        except AssertionError:
+            return False
+        else:
+            return True
+
+    def to_readable_string(self) -> str:
+        """
+        Return the string representation of a record packet, with a human-readable pandas dataframe.
+
+        Returns
+        -------
+        str
+            Human-readable form of the record packet.
+        """
+        return f"Metadata: {self.metadata}\nDataframe: {self.to_dataframe()}"
+
+    def to_anonymized_string(self) -> str:
+        """
+        Return the string representation of a record packet, anonymizing the data within before conversion.
+
+        Returns
+        -------
+        str
+            Unique identifier generated from the data within a record packet.
+        """
+        return f"Metadata: {self.metadata}\nDataframe dimensions: {self.to_dataframe().shape}\nDataframe digest: {hashlib.sha256(self.to_dataframe().values.tobytes()).hexdigest()}"
+
+    def __str__(self) -> str:
+        """Return a string representation of a record packet, with the underlying dataframe anonymized."""
+        return self.to_anonymized_string()

build/lib/build/lib/another_ayx_python_sdk/core/register_plugin.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Function definition for registering a plugin with the SDK."""
+import sys
+from typing import Literal, Optional, TYPE_CHECKING, Type
+
+if TYPE_CHECKING:
+    from another_ayx_python_sdk.core.plugin import Plugin  # noqa: F401
+    from another_ayx_python_sdk.providers.e1_provider.e1_plugin_proxy import (  # noqa: F401
+        E1PluginProxy,
+    )
+
+_ProviderTypes = Literal["e1", "e2", "file"]
+
+
+def register_plugin(
+    plugin_class: Type["Plugin"], version: Optional[str] = None
+) -> Optional[Type["E1PluginProxy"]]:
+    """
+    Register a plugin with the SDK.
+
+    The return value of this function should be assigned to a variable
+    called AyxPlugin in the entrypoint to the tool in order for the
+    E1 Python SDK to properly recognize it.
+
+    Parameters
+    ----------
+    plugin_class
+        Python Plugin to register. This plugin is written using the Python SDK.
+
+    Returns
+    -------
+    E1PluginProxy, optional
+        The Plugin, subclassed for the provider that it was registered with.
+
+    """
+    # TODO: Add v2 registration
+    return {"e1": register_e1_plugin, "file": register_file_provider_plugin}[
+        _get_provider()
+    ](plugin_class, version)
+
+
+def register_e1_plugin(
+    user_plugin_class: Type["Plugin"], version: Optional[str] = None
+) -> Type["E1PluginProxy"]:
+    """
+    Register a plugin with the E1 SDK Provider.
+
+    Parameters
+    ----------
+    user_plugin_class
+        Python Plugin to register.
+
+    version
+        Version of the tool being used.
+
+    Returns
+    -------
+    E1PluginProxy
+        Copy of the passed-in Plugin, subclassed for the E1 Provider.
+
+    """
+    # This makes a copy of the plugin proxy class. This is an unfortunate requirement
+    # given the state of the existing python SDK since the environment shares an
+    # interpreter. Without making a copy of the plugin proxy class on registration
+    # the stateful property setting of user_plugin_class would be overwritten each
+    # time a plugin loads and calls register_e1_plugin, since the library is only
+    # loaded into memory once.
+    from another_ayx_python_sdk.providers.e1_provider.e1_plugin_proxy import (  # noqa: F811
+        E1PluginProxy,
+    )
+
+    class ProxyPluginCopy(E1PluginProxy):
+        pass
+
+    ProxyPluginCopy.user_plugin_class = user_plugin_class
+    tool_directory_name = user_plugin_class.__name__
+
+    if version:
+        tool_directory_name = f"{tool_directory_name}_v{version}"
+    ProxyPluginCopy.user_plugin_directory_name = tool_directory_name
+
+    return ProxyPluginCopy
+
+
+def register_file_provider_plugin(
+    user_plugin_class: Type["Plugin"], version: Optional[str] = None
+) -> None:
+    """
+    Register a plugin with the File Provider.
+
+    Parameters
+    ----------
+    user_plugin_class
+        Python Plugin to register.
+    """
+    import another_ayx_python_sdk.providers.file_provider.file_provider as ayx_file_provider
+
+    ayx_file_provider.user_plugin_class = user_plugin_class
+    return None
+
+
+def _get_provider() -> _ProviderTypes:
+    """
+    Determine which provider is being used to run the Plugin.
+
+    Returns
+    -------
+    _ProviderTypes
+        A Literal that states which provider is being used: e1, e2 (AMP), or file.
+    """
+    if "AlteryxEngineCmd.exe" in sys.executable:
+        return "e1"
+
+    # For now, assume if it's not the e1 provider, then it's the file provider
+    return "file"

build/lib/build/lib/another_ayx_python_sdk/core/testing.py

@@ -0,0 +1,191 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convenience functions, classes, and fixtures for user-generated unit testing."""
+import asyncio
+from collections import namedtuple
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Type
+
+from another_ayx_python_sdk.core import Anchor, PluginV2
+from another_ayx_python_sdk.providers.amp_provider import AMPProviderV2
+from another_ayx_python_sdk.providers.amp_provider.sdk_tool_service_v2 import SdkToolServiceV2
+
+import pyarrow as pa
+
+
+@dataclass
+class _IncomingConnection:
+    name: str
+    metadata: Optional[pa.schema]
+
+
+@dataclass
+class _IncomingAnchor:
+    name: str
+    connections: List[_IncomingConnection]
+
+
+@dataclass
+class _OutgoingAnchor:
+    name: str
+    num_connections: int
+    metadata: Optional[pa.schema]
+
+
+@dataclass
+class _PluginInitializationData:
+    configXml: str  # noqa
+    incomingAnchors: List[_IncomingAnchor]  # noqa
+    outgoingAnchors: List[_OutgoingAnchor]  # noqa
+
+
+@dataclass
+class _DummyCtrlIn:
+    plugin_initialization_data: _PluginInitializationData
+
+
+BatchTuple = namedtuple("BatchTuple", ["input_data", "expected_output_data"])
+
+
+class SdkToolTestService:
+    """A minimal in-process client for testing user plugins."""
+
+    MAPPING = ["NONE", "INFO", "WARN", "ERROR", "FILEINPUT", "FILEOUTPUT"]
+
+    def __init__(
+        self,
+        plugin_class: Type[PluginV2],
+        input_anchor_config: Dict[str, Optional[pa.schema]],
+        output_anchor_config: Dict[str, Optional[pa.schema]],
+        config_mock: str,
+    ):
+        self.service = SdkToolServiceV2()
+
+        provider = AMPProviderV2()
+        incoming_anchors = [
+            _IncomingAnchor(
+                name=key,
+                connections=[
+                    _IncomingConnection(
+                        name=str(i + 1), metadata=input_anchor_config.get(key)
+                    )
+                ],
+            )
+            for i, key in enumerate(input_anchor_config.keys())
+        ]
+
+        outgoing_anchors = [
+            _OutgoingAnchor(
+                name=key, num_connections=1, metadata=output_anchor_config.get(key)
+            )
+            for key in output_anchor_config.keys()
+        ]
+
+        # TODO: generate/pass config_mock in as a dict/json, convert to xml here
+        init_data = _DummyCtrlIn(
+            _PluginInitializationData(
+                configXml=config_mock,
+                incomingAnchors=incoming_anchors,
+                outgoingAnchors=outgoing_anchors,
+            )
+        )
+
+        provider.set_anchors(init_data)
+        provider.environment.set_tool_config(config_mock)
+
+        self.plugin = plugin_class(provider)
+        self.service.driver.plugin = self.plugin
+
+        self._data_streams: Dict[str, List[pa.RecordBatch]] = {}
+        self._io_stream: List[str] = []
+        # TODO: auto-generate plugin initialization from configXML
+
+    @property
+    def io_stream(self) -> List[str]:
+        """
+        Store any strings sent over through provider.io.
+
+        This is roughly the format the resulting strings follow:
+            "(INFO|WARN|ERROR):(.*)"
+        For example, this provider.io call in the plugin code:
+            self.provider.io.info("Test Code")
+        would show up as "INFO:Test Code" in self.io_stream.
+
+        Returns
+        -------
+        List[str]
+            All strings generated by provider.io calls within the plugin code.
+        """
+        return self._io_stream
+
+    @property
+    def data_streams(self) -> Dict[str, List[pa.RecordBatch]]:
+        """
+        Capture any data written via provider.io.write_to_anchor().
+
+        The completed stream is represented by a dictionary, typed like so:
+        {
+            "<Output Anchor name>": [pa.RecordBatch]
+        }
+        If provider.io.write_to_anchor is never called, the data_streams attribute should be an empty dictionary.
+
+        Returns
+        -------
+        Dict[str, List[pa.RecordBatch]
+            A mapping between output anchor names and the corresponding list of RecordBatches
+            written to that output anchor.
+        """
+        return self._data_streams
+
+    def run_on_record_batch(self, batch: pa.RecordBatch, anchor: Anchor) -> None:
+        """Prepare and run the plugin's on_record_batch method, and capture the I/O and data streams."""
+        self._flush_and_save_streams()
+        self.service.driver.record_batch_received(batch, anchor)
+        self._flush_and_save_streams()
+
+    def run_on_incoming_connection_complete(self, anchor: Anchor) -> None:
+        """Prepare and run the plugin's on_incoming_connection_complete method, and capture the I/O and data streams."""
+        self._flush_and_save_streams()
+        self.service.driver.incoming_connection_complete_callback(anchor)
+        self._flush_and_save_streams()
+
+    def run_on_complete(self) -> None:
+        """Prepare and run the plugin's on_complete method, and capture the I/O and data streams."""
+        self._flush_and_save_streams()
+        self.service.driver.on_complete_callback()
+        self._flush_and_save_streams()
+
+    def _flush_and_save_streams(self) -> None:
+        outputs: Dict[str, List[pa.Table]] = {}
+        io_outputs: List[str] = []
+        while not self.service.driver.ctrl_io.ctrl_out.empty():
+            item = asyncio.get_event_loop().run_until_complete(
+                self.service.driver.ctrl_io.ctrl_out.get()
+            )
+            io_outputs.append(
+                f"{self.MAPPING[item.output_message.message_type]}:{item.output_message.message}"
+            )
+        while not self.service.driver.record_io.pending_writes.empty():
+            item = asyncio.get_event_loop().run_until_complete(
+                self.service.driver.record_io.pending_writes.get()
+            )
+            if item["anchor_name"] not in outputs.keys():
+                outputs[item["anchor_name"]] = []
+            outputs[item["anchor_name"]].append(
+                pa.Table.from_pydict(item["data"].to_pydict())
+            )
+        self._data_streams = {
+            key: pa.concat_tables(value).to_batches() for key, value in outputs.items()
+        }
+        self._io_stream = io_outputs

build/lib/build/lib/another_ayx_python_sdk/core/utils.py

@@ -0,0 +1,146 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions definitions for plugin SDK."""
+
+import datetime
+from typing import Dict
+
+from another_ayx_python_sdk.core.field import Field, FieldType
+
+import pyarrow as pa
+
+
+def to_date(stdate: str) -> "datetime.date":
+    """Turn a date represented by a string to a Python date.
+
+    (only the date part and not the time of the day)
+    """
+    return datetime.datetime.strptime(stdate, "%m/%d/%Y").date()
+
+
+def to_time(sttime: str) -> "datetime.time":
+    """Turn a date represented by a string to a Python time."""
+    return datetime.datetime.strptime(sttime, "%H:%M:%S").time()
+
+
+def to_datetime(stdatetime: str) -> "datetime.datetime":
+    """Turn a date represented by a string to a Python datetime."""
+    return datetime.datetime.strptime(stdatetime, "%m/%d/%Y %H:%M:%S")
+
+
+def is_spatial(metadata: "pa.Metadata") -> bool:
+    """Return true if the given arrow type is a spatial object (string + 'ayx' meta info)."""
+    return metadata.get(b"ayx.source", None) == b"WKT"
+
+
+def create_schema(col_meta: Dict = {}) -> "pa.Schema":
+    """Create a Python Arrow Schema given a Dict of Metadata."""
+    fields = []
+    for name, data in col_meta.items():
+        if isinstance(data, dict):
+            fd = Field(
+                name,
+                field_type=data.get("type", FieldType.string),
+                size=data.get("size", 0),
+                scale=data.get("scale", 0),
+                source=data.get("source", ""),
+                description=data.get("description", ""),
+            )
+        else:
+            fd = Field(name, data)
+        fields.append(fd.to_arrow())
+    return pa.schema(fields)
+
+
+def get_ayx_meta(metaname: str) -> str:
+    """Add ayx. prefix to metadata name."""
+    if metaname in ["type", "size", "scale", "source", "description"]:
+        return "ayx." + metaname
+    raise ValueError("bad metadata name: " + metaname)
+
+
+def set_metadata(
+    tbl: "pa.Table", col_meta: Dict = {}, schema: "pa.Schema" = None
+) -> "pa.Table":
+    """Store column-level metadata as byte strings.
+
+    Column-level metadata is stored in the table columns schema fields.
+
+    To update the metadata, first new fields are created for all columns.
+    Next a schema is created using the new fields and updated table metadata.
+    Finally a new table is created by replacing the old one's schema, but
+    without copying any data.
+
+    Args:
+    ----
+        tbl (pyarrow.Table): The table to store metadata in
+        col_meta: A dictionary with column metadata in the form
+            {
+                'column_1': {'type': FieldType.int64, 'size': 8},
+                'column_2': {'size': 64, 'source': 'something'}
+            }
+    """
+    # Create updated column fields with new metadata
+    if schema:
+        return pa.Table.from_arrays(list(tbl.itercolumns()), schema=schema)
+
+    if col_meta:
+        fields = []
+        for col in tbl.schema:
+            if col.name in col_meta:
+                # Get updated column metadata
+                metadata = col.metadata.copy() or {}
+                for k, v in col_meta[col.name].items():
+                    metadata[get_ayx_meta(k).encode("utf-8")] = str(v).encode("utf-8")
+                # Update field with updated metadata
+                col = pa.field(
+                    col.name, col.type, nullable=col.nullable, metadata=metadata
+                )
+            fields.append(col)
+
+        # Create new schema with updated field metadata
+        schema = pa.schema(fields)
+
+        # With updated schema build new table (shouldn't copy data)
+        # tbl = pa.Table.from_batches(tbl.to_batches(), schema)
+        tbl = pa.Table.from_arrays(list(tbl.itercolumns()), schema=schema)
+
+    return tbl
+
+
+def decode_metadata(metadata: Dict) -> Dict:
+    """Arrow stores metadata keys and values as bytes."""
+    if not metadata:
+        # None or {} are not decoded
+        return metadata
+
+    decoded = {}
+    for k, v in metadata.items():
+        key = k.decode("utf-8")
+        if key[0:4] == "ayx.":
+            key = key[4:]
+        val = v.decode("utf-8")
+        decoded[key] = val
+    return decoded
+
+
+def get_metadata(tbl: "pa.Table", col_name: str = "") -> Dict:
+    """Get all column metadata as dicts or just one column, given col_name."""
+    if col_name:
+        for col in tbl.schema:
+            if col.name == col_name:
+                return decode_metadata(col.metadata)
+        return {}
+    else:
+        return {col.name: decode_metadata(col.metadata) for col in tbl.schema}

build/lib/build/lib/another_ayx_python_sdk/examples/AyxSdkInput/main.py

@@ -0,0 +1,89 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Example input tool."""
+from typing import NamedTuple
+
+from another_ayx_python_sdk.core import PluginV2
+from another_ayx_python_sdk.providers.amp_provider.amp_provider_v2 import AMPProviderV2
+
+from pyarrow import Table
+
+
+class AyxSdkInput(PluginV2):
+    """Concrete implementation of an AyxPlugin."""
+
+    def __init__(self, provider: AMPProviderV2) -> None:
+        """Construct a plugin."""
+        self.provider = provider
+        self.config_value = 0.42
+        self.provider.io.info("Plugin initialized.")
+
+    def on_incoming_connection_complete(self, anchor: NamedTuple) -> None:
+        """
+        Call when an incoming connection is done sending data including when no data is sent on an optional input anchor.
+
+        This method IS NOT called during update-only mode.
+
+        Parameters
+        ----------
+        anchor
+            NamedTuple containing anchor.name and anchor.connection.
+        """
+        raise NotImplementedError("Input tools don't receive batches.")
+
+    def on_record_batch(self, batch: "Table", anchor: NamedTuple) -> None:
+        """
+        Process the passed record batch that comes in on the specified anchor.
+
+        The method that gets called whenever the plugin receives a record batch on an input.
+
+        This method IS NOT called during update-only mode.
+
+        Parameters
+        ----------
+        batch
+            A pyarrow Table containing the received batch.
+        anchor
+            A namedtuple('Anchor', ['name', 'connection_name']) containing input connection identifiers.
+        """
+        raise NotImplementedError("Input tools don't receive batches.")
+
+    def on_complete(self) -> None:
+        """
+        Clean up any plugin resources, or push records for an input tool.
+
+        This method gets called when all other plugin processing is complete.
+
+        In this method, a Plugin designer should perform any cleanup for their plugin.
+        However, if the plugin is an input-type tool (it has no incoming connections),
+        processing (record generation) should occur here.
+
+        Note: A tool with an optional input anchor and no incoming connections should
+        also write any records to output anchors here.
+        """
+        import pandas as pd
+        import pyarrow as pa
+
+        df = pd.DataFrame(
+            {
+                "x": [1, 2, 3],
+                "y": ["hello", "world", "from ayx_python_sdk!"],
+                "z": [self.config_value, self.config_value, self.config_value],
+            }
+        )
+
+        packet = pa.Table.from_pandas(df)
+
+        self.provider.write_to_anchor("Output", packet)
+        self.provider.io.info("AyxSdkInput tool done.")