AWSGlueDataplanePython 5.0.0__py3-none-any.whl

awsglue/transforms/relationalize.py

@@ -0,0 +1,105 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+from awsglue.transforms import GlueTransform
+from awsglue.utils import _global_args
+
+class Relationalize(GlueTransform):
+    """
+    Relationalizes a dynamic frame. i.e. produces a list of frames that are
+    generated by unnesting nested columns and pivoting array columns. The
+    pivoted array column can be joined to the root table using the joinkey
+    generated in unnest phase
+    :param frame: DynamicFrame to relationalize
+    :param staging_path: path to store partitions of pivoted tables in csv format. Pivoted tables are read back from
+        this path
+    :param name: name for the root table
+    :param options: dict of optional parameters for relationalize
+    :param transformation_ctx: context key to retrieve metadata about the current transformation
+    :param info: String, any string to be associated with errors in this transformation.
+    :param stageThreshold: Long, number of errors in the given transformation for which the processing needs to error out.
+    :param totalThreshold: Long, total number of errors upto and including in this transformation
+      for which the processing needs to error out.
+    :return: DynamicFrameCollection
+    """
+
+    # TODO: Make staging_path a mandatory argument
+    def __call__(self, frame, staging_path=None, name='roottable', options=None, transformation_ctx = "", info="",
+        stageThreshold=0, totalThreshold=0):
+        options = options or {}
+        # TODO: Remove special handling of staging_path and make it mandatory after TempDir is made a mandatory argument
+        # We are directly accessing the args variable assuming that it is available in the global scope. This is to
+        # maintain backward compatibility with the relationalize call that did not have the mandatory staging_path arg
+        if staging_path is None:
+            if _global_args['TempDir'] is not None and _global_args['TempDir'] != "":
+                staging_path = _global_args['TempDir']
+            else:
+                raise RuntimeError("Unable to set staging_path using args "+str(_global_args))
+        return frame.relationalize(name, staging_path, options, transformation_ctx, info, stageThreshold, totalThreshold)
+
+    @classmethod
+    def describeArgs(cls):
+        arg1 = {"name": "frame",
+                "type": "DynamicFrame",
+                "description": "The DynamicFrame to relationalize",
+                "optional": False,
+                "defaultValue": None}
+        arg2 = {"name": "staging_path",
+                "type": "String",
+                "description": "path to store partitions of pivoted tables in csv format",
+                "optional": True,
+                "defaultValue": None}
+        arg3 = {"name": "name",
+                "type": "String",
+                "description": "Name of the root table",
+                "optional": True,
+                "defaultValue": "roottable"}
+        arg4 = {"name": "options",
+                "type": "Dictionary",
+                "description": "dict of optional parameters for relationalize",
+                "optional": True,
+                "defaultValue": "{}"}
+        arg5 = {"name": "transformation_ctx",
+                "type": "String",
+                "description": "A unique string that is used to identify stats / state information",
+                "optional": True,
+                "defaultValue": ""}
+        arg6 = {"name": "info",
+                "type": "String",
+                "description": "Any string to be associated with errors in the transformation",
+                "optional": True,
+                "defaultValue": "\"\""}
+        arg7 = {"name": "stageThreshold",
+                "type": "Integer",
+                "description": "Max number of errors in the transformation until processing will error out",
+                "optional": True,
+                "defaultValue": "0"}
+        arg8 = {"name": "totalThreshold",
+                "type": "Integer",
+                "description": "Max number of errors total until processing will error out.",
+                "optional": True,
+                "defaultValue": "0"}
+
+        return [arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8]
+
+    @classmethod
+    def describeTransform(cls):
+        return "Flatten nested schema and pivot out array columns from the flattened frame"
+
+    @classmethod
+    def describeErrors(cls):
+        return []
+
+    @classmethod
+    def describeReturn(cls):
+        return {"type": "DynamicFrameCollection",
+                "description": "DynamicFrameCollection resulting from Relationalize call"}

awsglue/transforms/repartition.py

@@ -0,0 +1,61 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+from awsglue.transforms import GlueTransform
+
+class Repartition(GlueTransform):
+    def __call__(self, frame, num_partitions, transformation_ctx = "", info = "",
+                 stageThreshold = 0, totalThreshold = 0):
+        return frame.repartition(num_partitions, transformation_ctx, info, stageThreshold, totalThreshold)
+
+    @classmethod
+    def describeArgs(cls):
+        arg1 = {"name": "num_partitions",
+                "type": "DynamicFrame",
+                "description": "Number of partitions",
+                "optional": False,
+                "defaultValue": None}
+        arg2 = {"name": "transformation_ctx",
+                "type": "String",
+                "description": "A unique string that is used to identify stats / state information",
+                "optional": True,
+                "defaultValue": ""}
+        arg3 = {"name": "info",
+                "type": "String",
+                "description": "Any string to be associated with errors in the transformation",
+                "optional": True,
+                "defaultValue": "\"\""}
+        arg4 = {"name": "stageThreshold",
+                "type": "Integer",
+                "description": "Max number of errors in the transformation until processing will error out",
+                "optional": True,
+                "defaultValue": "0"}
+        arg5 = {"name": "totalThreshold",
+                "type": "Integer",
+                "description": "Max number of errors total until processing will error out.",
+                "optional": True,
+                "defaultValue": "0"}
+
+        return [arg1, arg2, arg3, arg4, arg5]
+
+    @classmethod
+    def describeTransform(cls):
+        return "Repartitions a DynamicFrame."
+
+    @classmethod
+    def describeErrors(cls):
+        return []
+
+    @classmethod
+    def describeReturn(cls):
+        return {"type": "DynamicFrame",
+                "description": "The repartitioned DynamicFrame."}

awsglue/transforms/resolve_choice.py

@@ -0,0 +1,85 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+from awsglue.transforms import GlueTransform
+
+class ResolveChoice(GlueTransform):
+    def __call__(self, frame, specs=None, choice="", database=None, table_name=None, transformation_ctx="", info="", stageThreshold=0, totalThreshold=0, catalog_id=None):
+        return frame.resolveChoice(specs, choice, database, table_name, transformation_ctx, info, stageThreshold, totalThreshold, catalog_id)
+
+    @classmethod
+    def describeArgs(cls):
+        arg1 = {"name": "frame",
+                "type": "DynamicFrame",
+                "description": "DynamicFrame to transform",
+                "optional": False,
+                "defaultValue": None}
+        arg2 = {"name": "specs",
+                "type": "List",
+                "description": "List of specs (path, action)",
+                "optional": True,
+                "defaultValue": None}
+        arg3 = {"name": "choice",
+                "type": "String",
+                "description": "resolve choice option",
+                "optional": True,
+                "defaultValue": ""}
+        arg4 = {"name": "database",
+                "type": "String",
+                "description": "Glue catalog database name, required for MATCH_CATALOG choice",
+                "optional": True,
+                "defaultValue": ""}
+        arg5 = {"name": "table_name",
+                "type": "String",
+                "description": "Glue catalog table name, required for MATCH_CATALOG choice",
+                "optional": True,
+                "defaultValue": ""}
+        arg6 = {"name": "transformation_ctx",
+                "type": "String",
+                "description": "A unique string that is used to identify stats / state information",
+                "optional": True,
+                "defaultValue": ""}
+        arg7 = {"name": "info",
+                "type": "String",
+                "description": "Any string to be associated with errors in the transformation",
+                "optional": True,
+                "defaultValue": "\"\""}
+        arg8 = {"name": "stageThreshold",
+                "type": "Integer",
+                "description": "Max number of errors in the transformation until processing will error out",
+                "optional": True,
+                "defaultValue": "0"}
+        arg9 = {"name": "totalThreshold",
+                "type": "Integer",
+                "description": "Max number of errors total until processing will error out.",
+                "optional": True,
+                "defaultValue": "0"}
+        arg10 = {"name": "catalog_id",
+                "type": "String",
+                "description": "Catalog id for match_catalog id.",
+                "optional": True,
+                "defaultValue": "accountId"}
+
+        return [arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10]
+
+    @classmethod
+    def describeTransform(cls):
+        return "Resolve choice type in this DynamicFrame."
+
+    @classmethod
+    def describeErrors(cls):
+        return []
+
+    @classmethod
+    def describeReturn(cls):
+        return {"type": "DynamicFrame",
+                "description": "DynamicFrame after resolving choice type."}

awsglue/transforms/transform.py

@@ -0,0 +1,92 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+class GlueTransform(object):
+    """Base class for all Glue Transforms.
+
+    All Glue transformations should inherit from GlueTransform and define a
+    __call__ method. They can optionally override the name classmethod or use
+    the default of the class name.
+    """
+
+    @classmethod
+    def apply(cls, *args, **kwargs):
+        transform = cls()
+        return transform(*args, **kwargs)
+
+    @classmethod
+    def name(cls):
+        return cls.__name__
+
+    @classmethod
+    def describeArgs(cls):
+        '''
+        Returns: a list of dictionaries, with each corresponding to
+        an argument, in the following format:
+                [{"name": "<name of argument>",
+                 "type": "<type of argument>",
+                 "description": "<description of argument>",
+                 "optional": "<Boolean>",
+                 "defaultValue": "<String default value or None>"}, ...]
+        Raises: NotImplementedError if not implemented by Transform
+        '''
+        raise NotImplementedError("describeArgs method not implemented for Transform {}".format(cls.__name__))
+
+    @classmethod
+    def describeReturn(cls):
+        '''
+        Returns: A dictionary with information about the return type,
+        in the following format:
+                {"type": "<return type>",
+                "description": "<description of output>"}
+        Raises: NotImplementedError if not implemented by Transform
+        '''
+        raise NotImplementedError("describeReturn method not implemented for Transform {}".format(cls.__name__))
+
+    @classmethod
+    def describeTransform(cls):
+        '''
+        Returns: A string describing the transform, e.g.
+                "Base class for all Glue Transforms"
+        Raises: NotImplementedError if not implemented by Transform
+        '''
+
+        raise NotImplementedError("describeTransform method not implemented for Transform {}".format(cls.__name__))
+
+    @classmethod
+    def describeErrors(cls):
+        '''
+        Returns: A list of dictionaries, each describing possible errors thrown by
+        this transform, in the following format:
+                [{"type": "<type of error>",
+                 "description": "<description of error>"}]
+        Raises: NotImplementedError if not implemented by Transform
+        '''
+        raise NotImplementedError("describeErrors method not implemented for Transform {}".format(cls.__name__))
+
+    @classmethod
+    def describe(cls):
+        return {"transform": {"name": cls.name(),
+                "args": cls.describeArgs(),
+                "returns": cls.describeReturn(),
+                "description": cls.describeTransform(),
+                "raises": cls.describeErrors(),
+                "location": "internal"}}
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __hash__(self):
+        return hash(tuple(sorted(self.__dict__.items())))
+
+    def __repr__(self):
+        return "<Transform: {}>".format(self.name())

awsglue/transforms/unbox.py

@@ -0,0 +1,112 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+from awsglue.transforms import GlueTransform
+
+class Unbox(GlueTransform):
+
+    def __call__(self, frame, path, format, transformation_ctx = "", info="", stageThreshold=0, totalThreshold=0, **options):
+        """
+        unbox a string field
+
+        :param frame: dynamicFrame on which to call unbox
+        :param path: full path to the StringNode you want to unbox
+        :param format: "avro" or "json"
+        :param info: String, any string to be associated with errors in this transformation.
+        :param stageThreshold: Long, number of errors in the given transformation for which the processing needs to error out.
+        :param totalThreshold: Long, total number of errors upto and including in this transformation
+          for which the processing needs to error out.
+        :param options:
+            separator: String,
+            escaper: String,
+            skipFirst: Boolean,
+            withSchema: String, schema string should always be called by using StructType.json()
+            withHeader: Boolean
+        """
+        return frame.unbox(path, format, transformation_ctx, info, stageThreshold, totalThreshold, **options)
+
+    @classmethod
+    def describeArgs(cls):
+        arg1 = {"name": "frame",
+                "type": "DynamicFrame",
+                "description": "The DynamicFrame on which to call Unbox",
+                "optional": False,
+                "defaultValue": None}
+        arg2 = {"name": "path",
+                "type": "String",
+                "description": "full path to the StringNode to unbox",
+                "optional": False,
+                "defaultValue": None}
+        arg3 = {"name": "format",
+                "type": "String",
+                "description": "file format -- \"avro\" or \"json\" only",
+                "optional": False,
+                "defaultValue": None}
+        arg4 = {"name": "transformation_ctx",
+                "type": "String",
+                "description": "A unique string that is used to identify stats / state information",
+                "optional": True,
+                "defaultValue": ""}
+        arg5 = {"name": "info",
+                "type": "String",
+                "description": "Any string to be associated with errors in the transformation",
+                "optional": True,
+                "defaultValue": "\"\""}
+        arg6 = {"name": "stageThreshold",
+                "type": "Integer",
+                "description": "Max number of errors in the transformation until processing will error out",
+                "optional": True,
+                "defaultValue": "0"}
+        arg7 = {"name": "totalThreshold",
+                "type": "Integer",
+                "description": "Max number of errors total until processing will error out.",
+                "optional": True,
+                "defaultValue": "0"}
+        arg8 = {"name": "separator",
+                "type": "String",
+                "description": "separator token",
+                "optional": True,
+                "defaultValue": "None, but individual readers may have their own defaults"}
+        arg9 = {"name": "escaper",
+                "type": "String",
+                "description": "escape token",
+                "optional": True,
+                "defaultValue": "None, but individual readers may have their own defaults"}
+        arg10 = {"name": "skipFirst",
+                "type": "Boolean",
+                "description": "whether to skip the first line of data",
+                "optional": True,
+                "defaultValue": "None, but individual readers may have their own defaults"}
+        arg11 = {"name": "withSchema",
+                "type": "String",
+                "description":"schema for data to unbox, should always be created by using StructType.json()",
+                "optional": True,
+                "defaultValue": "None, but individual readers may have their own defaults"}
+        arg12 = {"name": "withHeader",
+                "type": "Boolean",
+                "description": "whether data being unpacked includes a header",
+                "optional": True,
+                "defaultValue": "None, but individual readers may have their own defaults"}
+        return [arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12]
+
+    @classmethod
+    def describeTransform(cls):
+        return "unbox a string field"
+
+    @classmethod
+    def describeErrors(cls):
+        return []
+
+    @classmethod
+    def describeReturn(cls):
+        return {"type": "DynamicFrame",
+                "description": "new DynamicFrame with unboxed DynamicRecords"}

awsglue/transforms/union.py

@@ -0,0 +1,66 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+from awsglue.transforms import GlueTransform
+
+class Union(GlueTransform):
+    def __call__(self, frame1, frame2, transformation_ctx = "",
+                 info = "", stageThreshold = 0, totalThreshold = 0):
+        return frame1.union(mappings, case_sensitive)
+
+    @classmethod
+    def describeArgs(cls):
+        arg1 = {"name": "frame1",
+                "type": "DynamicFrame",
+                "description": "First DynamicFrame to union.",
+                "optional": False,
+                "defaultValue": None}
+        arg2 = {"name": "frame2",
+                "type": "DynamicFrame",
+                "description": "Second DynamicFrame to union.",
+                "optional": False,
+                "defaultValue": None}
+        arg3 = {"name": "transformation_ctx",
+                "type": "String",
+                "description": "A unique string that is used to identify stats / state information",
+                "optional": True,
+                "defaultValue": ""}
+        arg4 = {"name": "info",
+                "type": "String",
+                "description": "Any string to be associated with errors in the transformation",
+                "optional": True,
+                "defaultValue": "\"\""}
+        arg5 = {"name": "stageThreshold",
+                "type": "Integer",
+                "description": "Max number of errors in the transformation until processing will error out",
+                "optional": True,
+                "defaultValue": "0"}
+        arg6 = {"name": "totalThreshold",
+                "type": "Integer",
+                "description": "Max number of errors total until processing will error out.",
+                "optional": True,
+                "defaultValue": "0"}
+
+        return [arg1, arg2, arg3, arg4, arg5, arg6]
+
+    @classmethod
+    def describeTransform(cls):
+        return "Union two DynamicFrames."
+
+    @classmethod
+    def describeErrors(cls):
+        return []
+
+    @classmethod
+    def describeReturn(cls):
+        return {"type": "DynamicFrame",
+                "description": "DynamicFrame containing all records from both input DynamicFrames."}

awsglue/transforms/unnest_frame.py

@@ -0,0 +1,75 @@
+# Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#  http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express
+# or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+from awsglue.transforms import GlueTransform
+
+class UnnestFrame(GlueTransform):
+    """
+    unnest a dynamic frame. i.e. flattens nested objects to top level elements.
+    It also generates joinkeys for array objects
+    """
+
+    def __call__(self, frame, transformation_ctx = "", info="", stageThreshold=0, totalThreshold=0):
+        """
+        unnest a dynamic frame. i.e. flattens nested objects to top level elements.
+        It also generates joinkeys for array objects
+        :param frame: DynamicFrame, the dynamicframe to unnest
+        :param info: String, any string to be associated with errors in this transformation.
+        :param stageThreshold: Long, number of errors in the given transformation for which the processing needs to error out.
+        :param totalThreshold: Long, total number of errors upto and including in this transformation
+          for which the processing needs to error out.
+        :return: a new unnested dynamic frame
+        """
+        return frame.unnest(transformation_ctx, info, stageThreshold, totalThreshold)
+
+    @classmethod
+    def describeArgs(cls):
+        arg1 = {"name": "frame",
+                "type": "DynamicFrame",
+                "description": "The DynamicFrame to unnest",
+                "optional": False,
+                "defaultValue": None}
+        arg2 = {"name": "transformation_ctx",
+                "type": "String",
+                "description": "A unique string that is used to identify stats / state information",
+                "optional": True,
+                "defaultValue": ""}
+        arg3 = {"name": "info",
+                "type": "String",
+                "description": "Any string to be associated with errors in the transformation",
+                "optional": True,
+                "defaultValue": "\"\""}
+        arg4 = {"name": "stageThreshold",
+                "type": "Integer",
+                "description": "Max number of errors in the transformation until processing will error out",
+                "optional": True,
+                "defaultValue": "0"}
+        arg5 = {"name": "totalThreshold",
+                "type": "Integer",
+                "description": "Max number of errors total until processing will error out.",
+                "optional": True,
+                "defaultValue": "0"}
+
+        return [arg1, arg2, arg3, arg4, arg5]
+
+    @classmethod
+    def describeTransform(cls):
+        return "unnest a dynamic frame. i.e. flatten nested objects to top level elements."
+
+    @classmethod
+    def describeErrors(cls):
+        return []
+
+    @classmethod
+    def describeReturn(cls):
+        return {"type": "DynamicFrame",
+                "description": "new unnested DynamicFrame"}