aepp 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

aepp/__init__.py

@@ -14,6 +14,7 @@ from aepp import connector
 from .configs import *
 from .__version__ import __version__
 from typing import Union
+import re
 
 ## other libraries
 from copy import deepcopy
@@ -83,8 +84,33 @@ def getPlatformEvents(
     return data
 
 
+def __make_filename_safe__(filename, replacement="_"):
+    # 1. Remove characters that are illegal in Windows or Linux
+    # Windows: < > : " / \ | ? * # Linux: / (and NULL)
+    filename = re.sub(r'[<>:"/\\|?*]', replacement, filename)
+    # 2. Remove control characters (ASCII 0-31)
+    filename = re.sub(r'[\x00-\x1f]', replacement, filename)
+    # 3. Trim whitespace and trailing dots (Windows doesn't like trailing dots)
+    filename = filename.strip().strip('.')
+    # 4. Handle Windows Reserved Names (CON, PRN, AUX, NUL, COM1, LPT1, etc.)
+    reserved_names = {
+        "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5",
+        "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5",
+        "LPT6", "LPT7", "LPT8", "LPT9"
+    }
+    # Check the base name without extension
+    base_name = os.path.splitext(filename)[0].upper()
+    if base_name in reserved_names:
+        filename = f"{replacement}{filename}"
+    # 5. Length limit (standard for most file systems is 255 chars)
+    if len(filename) > 255:
+        filename = filename[:255]
+    # 6. Default to a placeholder if the string is empty after cleaning
+    return filename if filename else "unnamed_file"
+
+
 def saveFile(
-    module: str = None,
+    module: str | None = None,
     file: object = None,
     filename: str = None,
     type_file: str = "json",
@@ -93,31 +119,39 @@ def saveFile(
     """
     Save the file in the approriate folder depending on the module sending the information.
      Arguments:
-          module: REQUIRED: Module requesting the save file.
+          module: OPTIONAL: Module requesting the save file.
           file: REQUIRED: an object containing the file to save.
           filename: REQUIRED: the filename to be used.
           type_file: REQUIRED: the type of file to be saveed(default: json)
           encoding : OPTIONAL : encoding used to write the file.
     """
-    if module is None:
-        raise ValueError("Require the module to create a folder")
     if file is None or filename is None:
         raise ValueError("Require a object for file and a name for the file")
     here = Path(Path.cwd())
-    folder = module.capitalize()
-    new_location = Path.joinpath(here, folder)
+    if module is not None:
+        folder = module.capitalize()
+        new_location = Path.joinpath(here, folder)
+    else:
+        new_location = here
     if new_location.exists() == False:
         new_location.mkdir()
+    filename = __make_filename_safe__(filename)
     if type_file == "json":
         filename = f"{filename}.json"
         complete_path = Path.joinpath(new_location, filename)
         with open(complete_path, "w", encoding=encoding) as f:
             f.write(json.dumps(file, indent=4))
-    else:
+    elif type_file == "txt":
         filename = f"{filename}.txt"
         complete_path = Path.joinpath(new_location, filename)
         with open(complete_path, "w", encoding=encoding) as f:
             f.write(file)
+    else:
+        filename = f"{filename}.{type_file}"
+        complete_path = Path.joinpath(new_location, filename)
+        with open(complete_path, "wb") as f:
+            f.write(file)
+    return complete_path
 
 def __titleSafe__(text: str) -> str:
     """
@@ -289,7 +323,7 @@ def extractSandboxArtefact(
         sandbox: REQUIRED: the instance of a ConnectObject that contains the sandbox information and connection.
         localFolder: OPTIONAL: the local folder where to extract the sandbox. If not provided, it will use the current working directory and name the folder the name of the sandbox.
         artefact: REQUIRED: the id or the name of the artefact to export.
-        artefactType: REQUIRED: the type of artefact to export. Possible values are: 'class','schema','fieldgroup','datatype','descriptor','dataset','identity','mergepolicy'
+        artefactType: REQUIRED: the type of artefact to export. Possible values are: 'class','schema','fieldgroup','datatype','descriptor','dataset','identity','mergepolicy',audience'
         region: OPTIONAL: the region of the sandbox (default: nld2). This is used to fetch the correct API endpoints for the identities. 
             Possible values: "va7","aus5", "can2", "ind2"
     """

aepp/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.2"
+__version__ = "0.4.3"

aepp/flowservice.py

@@ -784,7 +784,7 @@ class FlowService:
                             authenticationRequired:bool=False
                             )-> dict:
         """
-        Create a streaming flow with or without transformation
+        Create a source flow streaming with or without transformation
             name : REQUIRED : The name of the Data Flow (Optional if you are just passing a datasetId).
             description : OPTIONAL : description of the Flow
             source_connection_id : REQUIRED : The ID of the source connection tied to Data Lake (Optional if you are just passing a datasetId).

aepp/queryservice.py

@@ -227,6 +227,8 @@ class QueryService:
             if orderby == "+":
                 orderby = "%2B"
             arguments["orderby"] = orderby
+        else:
+            arguments["orderby"] = "-created"
         if start is not None:
             arguments["start"] = start
         if limit is not None:
@@ -706,27 +708,35 @@ class QueryService:
         res = self.connector.deleteData(self.endpoint + path)
         return res
 
-    def createQueryTemplate(self, queryData: dict = None) -> dict:
+    def createQueryTemplate(self, queryData: dict = None,name: str = None,sql: str = None,queryParameters: dict = None) -> dict:
         """
         Create a query template based on the dictionary passed.
         Arguments:
-            queryData : REQUIED : An object that contains "sql", "queryParameter" and "name" keys.
-        more info : https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Query-Templates/create_query_template
+            queryData : REQUIED : An object that contains "sql" and "name" keys.
+            name : OPTIONAL : Name of the template
+            sql : OPTIONAL : SQL query as a string.
+            queryParameters : OPTIONAL : in case you are using template, providing the paramter in a dictionary.
+        more info : https://developer.adobe.com/experience-platform-apis/references/query-service/#tag/Query-Templates
         """
         path = "/query-templates"
         if self.loggingEnabled:
             self.logger.debug(f"Starting createTemplate")
-        if isinstance(queryData, dict):
-            if (
-                "sql" not in queryData.keys()
-                or "queryParameters" not in queryData.keys()
-                or "name" not in queryData.keys()
-            ):
-                raise KeyError(
-                    "Minimum key value are not respected.\nPlease see here for more info :\nhttps://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Query-Templates/create_query_template "
-                )
+        if queryData is not None:
+            if isinstance(queryData, dict):
+                if ("sql" not in queryData.keys() or "name" not in queryData.keys()):
+                    raise KeyError(
+                        "Minimum key value are not respected.\nPlease see here for more info :\nhttps://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Query-Templates/create_query_template "
+                    )
+            else:
+                raise Exception("expected a dictionary for queryData")
         else:
-            raise Exception("expected a dictionary for queryData")
+            if name is None or sql is None:
+                raise Exception(
+                    "Either queryData dictionary or name and sql parameters are required."
+                )
+            queryData = {"name": name, "sql": sql}
+            if queryParameters is not None:
+                queryData["queryParameters"] = queryParameters
         res = self.connector.postData(
             self.endpoint + path, headers=self.header, data=queryData
         )

aepp/schema.py

@@ -261,6 +261,7 @@ class Schema:
             classFilter: str = None,
             excludeAdhoc: bool = True,
             output: str = 'raw',
+            prop: str = None,
             **kwargs
     ) -> list:
         """
@@ -285,6 +286,8 @@ class Schema:
             params["property"] = f"meta:extends=={classFilter}"
         elif excludeAdhoc:
             params["property"] = "meta:extends!=https://ns.adobe.com/xdm/data/adhoc"
+        if prop is not None:
+            params["property"] = prop
         verbose = kwargs.get("debug", False)
         privateHeader = deepcopy(self.header)
         format = kwargs.get("format", "xed-id")
@@ -1783,7 +1786,7 @@ class Schema:
         ]
         res = self.connector.patchData(self.endpoint + path,data=operation)
         return res
-    
+
     def enableSchemaForRealTime(self,schemaId:str=None)->dict:
         """
         Enable a schema for real time based on its ID.
@@ -1808,6 +1811,30 @@ class Schema:
         res = self.connector.patchData(self.endpoint + path,data=operation)
         return res
     
+    def enableSchemaForUPS(self,schemaId:str=None)->dict:
+        """
+        Enable a schema for UPS based on its ID.
+        Arguments:
+            schemaId : REQUIRED : The schema ID required to be updated
+        """
+        if schemaId is None:
+            raise Exception("Require a schema ID")
+        if schemaId.startswith("https://"):
+            from urllib import parse
+            schemaId = parse.quote_plus(schemaId)
+        if self.loggingEnabled:
+            self.logger.debug(f"Starting enableSchemaForUPS")
+        path = f"/{self.container}/schemas/{schemaId}/"
+        operation = [
+           { 
+            "op": "add",
+            "path": "/meta:immutableTags",
+            "value": ["union"]
+            }
+        ]
+        res = self.connector.patchData(self.endpoint + path,data=operation)
+        return res
+    
     def FieldGroupManager(self,fieldGroup:Union[dict,str,None],title:str=None,fg_class:list=["experienceevent","profile"]) -> 'FieldGroupManager':
          """
          Generates a field group Manager instance using the information provided by the schema instance.

aepp/synchronizer.py

@@ -1025,6 +1025,12 @@ class Synchronizer:
                     raise Exception("the audience could not be created in the target sandbox")
             else: ## audience already exists in target
                 if verbose:
-                    print(f"audience '{audience_name}' already exists in target {target}, saving it")
-                self.dict_targetComponents[target]['audience'][audience_name] = [el for el in t_audiences if el['name'] == audience_name][0]
+                    print(f"audience '{audience_name}' already exists in target {target}, updating it")
+                t_audience = [el for el in t_audiences if el['name'] == audience_name][0]
+                t_audience['description'] = baseAudience.get('description','')
+                t_audience['expression'] = baseAudience.get('expression',[])
+                t_audience['ansibleDataModel'] = baseAudience.get('ansibleDataModel',{})
+                t_audience['evaluationInfo'] = baseAudience.get('evaluationInfo',{'batch': {'enabled': True}, 'continuous': {'enabled': False},'synchronous': {'enabled': False}})
+                res = targetAudiences.putAudience(t_audience['id'],t_audience)
+                self.dict_targetComponents[target]['audience'][audience_name] = res
 

{aepp-0.4.2.dist-info → aepp-0.4.3.dist-info}/METADATA

@@ -1,10 +1,12 @@
 Metadata-Version: 2.4
 Name: aepp
-Version: 0.4.2
+Version: 0.4.3
 Summary: Package to manage AEP API endpoint and some helper functions
 Home-page: https://github.com/adobe/aepp
 Author: Julien Piccini
-Author-email: piccini.julien@gmail.com
+Author-email: Julien Piccini <piccini.julien@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/adobe/aepp
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Classifier: License :: OSI Approved :: Apache Software License
@@ -21,16 +23,11 @@ Requires-Dist: pathlib2
 Requires-Dist: tenacity
 Requires-Dist: deprecation
 Requires-Dist: datamodel-code-generator
+Requires-Dist: rich
 Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license-file
-Dynamic: requires-dist
 Dynamic: requires-python
-Dynamic: summary
 
 # Adobe Experience Platform API made for humans
 

{aepp-0.4.2.dist-info → aepp-0.4.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
-aepp/__init__.py,sha256=d5hE6VcHz52B5o_AbYO368iG8FTsJlt91nBWej8WL8A,26381
-aepp/__version__.py,sha256=Y-I6FXKes4hB39WcBm3HZ1OCtCWiErPB6YZ-YSgexuE,21
+aepp/__init__.py,sha256=tKlipgknDl84iALUNgR9mktILF3gSk1GgMUw_Gg-HXE,27912
+aepp/__version__.py,sha256=-0DFAS3GW84dLmJEuzbtUA7qvIf_cBb4LhqSg0oFfeI,21
 aepp/accesscontrol.py,sha256=PB3FcrO4bvDjdNxjHx7p_20hp4ahBXewoOSxuTGMXC8,17423
 aepp/catalog.py,sha256=hK9m3SAP0fhgkYqu14Tcfq14qBhw54tLCOF0mH31b1M,68237
 aepp/classmanager.py,sha256=CTYGkg5ygB8HtRia6DfT9WLBqXJOVg7pSM9jBB25Bqw,64707
@@ -17,34 +17,26 @@ aepp/destinationinstanceservice.py,sha256=zEZbKi519cOOdxWMZ3mv9ccP6yjNAlNwqrQMlz
 aepp/edge.py,sha256=F2QZApmITObXB8hRWXftHBZ82KNqVZ7iSNuovT8qnk4,16041
 aepp/exportDatasetToDataLandingZone.py,sha256=C6jg3XttFC-0mswa3ypZb6qx3MCQ8_A_3kyKspurXJA,18629
 aepp/fieldgroupmanager.py,sha256=OQkSuGiSxU5ToSPqKmB_3Pmfg6aZjQMGL6LFnODqEiY,101560
-aepp/flowservice.py,sha256=gVayD3vIMu7-d9TGpVtkbQ1r9iwAjJHWNRVtxKURWKc,107610
+aepp/flowservice.py,sha256=nj9dq3zBRT5I8Xb2EwI4VOToJX575Su5lNFvQ6o4HKI,107617
 aepp/hygiene.py,sha256=VEspnyu9eUlcK3wLeJYclaFaOWl5G5I5MRwmVA-RnUg,15385
 aepp/identity.py,sha256=E9MCIgntScMssduqKZqehT6FqSfTjWHcq7E7wESj3Zc,20833
 aepp/ingestion.py,sha256=OamE7NDei2Ev5vXIDkMlzvdyBaN41nkIGmpAnUlQoZI,22372
 aepp/observability.py,sha256=bKe74nlXYB5E5syh7Lj4VqIgwUI3OjMxK383P05EdLU,9951
 aepp/policy.py,sha256=JbpvfCKJl2kE2McK2mn_ZI5HKd_6pTnrfMoUdyJesWQ,24924
 aepp/privacyservice.py,sha256=V6BkJeZG1LDBCyEQm9Gx0i68iRHG6uxSJiVnXzkHapI,8790
-aepp/queryservice.py,sha256=mYcnzBG6PmNZYepQydqMhG-oFD4eIDEzb3mAKtKMgZE,61739
+aepp/queryservice.py,sha256=wB9GiaMwJszNjqkYjkfEDUhdT2IoI22jA3Kt_6ki4Hk,62373
 aepp/sandboxes.py,sha256=UwlSFkO2OOmH--6ISz8rxwDu2LcLH1MPqoH7yOEAZHc,29363
-aepp/schema.py,sha256=PmYXSkmhmVVUbXQqsEVSAaTMYKOYha8FIYoySwPgMs4,122150
+aepp/schema.py,sha256=aLYDM5lCANNddk-NZPNxCxazg9HpELalKlFxQz55dRs,123111
 aepp/schemamanager.py,sha256=hwItd4vXsPFeV25gX1Fbeiu07-BCg4z_VRQREMgJZ58,50738
 aepp/segmentation.py,sha256=oSgR2yx4nawYN5XAeHV_wefvmXEf0nb-bCguaDmp8F8,43555
 aepp/sensei.py,sha256=oYNy5BSWAEqsDkEexcQso6NfA6ntGGMnCOyHri0pJs8,7761
 aepp/som.py,sha256=XNm_Lu2wt2kpSSpldLptuER2eludFXeO9fI6i3iNCzo,34175
-aepp/synchronizer.py,sha256=UA48GKJ5OaasMxbS2O7A9T1i6HrehRx8_7jtxaNFD9M,77388
+aepp/synchronizer.py,sha256=nkZ3dn335JmwzzO3PqYkMhU7ZfiOHGKObfL1yZrnHLY,77932
 aepp/tags.py,sha256=t2qBallTcWR4IOXcDBmrPpqjbSay1z3E2bcRijzVm1s,17641
 aepp/utils.py,sha256=tG-YVXylm38-bynqfp5N_Mzyo7mhlZj-dLo7wLoO4tM,1200
-aepp-0.4.2.dist-info/licenses/LICENSE,sha256=HjYTlfne3BbS5gNHzNqJ5COCiTQLUdf87QkzRyFbE4Y,10337
-tests/__init__.py,sha256=d6zWJsJFZrQd5wAYM7sezSxwXbuMMWfNPkK_vpaUzFA,623
-tests/catalog_test.py,sha256=O4kkG0C_dXk3E77pSzWIt1ewfyKjfZqgbJmBwWwx0po,2246
-tests/dataaccess_test.py,sha256=bnHwOjPPauTM8s1c6O7iUYC--gqt6tPzT94aEZHDC-c,1238
-tests/datasets_test.py,sha256=zQmEPhzzCOxU8B4O7xFpuNICthc8dK8g4oaLXxTaP2I,1093
-tests/destinationinstanceservice_test.py,sha256=uC4qexoMJT1_6b7lA1lnQpazpvIIWyKDBKc7R8A4jrU,2105
-tests/exportDatasetToDatalandingZone_test.py,sha256=193AgQR8yhnQmRWV9pgYz1X2Hz-YEp3ESuNEh3hrTAM,7603
-tests/flowservice_test.py,sha256=Y1mpYWbKYL_x-ZlIY-EuOuNvlzVV1ERlKseDO7gN3Ss,4208
-tests/schema_test.py,sha256=6UsgdsizKmii1hzREpBEKWvZouXdJMvU68UKSxlt1uk,2774
-tests/som_test.py,sha256=a4ut0pEg1HJVMTESaPITmj7YkF54eWCMzKxTMIS-VvM,12101
-aepp-0.4.2.dist-info/METADATA,sha256=nCyugz0f9W1kg7B-VVHAlYrKQt7Pifwq4Jh7CJ0SmRc,5470
-aepp-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aepp-0.4.2.dist-info/top_level.txt,sha256=Gn88pv1ywuEAgOvhmmXXhN4dosEfCrBNDskje3nqS34,11
-aepp-0.4.2.dist-info/RECORD,,
+aepp-0.4.3.dist-info/licenses/LICENSE,sha256=HjYTlfne3BbS5gNHzNqJ5COCiTQLUdf87QkzRyFbE4Y,10337
+aepp-0.4.3.dist-info/METADATA,sha256=xvZ4EKktA2D5XdlRSK3HGvqYijONkYB0pydXusr0YvA,5440
+aepp-0.4.3.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+aepp-0.4.3.dist-info/entry_points.txt,sha256=e7HAumUTymoUiCuVRzFlcchennUBLcjxvuiimySF98Y,48
+aepp-0.4.3.dist-info/top_level.txt,sha256=dtZJI8SzhWVgZRl68PHKZX_fD6amvDiFR-lqD9FSJvE,5
+aepp-0.4.3.dist-info/RECORD,,

{aepp-0.4.2.dist-info → aepp-0.4.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

aepp-0.4.3.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+aepp = aepp.cli.__main__:main

aepp-0.4.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+aepp

aepp-0.4.2.dist-info/top_level.txt

@@ -1,2 +0,0 @@
-aepp
-tests

tests/__init__.py

@@ -1,10 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-

tests/catalog_test.py

@@ -1,80 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.schema import Schema
-import unittest
-from unittest.mock import patch, MagicMock
-
-
-class CatalogTest(unittest.TestCase):
-
-    def test_catalog_get_catalog__resource(self):
-        assert True
-
-    def test_catalog_decode_stream_batch(self):
-        assert True
-
-
-    def test_catalog_json_stream_messages(self):
-        assert True
-
-    def test_catalog_get_batches(self):
-        assert True
-
-    def test_catalog_get_failed_batches_df(self):
-        assert True
-
-    def test_catalog_get_batch(self):
-        assert True
-
-    def test_catalog_create_batch(self):
-        assert True
-
-    def test_catalog_get_resources(self):
-        assert True
-
-    def test_catalog_get_data_sets(self):
-        assert True
-
-    def test_catalog_create_datasets(self):
-        assert True
-
-    def test_catalog_get_data_set(self):
-        assert True
-
-    def test_catalog_delete_data_set(self):
-        assert True
-
-    def test_catalog_get_data_set_views(self):
-        assert True
-
-    def test_catalog_get_data_set_view(self):
-        assert True
-
-    def test_catalog_get_dataset_view_files(self):
-        assert True
-
-    def test_catalog_enable_dataset_profile(self):
-       assert True
-
-    def test_catalog_enable_dataset_identity(self):
-        assert True
-
-    def test_catalog_disable_dataset_profile(self):
-        assert True
-
-    def test_catalog_disable_dataset_identity(self):
-        assert True
-
-    def test_catalog_create_union_profile_dataset(self):
-        assert True
-
-    def test_catalog_get_mapper_errors(self):
-        assert True

tests/dataaccess_test.py

@@ -1,36 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.schema import Schema
-import unittest
-from unittest.mock import patch, MagicMock
-
-class DataAccessTest(unittest.TestCase):
-
-    def test_dataaccess_get_batch_files(self):
-        assert True
-
-    def test_dataaccess_get_batch_failed(self):
-        assert True
-
-    def test_dataaccess_get_batch_meta(self):
-        assert True
-
-    def test_dataaccess_getHeadFile(self):
-        assert True
-
-    def test_dataaccess_get_files(self):
-        assert True
-
-    def test_dataaccess_get_preview(self):
-        assert True
-
-    def test_dataaccess_get_resource(self):
-        assert True

tests/datasets_test.py

@@ -1,30 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.schema import Schema
-import unittest
-from unittest.mock import patch, MagicMock
-
-class DatasetsTest(unittest.TestCase):
-
-    def test_datasets_get_label_schema(self):
-        assert True
-
-    def test_datasets_head_label(self):
-        assert True
-
-    def test_datasets_delete_labels(self):
-        assert True
-
-    def test_datasets_create_labels(self):
-        assert True
-
-    def test_datasets_update_labels(self):
-        assert True

tests/destinationinstanceservice_test.py

@@ -1,36 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.destinationinstanceservice import DestinationInstanceService
-import unittest
-from unittest.mock import patch, MagicMock, ANY
-
-
-class DestinationInstanceServiceTest(unittest.TestCase):
-    ADHOC_INPUT = {"flow1": ["dataset1"], "flow2": ["dataset2", "dataset3"]}
-    ADHOC_EXPECTED_PAYLOAD = {'activationInfo': {'destinations': [{'flowId': 'flow1', 'datasets': [{'id': 'dataset1'}]}, {'flowId': 'flow2', 'datasets': [{'id': 'dataset2'}, {'id': 'dataset3'}]}]}}
-    
-    @patch("aepp.connector.AdobeRequest")
-    def test_create_adhoc_dataset_export(self, mock_connector):
-        instance_conn = mock_connector.return_value
-        instance_conn.postData.return_value = {'foo'}
-        destination_instance_service_obj = DestinationInstanceService()
-        result = destination_instance_service_obj.createAdHocDatasetExport(self.ADHOC_INPUT)
-        assert(result is not None)
-        instance_conn.postData.assert_called_once()
-        instance_conn.postData.assert_called_with(ANY, data=self.ADHOC_EXPECTED_PAYLOAD)
-
-    @patch("aepp.connector.AdobeRequest")
-    def test_create_adhoc_dataset_export_invalid_input(self, mock_connector):
-        destination_instance_service_obj = DestinationInstanceService()
-        with self.assertRaises(Exception) as cm:
-            destination_instance_service_obj.createAdHocDatasetExport(None)
-        self.assertEqual('Require a dict for defining the flowId to datasetIds mapping', str(cm.exception))
-    

tests/exportDatasetToDatalandingZone_test.py

@@ -1,161 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.exportDatasetToDataLandingZone import ExportDatasetToDataLandingZone
-import unittest
-from unittest.mock import patch, MagicMock, ANY
-from tenacity import RetryError
-
-
-class ExportDatasetToDataLandingZoneTest(unittest.TestCase):
-    flow_response = {
-        "id": "df193495-9837-407f-9d50-e51efc067cb5",
-        "sourceConnectionIds": [
-            "78754315-852c-4c4e-8f1a-d12b83264f3f"
-        ],
-        "state": "disabled",
-        "etag": "test_etag"}
-
-    connection_spec = {
-        "name": "test",
-        "id": "test_connection_id"
-    }
-
-
-    source_connection = {
-        "id": "test_source_connection_id",
-        "params": {
-            "datasets": [{
-                "dataSetId": "test"
-            }]}
-    }
-
-    base_connection = {
-        "id": "test_base_connection_id"
-    }
-
-    target_connection = {
-        "id": "test_target_connection_id"
-    }
-
-    flow_run = {
-        "items": [
-            {
-                "id": "test_id",
-                "metrics": {
-                    "durationSummary": {
-                        "startedAtUTC": 1691184699,
-                        "completedAtUTC": 1691184819
-                    },
-                    "fileSummary": {
-                        "outputFileCount": 24
-                    },
-                    "sizeSummary": {
-                        "outputBytes": 77494320
-                    },
-                    "recordSummary": {
-                        "outputRecordCount": 1065802
-                    }
-                }
-            }
-        ]
-    }
-
-    adhoc_success_response = {
-        "destinations":[
-            {
-                "datasets":[
-                    {
-                        "id":"641ce00b8f31811b98dd3b56",
-                        "statusURL":"https: //platform-stage.adobe.io/data/foundation/flowservice/runs/aa39ad3d-24ba-4579-9cdc-55536c408721",
-                        "flowId":"3eaa2c0b-e24b-46bd-8eee-bbaf9d6cf2cb"
-                    }
-                ]
-            }
-        ]
-    }
-
-
-    adhoc_non_retry_error = {
-        "error_code": "401013",
-        "message": "Oauth token is not valid"
-    }
-
-    adhoc_retry_error = {
-        "message": "Following order ID(s) are not ready for dataset export"
-    }
-
-    config = {
-        "org_id": "3ADF23C463D98F640A494032@AdobeOrg",
-        "client_id": "35e6e4d205274c4ca1418805ac41153b",
-        "tech_id": "test005@techacct.adobe.com",
-        "pathToKey": "/Users/Downloads/config/private.key",
-        "auth_code": "",
-        "secret": "test",
-        "date_limit": 0,
-        "sandbox": "prod",
-        "environment": "stage",
-        "token": "token",
-        "oauthTokenEndpoint": "",
-        "imsEndpoint": "https://ims-na1-stg1.adobelogin.com",
-        "private_key": ""
-    }
-
-    @patch('aepp.utils.Utils.check_if_exists', MagicMock(return_value = "test_dataflow_id"))
-    @patch('aepp.flowservice.FlowService.getFlow', MagicMock(return_value = flow_response))
-    @patch('aepp.flowservice.FlowService.getSourceConnection', MagicMock(return_value = source_connection))
-    @patch('aepp.flowservice.FlowService.getRun', MagicMock(return_value = flow_run))
-    @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_success_response))
-    @patch("aepp.connector.AdobeRequest", MagicMock())
-    def test_create_dataflow_if_exist(self):
-        export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock())
-        export_obj.createDataFlowRunIfNotExists(dataset_id="test", compression_type="gzip", data_format="parquet", export_path="test", on_schedule=False, config_path="test", entity_name="test", initial_delay=0)
-
-    @patch('aepp.utils.Utils.check_if_exists', MagicMock(return_value = "test_dataflow_id"))
-    @patch('aepp.flowservice.FlowService.getFlow', MagicMock(return_value = flow_response))
-    @patch('aepp.flowservice.FlowService.getSourceConnection', MagicMock(return_value = source_connection))
-    @patch("aepp.connector.AdobeRequest", MagicMock())
-    def test_create_dataflow_on_schedule(self):
-        export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock())
-        export_obj.createDataFlowRunIfNotExists(dataset_id="test", compression_type="gzip", data_format="parquet", export_path="test", on_schedule=True, config_path="test",entity_name="test", initial_delay=0)
-
-    @patch('aepp.flowservice.FlowService.createConnection', MagicMock(return_value = base_connection))
-    @patch('aepp.flowservice.FlowService.getConnectionSpecIdFromName', MagicMock(return_value = connection_spec))
-    @patch('aepp.flowservice.FlowService.createSourceConnection', MagicMock(return_value = source_connection))
-    @patch('aepp.flowservice.FlowService.createTargetConnection', MagicMock(return_value = target_connection))
-    @patch('aepp.flowservice.FlowService.createFlow', MagicMock(return_value = flow_response))
-    @patch('aepp.utils.Utils.save_field_in_config', MagicMock())
-    @patch("aepp.connector.AdobeRequest", MagicMock())
-    def test_create_dataflow_if_not_exist(self):
-        export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock())
-        return_dataflow_id = export_obj.createDataFlow(dataset_id="test", compression_type="gzip", data_format="parquet", export_path="test", on_schedule=False, config_path="test", entity_name="test")
-        assert (return_dataflow_id == self.flow_response["id"])
-
-    @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_success_response))
-    @patch("aepp.connector.AdobeRequest", MagicMock())
-    def test_retry_on_success_response(self):
-        export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock())
-        assert(export_obj.retryOnNotReadyException("test", "test", 1, 1) == self.adhoc_success_response)
-
-    @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_non_retry_error))
-    @patch("aepp.connector.AdobeRequest", MagicMock())
-    def test_no_retry_error(self):
-        export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock())
-        assert (export_obj.retryOnNotReadyException("test", "test", 1, 1) == self.adhoc_non_retry_error)
-
-    @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_retry_error))
-    @patch("aepp.connector.AdobeRequest", MagicMock())
-    def test_retry_error(self):
-        export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock())
-        try:
-            export_obj.retryOnNotReadyException("test", "test", 1, 1)
-            self.fail("expect a retry error")
-        except RetryError:
-            pass

tests/flowservice_test.py

@@ -1,150 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.schema import Schema
-import unittest
-from unittest.mock import patch, MagicMock
-
-
-class FlowserviceTest(unittest.TestCase):
-
-    def test_flowservice_get_resource(self):
-        assert True
-
-    def test_flowservice_get_connections(self):
-        assert True
-
-    def test_flowservice_create_connection(self):
-        assert True
-
-    def test_flowservice_create_streaming_connection(self):
-        assert True
-
-    def test_flowservice_get_connection(self):
-        assert True
-
-    def test_flowservice_connection(self):
-        assert True
-
-    def test_flowservice_delete_connection(self):
-        assert True
-
-    def test_flowservice_get_connection_specs(self):
-        assert True
-
-    def test_flowservice_get_connection_specs_map(self):
-        assert True
-
-    def test_flowservice_get_connection_spec(self):
-        assert True
-
-    def test_flowservice_get_connection_specid_from_name(self):
-        assert True
-
-    def test_flowservice_get_flows(self):
-        assert True
-
-    def test_flowservice_get_flow(self):
-        assert True
-
-    def test_flowservice_delete_flow(self):
-        assert True
-
-    def test_flowservice_create_flow(self):
-        assert True
-
-    def test_flowservice_create_flow_data_lake_to_data_landing_zone(self):
-        assert True
-
-    def test_flowservice_create_data_landing_zone_to_datalake(self):
-        assert True
-
-    def test_flowservice_updateFlow(self):
-        assert True
-
-    def test_flowservice_get_flow_specs(self):
-        assert True
-
-    def test_flowservice_get_flow_spec_id_from_names(self):
-         assert True
-
-    def test_flowservice_get_flow_spec(self):
-        assert True
-
-    def test_flowservice_get_runs(self):
-        assert True
-
-    def test_flowservice_create_run(self):
-        assert True
-
-    def test_flowservice_get_run(self):
-        assert True
-
-    def test_flowservice_get_source_connections(self):
-        assert True
-
-    def test_flowservice_get_source_connection(self):
-        assert True
-
-
-    def test_flowsevrice_delete_source_connection(self):
-        assert True
-
-    def test_flowservice_create_source_connection(self):
-        assert True
-
-    def test_flowservice_create_source_connection_streaming(self):
-        assert True
-
-    def test_flowservice_create_source_connectionDataLandingZone(self):
-        assert True
-
-    def test_flowservice_create_source_connection_datalake(self):
-        assert True
-
-    def test_flowservice_update_source_connection(self):
-        assert True
-
-
-    def test_flowservice_get_target_connections(self):
-        assert True
-
-    def test_flowservice_get_target_connection(self):
-        assert True
-
-    def test_flowservice_delete_target_connection(self):
-        assert True
-
-    def test_flowservice_create_target_connection(self):
-        assert True
-
-    def test_flowservice_create_target_connection_data_landin_zone(self):
-        assert True
-
-    def test_flowservice_create_target_connection_datalake(self):
-        assert True
-
-    def test_flowservice_update_target_connection(self):
-        assert True
-
-    def test_flowservice_update_policy(self):
-        assert True
-
-    def test_flowservice_get_landing_zone_container(self):
-        assert True
-
-    def test_flowservice_get_landing_zone_credential(self):
-        assert True
-
-    def test_flowservice_explore_landing_zone(self):
-        assert True
-
-    def test_flowservice_get_landing_zone_content(self):
-        assert True

tests/schema_test.py

@@ -1,61 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.schema import Schema
-import unittest
-from unittest.mock import patch, MagicMock
-
-
-class SchemaTest(unittest.TestCase):
-
-    @patch("aepp.connector.AdobeRequest")
-    def test_schema_get_resource(self, mock_connector):
-        instance_conn = mock_connector.return_value
-        instance_conn.getData.return_value = "foo"
-        schema_obj = Schema()
-        result = schema_obj.getResource(MagicMock(), MagicMock(), MagicMock(), MagicMock())
-        assert(result is not None)
-        instance_conn.getData.assert_called_once()
-
-    @patch("aepp.connector.AdobeRequest")
-    def test_schema_update_sandbox(self, mock_connector):
-        schema_obj = Schema()
-        test_sandbox = "prod"
-        schema_obj.updateSandbox(test_sandbox)
-        assert(schema_obj.sandbox == test_sandbox)
-
-    @patch("aepp.connector.AdobeRequest")
-    def test_schema_get_stats(self, mock_connector):
-        instance_conn = mock_connector.return_value
-        instance_conn.getData.return_value = MagicMock()
-        schema_obj = Schema()
-        stats_result = schema_obj.getStats()
-        assert (stats_result is not None)
-        assert(stats_result == instance_conn.getData.return_value)
-        instance_conn.getData.assert_called_once()
-
-    @patch("aepp.connector.AdobeRequest")
-    def test_schema_get_tenant_id(self, mock_connector):
-        instance_conn = mock_connector.return_value
-        instance_conn.getStats.return_value = MagicMock()
-        schema_obj = Schema()
-        tenant_id_result = schema_obj.getTenantId()
-        assert (tenant_id_result is not None)
-        instance_conn.getData.assert_called_once()
-
-    @patch("aepp.connector.AdobeRequest")
-    def test_schema_get_behavior(self, mock_connector):
-        instance_conn = mock_connector.return_value
-        instance_conn.getData.return_value = {"results":[1234,5678]}
-        schema_obj = Schema()
-        stats_result = schema_obj.getBehaviors()
-        assert (stats_result is not None)
-        assert (stats_result == instance_conn.getData.return_value.get("results"))
-        instance_conn.getData.assert_called_once()

tests/som_test.py

@@ -1,287 +0,0 @@
-#  Copyright 2023 Adobe. All rights reserved.
-#  This file is licensed to you under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License. You may obtain a copy
-#  of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
-#  OF ANY KIND, either express or implied. See the License for the specific language
-#  governing permissions and limitations under the License.
-
-from aepp.som import Som
-import unittest
-from unittest.mock import patch, MagicMock
-
-
-class SomTest(unittest.TestCase):
-
-    def test_som_instantiation_empty(self):
-        mysom = Som()
-        assert bool(mysom.get()) == False
-
-    def test_som_instantiation_notEmpty(self):
-        mysom = Som({'data':'key'})
-        assert bool(mysom.get()) == True
-
-    def test_som_assignments_simple(self):
-        mysom = Som()
-        mysom.assign('string','1')
-        assert type(mysom.get('string')) == str
-        assert mysom.get('string') == '1'
-        mysom.assign('integer',1)
-        assert mysom.get('integer') == 1
-        mysom.assign('mylist',[0])
-        assert type(mysom.get('mylist')) == list
-        mysom.assign('a.b','test')
-        assert mysom.get('a.b') == 'test'
-        mysom.assign('myset',set([1,2]))
-        assert type(mysom.get('myset')) == set
-        mysom.assign('mytuple',(1,2))
-        assert type(mysom.get('mytuple')) == tuple
-        mysom.assign('my.fallback',None,'fallback')
-        assert mysom.get('my.fallback') == 'fallback'
-        assert mysom.get('my.fallbacks',False) == False ## testing Falsy fallback
-    
-    def test_som_assignments_list(self):
-        mysom = Som()
-        mysom.assign('mylist',[0])
-        mysom.assign('mylist',1)
-        assert type(mysom.get('mylist')) == list
-        assert len(mysom.get('mylist')) == 2
-        mysom.assign('mylist2',2,params={"type":list})
-        assert type(mysom.get('mylist2')) == list
-        mysom.assign('data.mylist.[0]',1)
-        assert type(mysom.get('data.mylist')) == list
-        assert len(mysom.get('data.mylist')) == 1
-        assert mysom.get('data.mylist.1') == True
-        mysom.assign('data.mylist.[1]',1)
-        assert len(mysom.get('data.mylist')) == 2
-        mysom.assign('data.mylist.[1]',3)
-        assert len(mysom.get('data.mylist')) == 2
-        assert mysom.get('data.mylist.1') == 3
-        mysom.assign('data.mylist',[1,2])
-        assert len(mysom.get('data.mylist')) == 3
-        assert type(mysom.get('data.mylist.2')) == list
-        mysom.assign('data.mylist',(1,2))
-        assert len(mysom.get('data.mylist')) == 4
-        assert type(mysom.get('data.mylist.3')) == tuple
-        mysom.assign('data.mylist',set([1,2]))
-        assert len(mysom.get('data.mylist')) == 5
-        assert type(mysom.get('data.mylist.4')) == set
-
-    def test_som_assignments_set(self):
-        mysom = Som()
-        mysom.assign('myset',set([0]))
-        mysom.assign('myset',1)
-        assert type(mysom.get('myset')) == set
-        assert len(mysom.get('myset')) == 2
-        mysom.assign('myset2',2,params={"type":set})
-        assert type(mysom.get('myset2')) == set
-        assert mysom.get('myset2.2') == True
-        mysom.assign('myset',[2,3])
-        assert type(mysom.get('myset')) == set
-        assert mysom.get('myset.2') == True
-        mysom.assign('myset',(3,4))
-        assert type(mysom.get('myset')) == set
-        assert mysom.get('myset.4') == True
-        mysom.assign('myset',set([4,5]))
-        assert type(mysom.get('myset')) == set
-        assert mysom.get('myset.5') == True
-    
-    def test_som_assignments_tuple(self):
-        mysom = Som()
-        mysom.assign('mytuple',tuple([0]))
-        mysom.assign('mytuple',1)
-        assert type(mysom.get('mytuple')) == tuple
-        assert len(mysom.get('mytuple')) == 1
-        mysom.assign('mytuple2',2,params={"type":tuple})
-        assert type(mysom.get('mytuple2')) == tuple
-        assert mysom.get('mytuple2.2') == True
-
-    def test_som_complex_assignment_get(self):
-        mysom = Som()
-        mysom.assign('my.deep.value','foo')
-        assert mysom.get('my.deep.value') == 'foo'
-        assert mysom.get('my.deep.values','nothing') == 'nothing'
-        assert mysom.get('my.deep.vvalues') is None
-        mysom.assign('my.deep.list',1,params={"type":list})
-        assert type(mysom.get('my.deep.list')) == list
-        assert len(mysom.get('my.deep.list')) == 1
-        mysom.assign('my.deep.list',2)
-        assert len(mysom.get('my.deep.list')) == 2
-        mysom.assign('my.deep.list',3,params={"override":True})
-        assert type(mysom.get('my.deep.list')) == int
-        mysom.assign('my.deep.list',1,params={"type":list})
-        assert type(mysom.get('my.deep.list')) == list
-        mysom.assign('my.deep.list',2,params={"type":set})
-        assert type(mysom.get('my.deep.list')) == set
-        assert len(mysom.get('my.deep.list')) == 2
-        mysom.assign('my.deep.list',3,params={"type":tuple})
-        assert type(mysom.get('my.deep.list')) == tuple
-        assert len(mysom.get('my.deep.list')) == 3
-        mysom.assign('my.deep.list',4)
-        assert type(mysom.get('my.deep.list')) == tuple
-        assert mysom.get('my.deep.list.0') == 4
-
-    def test_som_merge(self):
-        mysom = Som({"data":{'path1':'value1','list1':['value1'],'set1':set(['value1']),'tuple1':tuple(['value1'])}})
-        assert mysom.get('data.path1') == 'value1'
-        data2 = {
-                'path1':'value2',
-                'path2':'value2',
-                'list1':['value2'],
-                'list2':['value2'],
-                'set1':set(['value2']),
-                'set2':set(['value2']),
-                'tuple1':tuple(['value2']),
-                'tuple2':tuple(['value2'])
-        }
-        mysom.merge('data',data2)
-        assert mysom.get('data.path1') == 'value2'
-        assert mysom.get('data.path2') == 'value2'
-        assert type(mysom.get('data.list1')) == list
-        assert len(mysom.get('data.list1')) == 2
-        assert len(mysom.get('data.list2')) == 1
-        assert type(mysom.get('data.set1')) == set
-        assert len(mysom.get('data.set1')) == 2
-        assert len(mysom.get('data.set2')) == 1
-        assert type(mysom.get('data.tuple1')) == tuple
-        assert len(mysom.get('data.tuple1')) == 2
-        assert len(mysom.get('data.tuple2')) == 1
-        mysom.merge('data.list1',[1])
-        mysom.merge('data.tuple1',tuple([1]))
-        mysom.merge('data.set1',set([1]))
-        assert len(mysom.get('data.list1')) == 3
-        assert len(mysom.get('data.tuple1')) == 3
-        assert len(mysom.get('data.set1')) == 3
-
-    def test_som_merge_to_list(self):
-        mysom = Som({"data":{'path1':'value1','set1':set(['value1']),'tuple1':tuple(['value1'])}})
-        assert mysom.get('data.path1') == 'value1'
-        data2 = {
-                'path1':['value2'],
-                'set1':['value2'],
-                'tuple1':['value2'],
-        }
-        mysom.merge('data',data2)
-        assert type(mysom.get('data.path1')) == list
-        assert len(mysom.get('data.path1')) == 2
-        assert type(mysom.get('data.set1')) == list
-        assert len(mysom.get('data.set1')) == 2
-        assert type(mysom.get('data.tuple1')) == list
-        assert len(mysom.get('data.tuple1')) == 2
-    
-    def test_som_merge_to_set(self):
-        mysom = Som({"data":{'path1':'value1','list1':['value1'],'tuple1':tuple(['value1'])}})
-        assert mysom.get('data.path1') == 'value1'
-        data2 = {
-                'path1':set(['value2']),
-                'list1':set(['value2']),
-                'tuple1':set(['value2']),
-        }
-        mysom.merge('data',data2)
-        assert type(mysom.get('data.path1')) == set
-        assert len(mysom.get('data.path1')) == 2
-        assert type(mysom.get('data.list1')) == set
-        assert len(mysom.get('data.list1')) == 2
-        assert type(mysom.get('data.tuple1')) == set
-        assert len(mysom.get('data.tuple1')) == 2
-
-    def test_som_merge_to_tuple(self):
-        mysom = Som({"data":{'path1':'value1','list1':['value1'],'set1':set(['value1'])}})
-        assert mysom.get('data.path1') == 'value1'
-        data2 = {
-                'path1':tuple(['value2']),
-                'list1':tuple(['value2']),
-                'set1':tuple(['value2']),
-        }
-        mysom.merge('data',data2)
-        assert type(mysom.get('data.path1')) == tuple
-        assert len(mysom.get('data.path1')) == 2
-        assert type(mysom.get('data.list1')) == tuple
-        assert len(mysom.get('data.list1')) == 2
-        assert type(mysom.get('data.set1')) == tuple
-        assert len(mysom.get('data.set1')) == 2
-
-    def test_som_remove(self):
-        mysom = Som({"data":{'path1':'value1','list1':['value1','value2','value3'],'set1':set(['value1','value2','value3']),'tuple1':tuple(['value1','value2','value3'])}})
-        assert mysom.get('data.path1') == 'value1'
-        mysom.remove('data.path1')
-        assert mysom.get('data.path1') is None
-        mysom.remove('data.list1.value2')
-        assert len(mysom.get('data.list1')) == 2
-        mysom.remove('data.list1.[0]')
-        assert len(mysom.get('data.list1')) == 1
-        mysom.remove('data.list1')
-        assert mysom.get('data.list1') is None
-        mysom.remove('data.set1.value1')
-        assert len(mysom.get('data.set1')) == 2
-        mysom.remove('data.set1')
-        assert mysom.get('data.set1') is None
-        mysom.remove('data.tuple1')
-        assert mysom.get('data.tuple1') is None
-
-    def test_clear(self):
-        mysom = Som({"data":{'path1':'value1','list1':['value1','value2','value3'],'set1':set(['value1','value2','value3']),'tuple1':tuple(['value1','value2','value3'])}})
-        assert mysom.get('data.path1') == 'value1'
-        mysom.clear()
-        assert mysom.get('data.path1') is None
-        assert mysom.get('data.list1') is None
-        assert mysom.get('data.set1') is None
-        assert mysom.get('data.tuple1') is None
-
-    def test_dataframe(self):
-        data = {"data":
-            {
-                "myfield":"value",
-                "myList" : [1,2,3],
-                "arrayObject1":[
-                {
-                    "field_name":"array_item1",
-                    "arrayObject2":[
-                    {
-                        "field_name":"array_item_1_1"
-                    },
-                    {
-                        "field_name":"array_item_1_2"
-                    }
-                    ]
-                },
-                {
-                    "field_name":"array_item2"
-                }
-                ]
-            }
-            }
-        mySom = Som(data)
-        df_flatten = mySom.to_dataframe()
-        df_expands = mySom.to_dataframe(expand_arrays=True)
-        assert len(df_flatten) == 1
-        assert len(df_expands) == 3
-        list_cols_flatten = ['data.myfield',
-                            'data.myList',
-                            'data.arrayObject1.[0].field_name',
-                            'data.arrayObject1.[0].arrayObject2.[0].field_name',
-                            'data.arrayObject1.[0].arrayObject2.[1].field_name',
-                            'data.arrayObject1.[1].field_name']
-        assert all(list_cols_flatten == df_flatten.columns)
-        list_cols_expands = ['data.myfield',
-                            'data.myList',
-                            'data.arrayObject1',
-                            'data.arrayObject1.field_name',
-                            'data.arrayObject1.arrayObject2',
-                            'data.arrayObject1.arrayObject2.field_name']
-        assert all(list_cols_expands == df_expands.columns)
-        
-
-
-
-
-
-        
-
-
-
-
-
-