aepp 0.4.1.post2__py3-none-any.whl → 0.4.2__py3-none-any.whl

aepp/__init__.py

@@ -153,14 +153,15 @@ def extractSandboxArtefacts(
         completePath = mypath / f'{sandbox.sandbox}'
     else:
         completePath = Path(localFolder)
-    from aepp import schema, catalog, identity
+    from aepp import schema, catalog, identity,customerprofile, segmentation
     sch = schema.Schema(config=sandbox)
     cat = catalog.Catalog(config=sandbox)
     ide = identity.Identity(config=sandbox,region=region)
     completePath.mkdir(exist_ok=True)
     globalConfig = {
         "imsOrgId":sandbox.org_id,
-        "tenantId":f"_{sch.getTenantId()}"
+        "tenantId":f"_{sch.getTenantId()}",
+        "sandbox":sandbox.sandbox
     }
     with open(f'{completePath}/config.json','w') as f:
         json.dump(globalConfig,f,indent=2)
@@ -184,6 +185,10 @@ def extractSandboxArtefacts(
     identityPath.mkdir(exist_ok=True)
     datasetPath = completePath / 'dataset'
     datasetPath.mkdir(exist_ok=True)
+    mergePolicyPath = completePath / 'mergePolicy'
+    mergePolicyPath.mkdir(exist_ok=True)
+    audiencePath = completePath / 'audience'
+    audiencePath.mkdir(exist_ok=True)
     myclasses = sch.getClasses()
     classesGlobal = sch.getClassesGlobal()
     behaviors = sch.getBehaviors()
@@ -257,6 +262,19 @@ def extractSandboxArtefacts(
     for el in identities:
         with open(f"{identityPath / el['code']}.json",'w') as f:
             json.dump(el,f,indent=2)
+    ## merge policies
+    ups = customerprofile.Profile(config=sandbox)
+    mymergePolicies = ups.getMergePolicies()
+    for el in mymergePolicies:
+        with open(f"{mergePolicyPath / el.get('id','unknown')}.json",'w') as f:
+            json.dump(el,f,indent=2)
+    ## audiences
+    mysegmentation = segmentation.Segmentation(config=sandbox) 
+    audiences = mysegmentation.getAudiences()
+    for el in audiences:
+        safe_name = __titleSafe__(el.get('name','unknown'))
+        with open(f"{audiencePath / safe_name}.json",'w') as f:
+            json.dump(el,f,indent=2)
 
 def extractSandboxArtefact(
     sandbox: 'ConnectObject' = None,
@@ -271,7 +289,7 @@ def extractSandboxArtefact(
         sandbox: REQUIRED: the instance of a ConnectObject that contains the sandbox information and connection.
         localFolder: OPTIONAL: the local folder where to extract the sandbox. If not provided, it will use the current working directory and name the folder the name of the sandbox.
         artefact: REQUIRED: the id or the name of the artefact to export.
-        artefactType: REQUIRED: the type of artefact to export. Possible values are: 'class','schema','fieldgroup','datatype','descriptor','dataset','identity'
+        artefactType: REQUIRED: the type of artefact to export. Possible values are: 'class','schema','fieldgroup','datatype','descriptor','dataset','identity','mergepolicy'
         region: OPTIONAL: the region of the sandbox (default: nld2). This is used to fetch the correct API endpoints for the identities. 
             Possible values: "va7","aus5", "can2", "ind2"
     """
@@ -287,7 +305,8 @@ def extractSandboxArtefact(
     sch = schema.Schema(config=sandbox)
     globalConfig = {
         "imsOrgId":sandbox.org_id,
-        "tenantId":f"_{sch.getTenantId()}"
+        "tenantId":f"_{sch.getTenantId()}",
+        "sandbox":sandbox.sandbox
     }
     with open(f'{completePath}/config.json','w') as f:
         json.dump(globalConfig,f,indent=2)
@@ -307,6 +326,10 @@ def extractSandboxArtefact(
         __extractDataset__(artefact,completePath,sandbox,region)
     elif artefactType == 'identity':
         __extractIdentity__(artefact,region,completePath,sandbox)
+    elif artefactType == 'mergepolicy':
+        __extractMergePolicy__(artefact,completePath,sandbox)
+    elif artefactType == 'audience':
+        __extractAudience__(artefact,completePath,sandbox)
     else:
         raise ValueError("artefactType not recognized")
     
@@ -472,4 +495,29 @@ def __extractDataset__(dataset: str,folder: Union[str, Path] = None,sandbox: 'Co
         json.dump(myDataset,f,indent=2)
     schema = myDataset.get('schemaRef',{}).get('id',None)
     if schema is not None:
-        __extractSchema__(schema,folder,sandbox,region)
+        __extractSchema__(schema,folder,sandbox,region)
+
+def __extractMergePolicy__(mergePolicy: str = None,folder:Union[str, Path]=None, sandbox: 'ConnectObject' = None,region:str=None):
+    from aepp import customerprofile
+    ups = customerprofile.Profile(config=sandbox)
+    mymergePolicies = ups.getMergePolicies()
+    mymergePolicy = [el for el in mymergePolicies if el.get('id','') == mergePolicy or el.get('name','') == mergePolicy][0]
+    if mymergePolicy['attributeMerge'].get('type','timestampOrdered') == 'dataSetPrecedence':
+        list_ds = mymergePolicy['attributeMerge'].get('order',[])
+        for ds in list_ds:
+            __extractDataset__(ds,folder,sandbox,region)
+    mergePolicyPath = Path(folder) / 'mergePolicy'
+    mergePolicyPath.mkdir(exist_ok=True)
+    with open(f"{mergePolicyPath / mymergePolicy.get('id','unknown')}.json",'w') as f:
+        json.dump(mymergePolicy,f,indent=2)
+
+def __extractAudience__(audienceName: str = None,folder:Union[str, Path]=None, sandbox: 'ConnectObject' = None):
+    from aepp import segmentation
+    mysegmentation = segmentation.Segmentation(config=sandbox) 
+    audiences = mysegmentation.getAudiences()
+    myaudience = [el for el in audiences if el.get('name','') == audienceName or el.get('id','') == audienceName][0]
+    audiencePath = Path(folder) / 'audience'
+    audiencePath.mkdir(exist_ok=True)
+    safe_name = __titleSafe__(myaudience.get('name','unknown'))
+    with open(f"{audiencePath / safe_name}.json",'w') as f:
+        json.dump(myaudience,f,indent=2)

aepp/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.1-2"
+__version__ = "0.4.2"

aepp/synchronizer.py

@@ -10,7 +10,7 @@
 
 import json
 import aepp
-from aepp import schema, schemamanager, fieldgroupmanager, datatypemanager,classmanager,identity,catalog
+from aepp import schema, schemamanager, fieldgroupmanager, datatypemanager,classmanager,identity,catalog,customerprofile,segmentation
 from copy import deepcopy
 from typing import Union
 from pathlib import Path
@@ -65,21 +65,83 @@ class Synchronizer:
             self.identityFolder = self.localfolder / 'identity'
             self.datasetFolder = self.localfolder / 'dataset'
             self.descriptorFolder = self.localfolder / 'descriptor'
+            self.mergePolicyFolder = self.localfolder / 'mergepolicy'
+            self.audienceFolder = self.localfolder / 'audience'
             if baseSandbox is not None:
                 self.baseSandbox = baseSandbox
+            else:
+                with open(self.localfolder / 'config.json','r') as f:
+                    local_config = json.load(f)
+                    self.baseSandbox = local_config.get('sandbox',None)
         self.dict_targetsConfig = {target: aepp.configure(org_id=config_object['org_id'],client_id=config_object['client_id'],scopes=config_object['scopes'],secret=config_object['secret'],sandbox=target,connectInstance=True) for target in targets}
         self.region = region
-        self.dict_targetComponents = {target:{'schema':{},'class':{},'fieldgroup':{},'datatype':{},'datasets':{},'identities':{},"schemaDescriptors":{}} for target in targets}
+        self.dict_baseComponents = {'schema':{},'class':{},'fieldgroup':{},'datatype':{},'datasets':{},'identities':{},"schemaDescriptors":{},'mergePolicy':{},'audience':{}}  
+        self.dict_targetComponents = {target:{'schema':{},'class':{},'fieldgroup':{},'datatype':{},'datasets':{},'identities':{},"schemaDescriptors":{},'mergePolicy':{},'audience':{}} for target in targets}
 
-    def syncComponent(self,component:Union[str,dict],componentType:str=None,verbose:bool=False)-> dict:
+    def getSyncFieldGroupManager(self,fieldgroup:str,sandbox:str=None)-> dict:
+        """
+        Get a field group Manager from the synchronizer.
+        It searches through the component cache to see if the FieldGroupManager for the target sandbox is already instantiated.
+        If not, it generate an error.
+        Arguments:
+            fieldgroup : REQUIRED : Either $id, or name or alt:Id of the field group to get
+            sandbox : REQUIRED : name of the sandbox to get the field group from
+        """
+        if sandbox is None:
+            raise ValueError("a sandbox name must be provided")
+        if sandbox == self.baseSandbox:
+            if fieldgroup in self.dict_baseComponents['fieldgroup'].keys():
+                return self.dict_baseComponents['fieldgroup'][fieldgroup]
+            elif fieldgroup in [self.dict_baseComponents['fieldgroup'][fg].id for fg in self.dict_baseComponents['fieldgroup'].keys()]:
+                fg_key = [fg for fg in self.dict_baseComponents['fieldgroup'].keys() if self.dict_baseComponents['fieldgroup'][fg].id == fieldgroup][0]
+                return self.dict_baseComponents['fieldgroup'][fg_key]
+            elif fieldgroup in [self.dict_baseComponents['fieldgroup'][fg].altId for fg in self.dict_baseComponents['fieldgroup'].keys()]:
+                fg_key = [fg for fg in self.dict_baseComponents['fieldgroup'].keys() if self.dict_baseComponents['fieldgroup'][fg].altId == fieldgroup][0]
+                return self.dict_baseComponents['fieldgroup'][fg_key]
+            else:
+                raise ValueError(f"the field group '{fieldgroup}' has not been synchronized to the sandbox '{sandbox}'")
+        else:
+            if fieldgroup in self.dict_targetComponents[sandbox]['fieldgroup'].keys():
+                return self.dict_targetComponents[sandbox]['fieldgroup'][fieldgroup]
+            elif fieldgroup in [self.dict_targetComponents[sandbox]['fieldgroup'][fg].id for fg in self.dict_targetComponents[sandbox]['fieldgroup'].keys()]:
+                fg_key = [fg for fg in self.dict_targetComponents[sandbox]['fieldgroup'].keys() if self.dict_targetComponents[sandbox]['fieldgroup'][fg].id == fieldgroup][0]
+                return self.dict_targetComponents[sandbox]['fieldgroup'][fg_key]
+            elif fieldgroup in [self.dict_targetComponents[sandbox]['fieldgroup'][fg].altId for fg in self.dict_targetComponents[sandbox]['fieldgroup'].keys()]:
+                fg_key = [fg for fg in self.dict_targetComponents[sandbox]['fieldgroup'].keys() if self.dict_targetComponents[sandbox]['fieldgroup'][fg].altId == fieldgroup][0]
+                return self.dict_targetComponents[sandbox]['fieldgroup'][fg_key]
+            else:
+                raise ValueError(f"the field group '{fieldgroup}' has not been synchronized to the sandbox '{sandbox}'")
+    
+    def getDatasetName(self,datasetId:str,sandbox:str=None)-> str:
+        """
+        Get a dataset name from the synchronizer base on the ID of the dataset.
+        Arguments:
+            datasetId : REQUIRED : id of the dataset to get
+            sandbox : REQUIRED : name of the sandbox to get the dataset from
+        """
+        if sandbox is None:
+            raise ValueError("a sandbox name must be provided")
+        if sandbox == self.baseSandbox:
+            if datasetId in [item.get('id') for key,item in self.dict_baseComponents['datasets'].items()]:
+                return [key for key,item in self.dict_baseComponents['datasets'].items() if item.get('id') == datasetId][0]
+            else:
+                raise ValueError(f"the dataset '{datasetId}' has not been synchronized to the sandbox '{sandbox}'")
+        else:
+            if datasetId in [item.get('id') for key,item in self.dict_targetComponents[sandbox]['datasets'].items()]:
+                return [key for key,item in self.dict_targetComponents[sandbox]['datasets'].items() if item.get('id') == datasetId][0]
+            else:
+                raise ValueError(f"the dataset '{datasetId}' has not been synchronized to the sandbox '{sandbox}'")
+
+    def syncComponent(self,component:Union[str,dict],componentType:str=None,force:bool=False,verbose:bool=False)-> dict:
         """
         Synchronize a component to the target sandbox.
         The component could be a string (name or id of the component in the base sandbox) or a dictionary with the definition of the component.
         If the component is a string, you have to have provided a base sandbox in the constructor.
         Arguments:
             component : REQUIRED : name or id of the component or a dictionary with the component definition
-            componentType : OPTIONAL : type of the component (e.g. "schema", "fieldgroup", "datatypes", "class", "identity", "dataset"). Required if a string is passed. 
-            It is not required but if the type cannot be inferred from the component, it will raise an error. 
+            componentType : OPTIONAL : type of the component (e.g. "schema", "fieldgroup", "datatypes", "class", "identity", "dataset", "mergepolicy", "audience"). Required if a string is passed. 
+                It is not required but if the type cannot be inferred from the component, it will raise an error. 
+            force : OPTIONAL : if True, it will force the synchronization of the component even if it already exists in the target sandbox. Works for Schema, FieldGroup, DataType and Class.
             verbose : OPTIONAL : if True, it will print the details of the synchronization process
         """
         if type(component) == str:
@@ -87,8 +149,8 @@ class Synchronizer:
                 raise ValueError("a base sandbox or a local folder must be provided to synchronize a component by name or id")
             if componentType is None:
                 raise ValueError("the type of the component must be provided if the component is a string")
-            if componentType not in ['schema', 'fieldgroup', 'datatypes', 'class', 'identity', 'dataset']:
-                raise ValueError("the type of the component is not supported. Please provide one of the following types: schema, fieldgroup, datatypes, class, identity, dataset")
+            if componentType not in ['schema', 'fieldgroup', 'datatypes', 'class', 'identity', 'dataset', 'mergepolicy', 'audience']:
+                raise ValueError("the type of the component is not supported. Please provide one of the following types: schema, fieldgroup, datatypes, class, identity, dataset, mergepolicy, audience")
             if componentType in ['schema', 'fieldgroup', 'datatypes', 'class']:
                 if self.baseConfig is not None:
                     base_schema = schema.Schema(config=self.baseConfig)
@@ -165,6 +227,30 @@ class Synchronizer:
                             break
                 if len(component) == 1: ## if the component is the catalog API response {'key': {dataset definition}}
                     component = component[list(component.keys())[0]] ## accessing the real dataset definition
+            elif componentType == "mergepolicy":
+                if self.baseConfig is not None:
+                    ups_base = customerprofile.Profile(config=self.baseConfig)
+                    base_mergePolicies = ups_base.getMergePolicies()
+                    if component in [el.get('id','') for el in base_mergePolicies] or component in [el.get('name','') for el in base_mergePolicies]:
+                        component = [el for el in base_mergePolicies if el.get('id','') == component or el.get('name','') == component][0]
+                elif self.localfolder is not None:
+                    for file in self.mergePolicyFolder.glob('*.json'):
+                        mp_file = json.load(FileIO(file))
+                        if mp_file.get('id','') == component or mp_file.get('name','') == component:
+                            component = mp_file
+                            break
+            elif componentType == 'audience':
+                if self.baseConfig is not None:
+                    seg_base = segmentation.Segmentation(config=self.baseConfig)
+                    base_audiences = seg_base.getAudiences()
+                    if component in [el.get('id','') for el in base_audiences] or component in [el.get('name','') for el in base_audiences]:
+                        component = [el for el in base_audiences if el.get('id','') == component or el.get('name','') == component][0]
+                elif self.localfolder is not None:
+                    for file in self.audienceFolder.glob('*.json'):
+                        au_file = json.load(FileIO(file))
+                        if au_file.get('id','') == component or au_file.get('name','') == component:
+                            component = au_file
+                            break
         elif type(component) == dict:
             if 'meta:resourceType' in component.keys():
                 componentType = component['meta:resourceType']
@@ -180,30 +266,41 @@ class Synchronizer:
                 componentType = 'identity'
             elif 'files' in component.keys():
                 componentType = 'dataset'
+            elif 'attributeMerge' in component.keys():
+                componentType = 'mergepolicy'
+            elif 'expression' in component.keys():
+                componentType = 'audience'
             else:
                 raise TypeError("the component type could not be inferred from the component or is not supported. Please provide the type as a parameter")
         ## Synchronize the component to the target sandboxes
         if componentType == 'datatypes':
-            self.__syncDataType__(component,verbose=verbose)
+            self.__syncDataType__(component,verbose=verbose,force=force)
         if componentType == 'fieldgroup':
-            self.__syncFieldGroup__(component,verbose=verbose)
+            self.__syncFieldGroup__(component,verbose=verbose,force=force)
         if componentType == 'schema':
-            self.__syncSchema__(component,verbose=verbose)
+            self.__syncSchema__(component,verbose=verbose,force=force)
         if componentType == 'class': 
-            self.__syncClass__(component,verbose=verbose)
+            self.__syncClass__(component,verbose=verbose,force=force)
         if componentType == 'identity':
             self.__syncIdentity__(component,verbose=verbose)
         if componentType == 'dataset':
             self.__syncDataset__(component,verbose=verbose)
+        if componentType == 'mergepolicy':
+            self.__syncMergePolicy__(component,verbose=verbose)
+        if componentType == 'audience':
+            self.__syncAudience__(component,verbose=verbose)
 
-    def __syncClass__(self,baseClass:'ClassManager',verbose:bool=False)-> dict:
+            
+    def __syncClass__(self,baseClass:'ClassManager',force:bool=False,verbose:bool=False)-> dict:
         """
         Synchronize a class to the target sandboxes.
         Arguments:
             baseClass : REQUIRED : class id or name to synchronize
+            force : OPTIONAL : if True, it will force the synchronization of the class even if it already exists in the target sandbox
         """
         if not isinstance(baseClass,classmanager.ClassManager):
             raise TypeError("the baseClass must be a classManager instance")
+        self.dict_baseComponents['class'][baseClass.title] = baseClass
         baseClassName = baseClass.title
         baseBehavior = baseClass.behavior
         for target in self.dict_targetsConfig.keys():
@@ -223,14 +320,16 @@ class Synchronizer:
                 self.dict_targetComponents[target]['class'][baseClassName] = t_newClass
 
             
-    def __syncDataType__(self,baseDataType:'DataTypeManager',verbose:bool=False)-> dict:
+    def __syncDataType__(self,baseDataType:'DataTypeManager',force:bool=False,verbose:bool=False)-> dict:
         """
         Synchronize a data type to the target sandbox.
         Arguments:
             baseDataType : REQUIRED : DataTypeManager object with the data type to synchronize
+            force : OPTIONAL : if True, it will force the synchronization of the data type even if it already exists in the target sandbox
         """
         if not isinstance(baseDataType,datatypemanager.DataTypeManager):
             raise TypeError("the baseDataType must be a DataTypeManager object")
+        self.dict_baseComponents['datatype'][baseDataType.title] = baseDataType
         name_base_datatype = baseDataType.title
         for target in self.dict_targetsConfig.keys():
             targetSchema = schema.Schema(config=self.dict_targetsConfig[target])
@@ -249,7 +348,7 @@ class Synchronizer:
                 base_paths = df_base['path'].tolist()
                 target_paths = df_target['path'].tolist()
                 diff_paths = list(set(base_paths) - set(target_paths))
-                if len(diff_paths) > 0: ## there are differences
+                if len(diff_paths) > 0 or force==True: ## there are differences
                     base_datatypes_paths = baseDataType.getDataTypePaths()
                     df_base_limited = df_base[df_base['origin'] == 'self'].copy() ## exclude field group native fields
                     df_base_limited = df_base_limited[~df_base_limited['path'].isin(list(base_datatypes_paths.keys()))] ## exclude base of datatype rows
@@ -258,7 +357,7 @@ class Synchronizer:
                     base_dict_path_dtTitle = {}
                     for path,dt_id in base_datatypes_paths.items():
                         tmp_dt_manager = baseDataType.getDataTypeManager(dt_id)
-                        self.__syncDataType__(tmp_dt_manager,verbose=verbose)
+                        self.__syncDataType__(tmp_dt_manager,force=force,verbose=verbose)
                         base_dict_path_dtTitle[path] = tmp_dt_manager.title
                     target_datatypes_paths = t_datatype.getDataTypePaths(som_compatible=True)
                     target_datatypes_paths_list = list(target_datatypes_paths.keys())
@@ -289,7 +388,7 @@ class Synchronizer:
                 base_dict_path_dtTitle = {}
                 for path,dt_id in base_datatypes_paths.items():
                     tmp_dt_manager = baseDataType.getDataTypeManager(dt_id)
-                    self.__syncDataType__(tmp_dt_manager,verbose=verbose)
+                    self.__syncDataType__(tmp_dt_manager,force=force,verbose=verbose)
                     base_dict_path_dtTitle[path] = tmp_dt_manager.title
                 target_datatypes_paths = new_datatype.getDataTypePaths(som_compatible=True)
                 target_datatypes_paths_list = list(target_datatypes_paths.keys())
@@ -309,14 +408,16 @@ class Synchronizer:
                     raise Exception("the data type could not be created in the target sandbox")
             self.dict_targetComponents[target]['datatype'][name_base_datatype] = t_datatype
 
-    def __syncFieldGroup__(self,baseFieldGroup:'FieldGroupManager',verbose:bool=False)-> dict:
+    def __syncFieldGroup__(self,baseFieldGroup:'FieldGroupManager',force:bool=True,verbose:bool=False)-> dict:
         """
         Synchronize a field group to the target sandboxes.
         Argument: 
             baseFieldGroup : REQUIRED : FieldGroupManager object with the field group to synchronize
+            force : OPTIONAL : if True, it will force the synchronization of the field group even if it already exists in the target sandbox
         """
         if not isinstance(baseFieldGroup,fieldgroupmanager.FieldGroupManager):
             raise TypeError("the baseFieldGroup must be a FieldGroupManager object")
+        self.dict_baseComponents['fieldgroup'][baseFieldGroup.title] = baseFieldGroup
         name_base_fieldgroup = baseFieldGroup.title
         base_fg_classIds = baseFieldGroup.classIds
         for target in self.dict_targetsConfig.keys():
@@ -350,7 +451,7 @@ class Synchronizer:
                 base_paths = df_base['path'].tolist()
                 target_paths = df_target['path'].tolist()
                 diff_paths = [path for path in base_paths if path not in target_paths]
-                if len(diff_paths) > 0:
+                if len(diff_paths) > 0 or force==True:
                     base_datatypes_paths = baseFieldGroup.getDataTypePaths()
                     ## handling fieldgroup native fields
                     df_base_limited = df_base[df_base['origin'] == 'fieldGroup'].copy() ## exclude datatypes
@@ -371,7 +472,7 @@ class Synchronizer:
                     base_dict_path_dtTitle = {}
                     for path,dt_id in base_datatypes_paths.items():
                         tmp_dt_manager = baseFieldGroup.getDataTypeManager(dt_id)
-                        self.__syncDataType__(tmp_dt_manager,verbose=verbose)
+                        self.__syncDataType__(tmp_dt_manager,force=force,verbose=verbose)
                         base_dict_path_dtTitle[path] = tmp_dt_manager.title
                     target_datatypes_paths = t_fieldgroup.getDataTypePaths(som_compatible=True)
                     target_datatypes_paths_list = list(target_datatypes_paths.keys())
@@ -423,7 +524,7 @@ class Synchronizer:
                 base_dict_path_dtTitle = {}
                 for path,dt_id in base_datatypes_paths.items():
                     tmp_dt_manager = baseFieldGroup.getDataTypeManager(dt_id)
-                    self.__syncDataType__(tmp_dt_manager,verbose=verbose)
+                    self.__syncDataType__(tmp_dt_manager,force=force,verbose=verbose)
                     base_dict_path_dtTitle[path] = tmp_dt_manager.title
                 for path,dt_title in base_dict_path_dtTitle.items():
                     tmp_t_dt = self.dict_targetComponents[target]['datatype'][dt_title]
@@ -442,16 +543,18 @@ class Synchronizer:
             self.dict_targetComponents[target]['fieldgroup'][name_base_fieldgroup] = t_fieldgroup
 
 
-    def __syncSchema__(self,baseSchema:'SchemaManager',verbose:bool=False)-> dict:
+    def __syncSchema__(self,baseSchema:'SchemaManager',force:bool=False,verbose:bool=False)-> dict:
         """
         Sync the schema to the target sandboxes.
         Arguments:
             baseSchema : REQUIRED : SchemaManager object to synchronize
+            force : OPTIONAL : if True, it will force the synchronization of field groups even if they already exist in the target schema
         """
         ## TO DO -> sync required fields
         if not isinstance(baseSchema,schemamanager.SchemaManager):
             raise TypeError("the baseSchema must be a SchemaManager object")
         name_base_schema = baseSchema.title
+        self.dict_baseComponents['schema'][name_base_schema] = baseSchema
         descriptors = baseSchema.getDescriptors()
         base_field_groups_names = list(baseSchema.fieldGroups.values())
         dict_base_fg_name_id = {name:fg_id for fg_id,name in baseSchema.fieldGroups.items()}
@@ -465,7 +568,7 @@ class Synchronizer:
                 t_schema = schemamanager.SchemaManager(targetSchemaAPI.data.schemas_altId[name_base_schema],config=self.dict_targetsConfig[target],sandbox=target)
                 new_fieldgroups = [fg for fg in base_field_groups_names if fg not in t_schema.fieldGroups.values()]
                 existing_fieldgroups = [fg for fg in base_field_groups_names if fg in t_schema.fieldGroups.values()]
-                if len(new_fieldgroups) > 0: ## if new field groups
+                if len(new_fieldgroups) > 0 or force==True: ## if new field groups
                     if verbose:
                         print('found new field groups to add to the schema')
                     for new_fieldgroup in new_fieldgroups:
@@ -478,7 +581,8 @@ class Synchronizer:
                             if verbose:
                                 print(f"field group '{new_fieldgroup}' is a custom field group, syncing it")
                             tmp_FieldGroup = baseSchema.getFieldGroupManager(new_fieldgroup)
-                            self.__syncFieldGroup__(tmp_FieldGroup,verbose=verbose)
+                            print(f"Creating new custom field group '{tmp_FieldGroup.title}'")
+                            self.__syncFieldGroup__(tmp_FieldGroup,verbose=verbose,force=force)
                             t_schema.addFieldGroup(self.dict_targetComponents[target]['fieldgroup'][new_fieldgroup].id)
                     t_schema.setDescription(baseSchema.description)
                     res = t_schema.updateSchema()
@@ -490,13 +594,35 @@ class Synchronizer:
                 for fg_name in existing_fieldgroups:
                     if baseSchema.tenantId[1:] in dict_base_fg_name_id[fg_name]: ## custom field group
                         tmp_fieldGroupManager = fieldgroupmanager.FieldGroupManager(dict_base_fg_name_id[fg_name],config=self.baseConfig,sandbox=target,localFolder=self.localfolder)
-                        self.__syncFieldGroup__(tmp_fieldGroupManager,verbose=verbose)
+                        self.__syncFieldGroup__(tmp_fieldGroupManager,force=force,verbose=verbose)
                     else:
                         if verbose:
                             print(f"field group '{fg_name}' is a OOTB field group, using it")
                         self.dict_targetComponents[target]['fieldgroup'][fg_name] = fieldgroupmanager.FieldGroupManager(dict_base_fg_name_id[fg_name],config=self.dict_targetsConfig[target],sandbox=target)
                 list_new_descriptors = self.__syncDescriptor__(baseSchema,t_schema,targetSchemaAPI=targetSchemaAPI,verbose=verbose)
+                ## handling the meta:refProperty setup if any
+                base_allOf = baseSchema.schema.get('allOf',[])
+                base_fg_name_metaref = {}
+                for refEl in base_allOf: ## retrieving the meta:refProperty from the base schema
+                    if 'meta:refProperty' in refEl.keys():
+                        tmp_base_fg_id = refEl['$ref']
+                        if baseSchema.tenantId[1:] in tmp_base_fg_id:
+                            tmp_base_fg_manager = self.getSyncFieldGroupManager(tmp_base_fg_id,sandbox=baseSchema.sandbox)
+                            base_fg_name_metaref[tmp_base_fg_manager.title] = refEl['meta:refProperty']
+                        else:
+                            base_fg_name_metaref[tmp_base_fg_id] = refEl['meta:refProperty']
+                for fg_name,ref_property in base_fg_name_metaref.items(): ## updating the target schema with the meta:refProperty
+                    for ref in t_schema.schema.get('allOf',[]):
+                        tmp_target_fg_id = ref['$ref']
+                        if baseSchema.tenantId[1:] in tmp_target_fg_id:
+                            tmp_target_fg_manager = self.getSyncFieldGroupManager(tmp_target_fg_id,sandbox=target)
+                            if fg_name == tmp_target_fg_manager.title:
+                                ref['meta:refProperty'] = ref_property
+                        else:
+                            if fg_name == ref['$ref']:
+                                ref['meta:refProperty'] = ref_property
                 self.dict_targetComponents[target]['schemaDescriptors'][name_base_schema] = list_new_descriptors
+                t_schema.updateSchema()
             else: ## schema does not exist in target
                 if verbose:
                     print(f"schema '{name_base_schema}' does not exist in target {target}, creating it")
@@ -506,7 +632,7 @@ class Synchronizer:
                 tenantidId = baseSchema.tenantId
                 if tenantidId[1:] in baseClassId: ## custom class
                     baseClassManager = classmanager.ClassManager(baseClassId,config=self.baseConfig,sandbox=target,localFolder=self.localfolder,sandboxBase=self.baseSandbox,tenantidId=tenantidId)
-                    self.__syncClass__(baseClassManager,verbose=verbose)
+                    self.__syncClass__(baseClassManager,force=force,verbose=verbose)
                     targetClassManager = self.dict_targetComponents[target]['class'][baseClassManager.title]
                     classId_toUse = targetClassManager.id
                 else:
@@ -523,7 +649,7 @@ class Synchronizer:
                         if verbose:
                             print(f"field group '{fg_name}' is a custom field group, using it")
                         tmp_FieldGroup = baseSchema.getFieldGroupManager(fg_name)
-                        self.__syncFieldGroup__(tmp_FieldGroup,verbose=verbose)
+                        self.__syncFieldGroup__(tmp_FieldGroup,force=force,verbose=verbose)
                         new_schema.addFieldGroup(self.dict_targetComponents[target]['fieldgroup'][fg_name].id)
                 new_schema.setDescription(baseSchema.description)
                 res = new_schema.createSchema()
@@ -535,6 +661,28 @@ class Synchronizer:
                 ## handling descriptors
                 list_new_descriptors = self.__syncDescriptor__(baseSchema,t_schema,targetSchemaAPI,verbose=verbose)
                 self.dict_targetComponents[target]['schemaDescriptors'][name_base_schema] = list_new_descriptors
+                ## handling the meta:refProperty setup if any
+                base_allOf = baseSchema.schema.get('allOf',[])
+                base_fg_name_metaref = {}
+                for refEl in base_allOf: ## retrieving the meta:refProperty from the base schema
+                    if 'meta:refProperty' in refEl.keys():
+                        tmp_base_fg_id = refEl['$ref']
+                        if baseSchema.tenantId[1:] in tmp_base_fg_id:
+                            tmp_base_fg_manager = self.getSyncFieldGroupManager(tmp_base_fg_id,sandbox=baseSchema.sandbox)
+                            base_fg_name_metaref[tmp_base_fg_manager.title] = refEl['meta:refProperty']
+                        else:
+                            base_fg_name_metaref[tmp_base_fg_id] = refEl['meta:refProperty']
+                for fg_name,ref_property in base_fg_name_metaref.items(): ## updating the target schema with the meta:refProperty
+                    for ref in t_schema.schema.get('allOf',[]):
+                        tmp_target_fg_id = ref['$ref']
+                        if baseSchema.tenantId[1:] in tmp_target_fg_id:
+                            tmp_target_fg_manager = self.getSyncFieldGroupManager(tmp_target_fg_id,sandbox=target)
+                            if fg_name == tmp_target_fg_manager.title:
+                                ref['meta:refProperty'] = ref_property
+                        else:
+                            if fg_name == ref['$ref']:
+                                ref['meta:refProperty'] = ref_property
+                t_schema.updateSchema()
             self.dict_targetComponents[target]['schema'][name_base_schema] = t_schema
 
     def __syncDescriptor__(self,baseSchemaManager:'SchemaManager'=None,targetSchemaManager:'SchemaManager'=None,targetSchemaAPI:'Schema'=None,verbose:bool=False)-> dict:
@@ -553,6 +701,7 @@ class Synchronizer:
         if not isinstance(targetSchemaManager,schemamanager.SchemaManager):
             raise TypeError("the targetSchemaManager must be a SchemaManager object")
         base_descriptors = baseSchemaManager.getDescriptors()
+        self.dict_baseComponents['schemaDescriptors'][baseSchemaManager.title] = {}
         if self.baseConfig is not None:
             baseSchemaAPI = schema.Schema(config=self.baseConfig)
             myschemas = baseSchemaAPI.getSchemas() ## to populate the data object
@@ -717,6 +866,7 @@ class Synchronizer:
         if not isinstance(identityDefiniton,dict):
             raise TypeError("the identityDefinition must be a dictionary")
         code_base_identity = identityDefiniton['code']
+        self.dict_baseComponents['identities'][code_base_identity] = identityDefiniton
         for target in self.dict_targetsConfig.keys():
             targetIdentity = identity.Identity(config=self.dict_targetsConfig[target],region=self.region)
             t_identities = targetIdentity.getIdentities()
@@ -742,7 +892,8 @@ class Synchronizer:
             baseDataset : REQUIRED : dictionary with the dataset definition
         """
         if len(baseDataset) == 1: ## if receiving the dataset as provided by the API {datasetId:{...definition}}
-            baseDataset = deepcopy(baseDataset[list(baseDataset.keys()[0])])
+            baseDataset = deepcopy(baseDataset[list(baseDataset.keys())[0]])
+        self.dict_baseComponents['datasets'][baseDataset['name']] = baseDataset
         base_datasetName = baseDataset['name']
         base_dataset_related_schemaId = baseDataset['schemaRef']['id']
         if self.baseConfig is not None:
@@ -786,4 +937,94 @@ class Synchronizer:
                 t_schemas = targetSchema.getSchemas()
                 baseSchemaManager = schemamanager.SchemaManager(base_dataset_related_schemaId,config=self.baseConfig,localFolder=self.localfolder,sandbox=self.baseSandbox)
                 self.__syncSchema__(baseSchemaManager,verbose=verbose)
+                self.dict_targetComponents[target]['datasets'][base_datasetName] = t_dataset
+
+    def __syncMergePolicy__(self,mergePolicy:dict,verbose:bool=False)->None:
+        """
+        Synchronize the dataset to the target sandboxes. Mostly creating a new dataset and associated artefacts when not already created.
+        Arguments:
+            mergePolicy : REQUIRED : The merge policy dictionary to sync
+        """
+        if not isinstance(mergePolicy,dict):
+            raise TypeError("the mergePolicy must be a dictionary")
+        self.dict_baseComponents['mergePolicy'][mergePolicy.get('id','unknown')] = mergePolicy
+        mergePolicy_name = mergePolicy.get('name','unknown')
+        if mergePolicy['attributeMerge'].get('type','timestampOrdered') == 'dataSetPrecedence':
+            if verbose:
+                print(f"handling dataset precedence for merge policy '{mergePolicy_name}'")
+                print("syncing the datasets involved in the precedence order")
+            base_list_precedenceDatasets = mergePolicy['attributeMerge'].get('order',[])
+            for ds_id in base_list_precedenceDatasets:
+                res = self.syncComponent(ds_id,componentType='dataset',verbose=verbose)
+        for target in self.dict_targetsConfig.keys():
+            targetCustomerProfile = customerprofile.Profile(config=self.dict_targetsConfig[target])
+            t_mergePolicies = targetCustomerProfile.getMergePolicies()
+            if mergePolicy_name not in [el.get('name','') for el in t_mergePolicies]: ## merge policy does not exist in target
+                if verbose:
+                    print(f"merge policy '{mergePolicy_name}' does not exist in target {target}, creating it")
+                mergePolicyDef = {
+                    "name":mergePolicy.get('name',''),
+                    "schema":mergePolicy.get('schema','_xdm.context.profile'),
+                    "identityGraph":mergePolicy.get('identityGraph','pdg'),
+                    "isActiveOnEdge":mergePolicy.get('isActiveOnEdge',False),
+                }
+                if mergePolicy['attributeMerge'].get('type','timestampOrdered') == 'dataSetPrecedence':
+                    target_list_precedenceDatasets = []
+                    for base_ds_id in mergePolicy['attributeMerge'].get('order',[]):
+                        base_ds_name = self.getDatasetName(base_ds_id,sandbox=target)
+                        target_ds_id = self.dict_targetComponents[target]['datasets'][base_ds_name]['id']
+                        target_list_precedenceDatasets.append(target_ds_id)
+                    mergePolicyDef['attributeMerge'] = {
+                        "type":mergePolicy['attributeMerge'].get('type','timestampOrdered'),
+                        "order":target_list_precedenceDatasets
+                    }
+                else:
+                    mergePolicyDef['attributeMerge'] = {'type':'timestampOrdered'}
+                res = targetCustomerProfile.createMergePolicy(mergePolicyDef)
+                if 'id' in res.keys():
+                    self.dict_targetComponents[target]['mergePolicy'][res['id']] = res
+                else:
+                    print(res)
+                    raise Exception("the merge policy could not be created in the target sandbox")
+            else: ## merge policy already exists in target
+                if verbose:
+                    print(f"merge policy '{mergePolicy_name}' already exists in target {target}, saving it")
+                self.dict_targetComponents[target]['mergePolicy'][mergePolicy_name] = [el for el in t_mergePolicies if el.get('name','') == mergePolicy_name][0]
+
+    def __syncAudience__(self,baseAudience:dict,verbose:bool=False)-> None:
+        """
+        Synchronize an audience to the target sandboxes.
+        Arguments:
+            baseAudience : REQUIRED : dictionary with the audience definition
+        """
+        if not isinstance(baseAudience,dict):
+            raise TypeError("the baseAudience must be a dictionary")
+        audience_name = baseAudience.get('name','unknown')
+        self.dict_baseComponents['audience'][audience_name] = baseAudience
+        for target in self.dict_targetsConfig.keys():
+            targetAudiences = segmentation.Segmentation(config=self.dict_targetsConfig[target])
+            t_audiences = targetAudiences.getAudiences()
+            if audience_name not in [el['name'] for el in t_audiences]: ## audience does not exist in target
+                if verbose:
+                    print(f"audience '{audience_name}' does not exist in target {target}, creating it")
+                audienceDef = {
+                    "name":baseAudience.get('name',''),
+                    "description":baseAudience.get('description',''),
+                    "type":baseAudience.get('type','SegmentDefinition'),
+                    "schema":baseAudience.get('schema','_xdm.context.profile'),
+                    "expression":baseAudience.get('expression',[]),
+                    "ansibleDataModel":baseAudience.get('ansibleDataModel',{}),
+                    "profileInstanceId":baseAudience.get('profileInstanceId',''),
+                    "evaluationInfo":baseAudience.get('evaluationInfo',{'batch': {'enabled': True}, 'continuous': {'enabled': False},'synchronous': {'enabled': False}})
+                }
+                res = targetAudiences.createAudience(audienceDef)
+                if 'id' in res.keys():
+                    self.dict_targetComponents[target]['audience'][res['id']] = res
+                else:
+                    print(res)
+                    raise Exception("the audience could not be created in the target sandbox")
+            else: ## audience already exists in target
+                if verbose:
+                    print(f"audience '{audience_name}' already exists in target {target}, saving it")
+                self.dict_targetComponents[target]['audience'][audience_name] = [el for el in t_audiences if el['name'] == audience_name][0]
 

{aepp-0.4.1.post2.dist-info → aepp-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aepp
-Version: 0.4.1.post2
+Version: 0.4.2
 Summary: Package to manage AEP API endpoint and some helper functions
 Home-page: https://github.com/adobe/aepp
 Author: Julien Piccini

{aepp-0.4.1.post2.dist-info → aepp-0.4.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
-aepp/__init__.py,sha256=igpC_xiVmPVq3IuG97OgBbcOfCSpke_9pFVzZ-5EhNo,23790
-aepp/__version__.py,sha256=K2OTN9vEj3gHG8ymfOgH7-v5pOLPLqTym-Ct75s_75s,23
+aepp/__init__.py,sha256=d5hE6VcHz52B5o_AbYO368iG8FTsJlt91nBWej8WL8A,26381
+aepp/__version__.py,sha256=Y-I6FXKes4hB39WcBm3HZ1OCtCWiErPB6YZ-YSgexuE,21
 aepp/accesscontrol.py,sha256=PB3FcrO4bvDjdNxjHx7p_20hp4ahBXewoOSxuTGMXC8,17423
 aepp/catalog.py,sha256=hK9m3SAP0fhgkYqu14Tcfq14qBhw54tLCOF0mH31b1M,68237
 aepp/classmanager.py,sha256=CTYGkg5ygB8HtRia6DfT9WLBqXJOVg7pSM9jBB25Bqw,64707
@@ -31,10 +31,10 @@ aepp/schemamanager.py,sha256=hwItd4vXsPFeV25gX1Fbeiu07-BCg4z_VRQREMgJZ58,50738
 aepp/segmentation.py,sha256=oSgR2yx4nawYN5XAeHV_wefvmXEf0nb-bCguaDmp8F8,43555
 aepp/sensei.py,sha256=oYNy5BSWAEqsDkEexcQso6NfA6ntGGMnCOyHri0pJs8,7761
 aepp/som.py,sha256=XNm_Lu2wt2kpSSpldLptuER2eludFXeO9fI6i3iNCzo,34175
-aepp/synchronizer.py,sha256=B_2G55b-WXVGQFzmCQF7WisUJIwj1XFNlS8gpG1MI0M,59480
+aepp/synchronizer.py,sha256=UA48GKJ5OaasMxbS2O7A9T1i6HrehRx8_7jtxaNFD9M,77388
 aepp/tags.py,sha256=t2qBallTcWR4IOXcDBmrPpqjbSay1z3E2bcRijzVm1s,17641
 aepp/utils.py,sha256=tG-YVXylm38-bynqfp5N_Mzyo7mhlZj-dLo7wLoO4tM,1200
-aepp-0.4.1.post2.dist-info/licenses/LICENSE,sha256=HjYTlfne3BbS5gNHzNqJ5COCiTQLUdf87QkzRyFbE4Y,10337
+aepp-0.4.2.dist-info/licenses/LICENSE,sha256=HjYTlfne3BbS5gNHzNqJ5COCiTQLUdf87QkzRyFbE4Y,10337
 tests/__init__.py,sha256=d6zWJsJFZrQd5wAYM7sezSxwXbuMMWfNPkK_vpaUzFA,623
 tests/catalog_test.py,sha256=O4kkG0C_dXk3E77pSzWIt1ewfyKjfZqgbJmBwWwx0po,2246
 tests/dataaccess_test.py,sha256=bnHwOjPPauTM8s1c6O7iUYC--gqt6tPzT94aEZHDC-c,1238
@@ -44,7 +44,7 @@ tests/exportDatasetToDatalandingZone_test.py,sha256=193AgQR8yhnQmRWV9pgYz1X2Hz-Y
 tests/flowservice_test.py,sha256=Y1mpYWbKYL_x-ZlIY-EuOuNvlzVV1ERlKseDO7gN3Ss,4208
 tests/schema_test.py,sha256=6UsgdsizKmii1hzREpBEKWvZouXdJMvU68UKSxlt1uk,2774
 tests/som_test.py,sha256=a4ut0pEg1HJVMTESaPITmj7YkF54eWCMzKxTMIS-VvM,12101
-aepp-0.4.1.post2.dist-info/METADATA,sha256=IQzC5C0ZRq_STMJgqONVT9yoVosg_E8kRL1VxuhhS50,5476
-aepp-0.4.1.post2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aepp-0.4.1.post2.dist-info/top_level.txt,sha256=Gn88pv1ywuEAgOvhmmXXhN4dosEfCrBNDskje3nqS34,11
-aepp-0.4.1.post2.dist-info/RECORD,,
+aepp-0.4.2.dist-info/METADATA,sha256=nCyugz0f9W1kg7B-VVHAlYrKQt7Pifwq4Jh7CJ0SmRc,5470
+aepp-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aepp-0.4.2.dist-info/top_level.txt,sha256=Gn88pv1ywuEAgOvhmmXXhN4dosEfCrBNDskje3nqS34,11
+aepp-0.4.2.dist-info/RECORD,,