aepp 0.5.1.post1__tar.gz → 0.5.2.post1__tar.gz

{aepp-0.5.1.post1/aepp.egg-info → aepp-0.5.2.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aepp
-Version: 0.5.1.post1
+Version: 0.5.2.post1
 Summary: Package to manage AEP API endpoint and some helper functions
 Author-email: Julien Piccini <piccini.julien@gmail.com>
 License: Apache-2.0
@@ -22,6 +22,7 @@ Requires-Dist: tenacity
 Requires-Dist: deprecation
 Requires-Dist: datamodel-code-generator
 Requires-Dist: rich
+Requires-Dist: matplotlib
 Dynamic: license-file
 
 # Adobe Experience Platform API made for humans

aepp-0.5.2.post1/aepp/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.2-1"

{aepp-0.5.1.post1 → aepp-0.5.2.post1}/aepp/classmanager.py

@@ -23,7 +23,7 @@ class ClassManager:
             schemaAPI:'Schema'=None,
             config: Union[dict,ConnectObject] = aepp.config.config_object,
             description: str = 'power by aepp',
-            localFolder:str=None,
+            localFolder:str | list | None = None,
             sandbox:str=None,
             **kwargs
             )->None:
@@ -48,13 +48,16 @@ class ClassManager:
         elif config is not None and localFolder is None:
             self.schemaAPI = Schema(config=config)
         elif localFolder is not None:
-            self.localfolder = Path(localFolder)
-            if self.localfolder.exists() is False:
+            if isinstance(localFolder, str):
+                self.localfolder = [Path(localFolder)]
+            elif isinstance(localFolder, list):
+                self.localfolder = [Path(lf) for lf in localFolder]
+            if any([folder.exists() is False for folder in self.localfolder]):
                 raise Exception(f"The local folder {self.localfolder} does not exist. Please create it and extract your sandbox before using it.")
             self.schemaAPI = None
-            self.classFolder = self.localfolder / 'class'
-            self.behavFolder = self.localfolder / 'behaviour'
-            if self.classFolder.exists() is False:
+            self.classFolder = [folder / 'class' for folder in self.localfolder]
+            self.behavFolder = [folder / 'behaviour' for folder in self.localfolder]
+            if any([folder.exists() is False for folder in self.classFolder]):
                 raise Exception(f"The local folder {self.classFolder} does not exist. Please create it and extract your sandbox before using it.")
         else:
             raise Exception("You need to provide a schemaAPI instance or a config object to connect to the API or a local folder to use the local storage")
@@ -71,9 +74,11 @@ class ClassManager:
         elif type(aepclass) == dict:
             self.tenantId = aepclass.get('meta:tenantNamespace',"  ")
         elif self.localfolder is not None:
-            config_json = json.load(FileIO(self.localfolder / 'config.json'))
-            if config_json.get('tenantId',None) is not None:
-                self.tenantId = config_json.get('tenantId')
+            for folder in self.localfolder:
+                config_json = json.load(FileIO(folder / 'config.json'))
+                if config_json.get('tenantId',None) is not None:
+                    self.tenantId = config_json.get('tenantId')
+                    break
         else:
             self.tenantId = "  "
         if type(aepclass) == dict:
@@ -82,10 +87,15 @@ class ClassManager:
                     self.aepclass = self.schemaAPI.getClass(aepclass['$id'],full=False,xtype='xed')
                     self.EDITABLE = True
                 elif self.localfolder is not None:
-                    for json_file in self.classFolder.glob('*.json'):
-                        tmp_def = json.load(FileIO(json_file))
-                        if tmp_def.get('$id') == aepclass['$id']:
-                            self.aepclass = tmp_def
+                    found = False
+                    for folder in self.classFolder:
+                        for json_file in folder.glob('*.json'):
+                            tmp_def = json.load(FileIO(json_file))
+                            if tmp_def.get('$id') == aepclass['$id']:
+                                self.aepclass = tmp_def
+                                found = True
+                                break
+                        if found:
                             break
                     self.EDITABLE = False
             elif self.tenantId[1:] not in aepclass['$id']:
@@ -93,10 +103,15 @@ class ClassManager:
                     self.aepclass = self.schemaAPI.getClass(aepclass['$id'],full=True,xtype='xed')
                     self.EDITABLE = False
                 elif self.localfolder is not None:
-                    for json_file in self.classFolder.glob('*.json'):
-                        tmp_def = json.load(FileIO(json_file))
-                        if tmp_def.get('$id') == aepclass['$id']:
-                            self.aepclass = tmp_def
+                    found = False
+                    for folder in self.classFolder:
+                        for json_file in folder.glob('*.json'):
+                            tmp_def = json.load(FileIO(json_file))
+                            if tmp_def.get('$id') == aepclass['$id']:
+                                self.aepclass = tmp_def
+                                found = True
+                                break
+                        if found:
                             break
                     self.EDITABLE = False
             self.__setAttributes__(self.aepclass)
@@ -106,20 +121,30 @@ class ClassManager:
                     self.aepclass = self.schemaAPI.getClass(aepclass,full=False,xtype='xed')
                     self.EDITABLE = True
                 elif self.localfolder is not None:
-                    for json_file in self.classFolder.glob('*.json'):
-                        tmp_def = json.load(FileIO(json_file))
-                        if tmp_def.get('$id') == aepclass or tmp_def.get('meta:altId') == aepclass:
-                            self.aepclass = tmp_def
+                    found = False
+                    for folder in self.classFolder:
+                        for json_file in folder.glob('*.json'):
+                            tmp_def = json.load(FileIO(json_file))
+                            if tmp_def.get('$id') == aepclass or tmp_def.get('meta:altId') == aepclass:
+                                self.aepclass = tmp_def
+                                found = True
+                                break
+                        if found:
                             break
                     self.EDITABLE = False
             else:
                 if self.schemaAPI is not None:
                     self.aepclass = self.schemaAPI.getClass(aepclass,full=True,xtype='xed')
                 elif self.localfolder is not None:
-                    for json_file in self.classFolder.glob('*.json'):
-                        tmp_def = json.load(FileIO(json_file))
-                        if tmp_def.get('$id') == aepclass or tmp_def.get('meta:altId') == aepclass:
-                            self.aepclass = tmp_def
+                    found = False
+                    for folder in self.classFolder:
+                        for json_file in folder.glob('*.json'):
+                            tmp_def = json.load(FileIO(json_file))
+                            if tmp_def.get('$id') == aepclass or tmp_def.get('meta:altId') == aepclass:
+                                self.aepclass = tmp_def
+                                found = True
+                                break
+                        if found:
                             break
                 self.EDITABLE = False
             self.__setAttributes__(self.aepclass)
@@ -168,10 +193,15 @@ class ClassManager:
         if self.schemaAPI is not None:
             self.behaviorDefinition = self.schemaAPI.getBehavior(behavId,full=True,xtype='xed')
         elif self.localfolder is not None:
-            for json_file in self.behavFolder.glob('*.json'):
-                tmp_def = json.load(FileIO(json_file))
-                if tmp_def.get('$id') == behavId:
-                    self.behaviorDefinition = tmp_def
+            found = False
+            for folder in self.behavFolder:
+                for json_file in folder.glob('*.json'):
+                    tmp_def = json.load(FileIO(json_file))
+                    if tmp_def.get('$id') == behavId:
+                        self.behaviorDefinition = tmp_def
+                        found = True
+                        break
+                if found:
                     break
         self.requiredFields = set()
     

{aepp-0.5.1.post1 → aepp-0.5.2.post1}/aepp/cli/__main__.py

@@ -1,7 +1,7 @@
 from ast import arg
 from matplotlib.pyplot import table
 import aepp
-from aepp import synchronizer, schema, schemamanager, fieldgroupmanager, datatypemanager, identity, queryservice,catalog,flowservice
+from aepp import synchronizer, schema, schemamanager, fieldgroupmanager, datatypemanager, identity, queryservice,catalog,flowservice,sandboxes, segmentation
 import argparse, cmd, shlex, json
 from functools import wraps
 from rich.console import Console
@@ -47,26 +47,25 @@ class ServiceShell(cmd.Cmd):
                 self.sandbox = str(dict_config.get("sandbox-name","prod"))
             else:
                 self.sandbox = str(kwargs.get("sandbox","prod"))
-            self.secret = str(dict_config.get("secret",kwargs.get("secret")))
-            self.org_id = str(dict_config.get("org_id",kwargs.get("org_id")))
-            self.client_id = str(dict_config.get("client_id",kwargs.get("client_id")))
-            self.scopes = str(dict_config.get("scopes",kwargs.get("scopes")))
+            self.secret = dict_config.get("secret",kwargs.get("secret"))
+            self.org_id = dict_config.get("org_id",kwargs.get("org_id"))
+            self.client_id = dict_config.get("client_id",kwargs.get("client_id"))
+            self.scopes = dict_config.get("scopes",kwargs.get("scopes"))
         else:
-            self.sandbox = str(kwargs.get("sandbox","prod"))
-            self.secret = str(kwargs.get("secret"))
-            self.org_id = str(kwargs.get("org_id"))
-            self.client_id = str(kwargs.get("client_id"))
-            self.scopes = str(kwargs.get("scopes"))
-        self.connectInstance = True
+            self.sandbox:str|None = kwargs.get("sandbox","prod")
+            self.secret:str|None = kwargs.get("secret")
+            self.org_id:str|None = kwargs.get("org_id")
+            self.client_id:str|None = kwargs.get("client_id")
+            self.scopes:str|None = kwargs.get("scopes")
         if self.sandbox is not None and self.secret is not None and self.org_id is not None and self.client_id is not None and self.scopes is not None:
             print("Configuring connection...")
             self.config = aepp.configure(
-                connectInstance=self.connectInstance,
                 sandbox=self.sandbox,
                 secret=self.secret,
                 org_id=self.org_id,
                 client_id=self.client_id,
-                scopes=self.scopes
+                scopes=self.scopes,
+                connectInstance=self.connectInstance
             )
             self.prompt = f"{self.config.sandbox}> "
             console.print(Panel(f"Connected to [bold green]{self.sandbox}[/bold green]", style="blue"))
@@ -82,7 +81,6 @@ class ServiceShell(cmd.Cmd):
         console.print(f"Configuration file created at {Path.cwd() / Path(filename_json)}", style="green")
         return
 
-
     # # --- Commands ---
     def do_config(self, arg:Any) -> None:
         """connect to an AEP instance"""
@@ -94,14 +92,14 @@ class ServiceShell(cmd.Cmd):
         parser.add_argument("-cid", "--client_id", help="Auto-login client ID")
         parser.add_argument("-cf", "--config_file", help="Path to config file", default=None)
         args = parser.parse_args(shlex.split(arg))
-        if args.config_file:
+        if args.config_file is not None:
             mypath = Path.cwd()
             dict_config = json.load(FileIO(mypath / Path(args.config_file)))
-            self.sandbox = str(args.sandbox) if args.sandbox else str(dict_config.get("sandbox-name",args.sandbox))
-            self.secret = str(dict_config.get("secret",args.secret))
-            self.org_id = str(dict_config.get("org_id",args.org_id))
-            self.client_id = str(dict_config.get("client_id",args.client_id))
-            self.scopes = str(dict_config.get("scopes",args.scopes))
+            self.sandbox:str|None = args.sandbox if args.sandbox else dict_config.get("sandbox-name",args.sandbox)
+            self.secret:str|None = dict_config.get("secret",args.secret)
+            self.org_id:str|None = dict_config.get("org_id",args.org_id)
+            self.client_id:str|None = dict_config.get("client_id",args.client_id)
+            self.scopes:str|None = dict_config.get("scopes",args.scopes)
             self.connectInstance = True
         else:
             if args.sandbox: self.sandbox = str(args.sandbox)
@@ -136,6 +134,42 @@ class ServiceShell(cmd.Cmd):
         else:
             console.print(Panel("(!) You must configure the connection first using the 'config' command.", style="red"))
     
+    @login_required
+    def do_get_sandboxes(self, args:Any) -> None:
+        """List all sandboxes for the current organization"""
+        parser = argparse.ArgumentParser(prog='get_sandboxes', add_help=True)
+        parser.add_argument("-sv", "--save",help="Save sandboxes to CSV file")
+        try:
+            args = parser.parse_args(shlex.split(args))
+            aepp_sandboxes = sandboxes.Sandboxes(config=self.config)
+            sandboxes_list = aepp_sandboxes.getSandboxes()
+            if sandboxes_list:
+                table = Table(title=f"Sandboxes in Org: {self.config.org_id}")
+                table.add_column("Name", style="cyan")
+                table.add_column("Title", style="magenta")
+                table.add_column("Type", style="green")
+                table.add_column("Region", style="yellow")
+                table.add_column("Created", style="medium_violet_red")
+                for sb in sandboxes_list:
+                    table.add_row(
+                        sb.get("name","N/A"),
+                        sb.get("title","N/A"),
+                        sb.get("type","N/A"),
+                        sb.get("region","N/A"),
+                        sb.get("createdDate","N/A"),
+                    )
+                console.print(table)
+                if args.save:
+                    df_sandboxes = pd.DataFrame(sandboxes_list)
+                    df_sandboxes.to_csv(f"sandboxes_{self.config.org_id}.csv", index=False)
+                    console.print(f"Sandboxes exported to sandboxes_{self.config.org_id}.csv", style="green")
+            else:
+                console.print("(!) No sandboxes found.", style="red")
+        except Exception as e:
+            console.print(f"(!) Error: {str(e)}", style="red")
+        except SystemExit:
+            return
+
     
     @login_required
     def do_get_schemas(self, args:Any) -> None:
@@ -740,7 +774,7 @@ class ServiceShell(cmd.Cmd):
                     ds.get("name","N/A"),
                     datetime.fromtimestamp(ds.get("created",1000)/1000).isoformat().split('T')[0],
                     str(ds.get("dataIngested",False)),
-                    ds.get("classification",{}).get("dataBehavior","unknown")
+                    ds.get('classification').get('dataBehavior','N/A'),
                 )
             console.print(table)
         except Exception as e:
@@ -748,6 +782,74 @@ class ServiceShell(cmd.Cmd):
         except SystemExit:
             return
     
+    @login_required
+    def do_get_datasets_tableName(self, args:Any) -> None:
+        parser = argparse.ArgumentParser(prog='get_datasets', add_help=True)
+        try:
+            args = parser.parse_args(shlex.split(args))
+            aepp_cat = catalog.Catalog(config=self.config)
+            datasets = aepp_cat.getDataSets(output='list')
+            table = Table(title=f"Datasets in Sandbox: {self.config.sandbox}")
+            table.add_column("Name", style="white")
+            table.add_column("Table Name", style="cyan",no_wrap=True)
+            table.add_column("Data Type", style="red")
+            for ds in datasets:
+                table.add_row(
+                    ds.get("name","N/A"),
+                    ds.get('tags',{}).get('adobe/pqs/table',["N/A"])[0],
+                    ds.get('classification').get('dataBehavior','N/A'),
+                )
+            console.print(table)
+        except Exception as e:
+            console.print(f"(!) Error: {str(e)}", style="red")
+        except SystemExit:
+            return
+        
+    @login_required
+    def do_get_observable_schema_json(self,args:Any) -> None:
+        """Get the observable schema for a dataset by name or ID"""
+        parser = argparse.ArgumentParser(prog='get_observable_schema', add_help=True)
+        parser.add_argument("dataset", help="Dataset ID or Dataset Name to retrieve observable schema for",type=str)
+        try:
+            args = parser.parse_args(shlex.split(args))
+            aepp_cat = catalog.Catalog(config=self.config)
+            datasets = aepp_cat.getDataSets(output='list')
+            for ds in datasets:
+                if ds.get("name","") == args.dataset or ds.get("id","") == args.dataset:
+                    datasetId = ds.get("id")
+            schema_json = aepp_cat.getDataSetObservableSchema(datasetId=datasetId,appendDatasetInfo=True)
+            myObs = catalog.ObservableSchemaManager(schema_json,config=self.config)
+            data = myObs.to_dict()
+            with open(f"{args.dataset}_observable_schema.json", 'w') as f:
+                json.dump(data, f, indent=4)
+            console.print(f"Saved Observable schema to {args.dataset}_observable_schema.json.", style="green")
+        except Exception as e:
+            console.print(f"(!) Error: {str(e)}", style="red")
+        except SystemExit:
+            return
+
+    @login_required
+    def do_get_observable_schema_csv(self,args:Any) -> None:
+        """Get the observable schema for a dataset by name or ID"""
+        parser = argparse.ArgumentParser(prog='get_observable_schema', add_help=True)
+        parser.add_argument("dataset", help="Dataset ID or Dataset Name to retrieve observable schema for",type=str)
+        try:
+            args = parser.parse_args(shlex.split(args))
+            aepp_cat = catalog.Catalog(config=self.config)
+            datasets = aepp_cat.getDataSets(output='list')
+            for ds in datasets:
+                if ds.get("name","") == args.dataset or ds.get("id","") == args.dataset:
+                    datasetId = ds.get("id")
+            schema_json = aepp_cat.getDataSetObservableSchema(datasetId=datasetId,appendDatasetInfo=True)
+            myObs = catalog.ObservableSchemaManager(schema_json,config=self.config)
+            data = myObs.to_dataframe()
+            data.to_csv(f"{args.dataset}_observable_schema.csv", index=False)
+            console.print(f"Saved Observable schema to {args.dataset}_observable_schema.csv.", style="green")
+        except Exception as e:
+            console.print(f"(!) Error: {str(e)}", style="red")
+        except SystemExit:
+            return
+    
     @login_required
     def do_get_datasets_infos(self, args:Any) -> None:
         """List all datasets in the current sandbox"""
@@ -756,23 +858,58 @@ class ServiceShell(cmd.Cmd):
             args = parser.parse_args(shlex.split(args))
             aepp_cat = catalog.Catalog(config=self.config)
             datasets = aepp_cat.getDataSets()
+            aepp_cat.data.infos = aepp_cat.data.infos.sort_values(by=['ups_storageSize','datalake_storageSize'], ascending=False)
             aepp_cat.data.infos.to_csv(f"{aepp_cat.sandbox}_datasets_infos.csv",index=False)
             console.print(f"Datasets infos exported to {aepp_cat.sandbox}_datasets_infos.csv", style="green")
             table = Table(title=f"Datasets in Sandbox: {self.config.sandbox}")
             table.add_column("ID", style="white")
             table.add_column("Name", style="white",no_wrap=True)
-            table.add_column("Datalake_rows", style="blue")
-            table.add_column("Datalake_storage", style="blue")
-            table.add_column("UPS_rows", style="magenta")
-            table.add_column("UPS_storage", style="magenta")
+            table.add_column("UPS Rows", style="cyan")
+            table.add_column("UPS Storage Size", style="green")
+            table.add_column("Datalake Rows", style="magenta")
+            table.add_column("Datalake Storage Size", style="yellow")
             for _, ds in aepp_cat.data.infos.iterrows():
                 table.add_row(
                     ds.get("id","N/A"),
                     ds.get("name","N/A"),
+                    str(ds.get("ups_rows","N/A")),
+                    str(ds.get("ups_storageSize","N/A")),
                     str(ds.get("datalake_rows","N/A")),
                     str(ds.get("datalake_storageSize","N/A")),
-                    str(ds.get("ups_rows","N/A")),
-                    str(ds.get("ups_storageSize","N/A"))
+                )
+            console.print(table)
+        except Exception as e:
+            console.print(f"(!) Error: {str(e)}", style="red")
+        except SystemExit:
+            return
+    
+    @login_required
+    def do_get_snapshot_datasets(self,args:Any) -> None:
+        """List all snapshot datasets in the current sandbox"""
+        parser = argparse.ArgumentParser(prog='get_snapshot_datasets', add_help=True)
+        try:
+            args = parser.parse_args(shlex.split(args))
+            aepp_cat = catalog.Catalog(config=self.config)
+            datasets = aepp_cat.getProfileSnapshotDatasets(explicitMergePolicy=True)
+            list_ds = []
+            for key, ds in datasets.items():
+                obj = ds
+                obj['id'] = key
+                list_ds.append(obj)
+            df_datasets = pd.DataFrame(list_ds)
+            df_datasets.to_csv(f"{self.config.sandbox}_snapshot_datasets.csv",index=False)
+            console.print(f"Snapshot Datasets exported to {self.config.sandbox}_snapshot_datasets.csv", style="green")
+            table = Table(title=f"Snapshot Datasets in Sandbox: {self.config.sandbox}")
+            table.add_column("ID", style="white")
+            table.add_column("Table Name", style="white")
+            table.add_column("Merge Policy Name", style="yellow")
+            table.add_column("Merge Policy ID", style="green")
+            for ds in list_ds:
+                table.add_row(
+                    ds.get("id","N/A"),
+                    ds.get("tags",{}).get('adobe/pqs/table',["N/A"])[0],
+                    ds.get('mergePolicyName','N/A'),
+                    [el.split(':')[1] for el in ds.get('tags',{}).get('unifiedProfile',[]) if el.startswith('mergePolicyId')][0]
                 )
             console.print(table)
         except Exception as e:
@@ -846,6 +983,37 @@ class ServiceShell(cmd.Cmd):
         except SystemExit:
             return
     
+    @login_required
+    def do_get_audiences(self, args:Any) -> None:
+        """List all audiences in the current sandbox"""
+        parser = argparse.ArgumentParser(prog='get_audiences', add_help=True)
+        try:
+            args = parser.parse_args(shlex.split(args))
+            aepp_audience = segmentation.Segmentation(config=self.config)
+            audiences = aepp_audience.getAudiences()
+            df_audiences = pd.DataFrame(audiences)
+            df_audiences.to_csv(f"{self.config.sandbox}_audiences.csv",index=False)
+            console.print(f"Audiences exported to {self.config.sandbox}_audiences.csv", style="green")
+            table = Table(title=f"Audiences in Sandbox: {self.config.sandbox}")
+            table.add_column("ID", style="cyan")
+            table.add_column("Name", style="magenta")
+            table.add_column("Evaluation", style="yellow")
+            table.add_column("Total Profiles", style="green")
+            table.add_column("Evaluation Date", style="white")
+            for aud in audiences:
+                table.add_row(
+                    aud.get("id","N/A"),
+                    aud.get("name","N/A"),
+                    '[red3]Batch[/red3]' if aud.get("evaluationInfo",{}).get("batch",{}).get('enabled') else '[chartreuse1]Streaming[/chartreuse1]' if aud.get("evaluationInfo",{}).get("continuous",{}).get('enabled') else '[blue_violet]Edge[/blue_violet]' if aud.get("evaluationInfo",{}).get("synchronous",{}).get('enabled') else 'N/A',
+                    str(aud.get('metrics',{}).get('data',{}).get('totalProfiles','N/A')),
+                    datetime.fromtimestamp(aud.get('metrics',{}).get('updateEpoch',0)).isoformat(),
+                )
+            console.print(table)
+        except Exception as e:
+            console.print(f"(!) Error: {str(e)}", style="red")
+        except SystemExit:
+            return
+    
     @login_required
     def do_get_flows(self, args:Any) -> None:
         """List flows in the current sandbox based on parameters provided. By default, list all sources and destinations."""
@@ -1203,18 +1371,14 @@ class ServiceShell(cmd.Cmd):
         console.print("Syncing artifact...", style="blue")
         parser = argparse.ArgumentParser(prog='extractArtifact', description='Extract artifacts from AEP')
         parser.add_argument('artifact', help='artifact to extract (name or id): "schema","fieldgroup","datatype","descriptor","dataset","identity","mergepolicy","audience"')
-        parser.add_argument('-at','--artifactType', help='artifact type ')
-        parser.add_argument('-t','--targets', help='target sandboxes')
-        parser.add_argument('-lf','--localfolder', help='Local folder to extract artifacts to',default='extractions')
+        parser.add_argument('-at','--artifactType', help='artifact type ',type=str)
+        parser.add_argument('-t','--targets', help='target sandboxes',nargs='+',type=str)
+        parser.add_argument('-lf','--localfolder', help='Local folder to extract artifacts to',default='extractions',nargs='+',type=str)
         parser.add_argument('-b','--baseSandbox', help='Base sandbox for synchronization')
         parser.add_argument('-rg','--region', help='Region to extract artifacts from: "ndl2" (default), "va7", "aus5", "can2", "ind2"',default='ndl2')
         parser.add_argument('-v','--verbose', help='Enable verbose output',default=True)
         try:
             args = parser.parse_args(shlex.split(args))
-            if ',' in args.targets:
-                args.targets = args.targets.split(',')
-            else:
-                args.targets = [args.targets]
             console.print("Initializing Synchronizor...", style="blue")
             if args.baseSandbox:
                 synchronizor = synchronizer.Synchronizer(

{aepp-0.5.1.post1 → aepp-0.5.2.post1}/aepp/datatypemanager.py

@@ -36,7 +36,7 @@ class DataTypeManager:
                 schemaAPI:'Schema'=None,
                 config: Union[dict,ConnectObject] = aepp.config.config_object,
                 description:str="",
-                localFolder:str=None,
+                localFolder:str|list|None=None,
                 sandbox:str=None,
                 **kwargs
                 )->None:
@@ -65,10 +65,13 @@ class DataTypeManager:
         elif config is not None and localFolder is None:
             self.schemaAPI = Schema(config=config)
         elif localFolder is not None:
-            self.localfolder = Path(localFolder)
-            self.datatypeFolder = self.localfolder / 'datatype'
-            self.datatypeGlobalFolder = self.datatypeFolder / 'global'
-            if self.localfolder.exists() is False:
+            if isinstance(localFolder, str):
+                self.localfolder = [Path(localFolder)]
+            elif isinstance(localFolder, list):
+                self.localfolder = [Path(folder) for folder in localFolder]
+            self.datatypeFolder = [folder / 'datatype' for folder in self.localfolder]
+            self.datatypeGlobalFolder = [folder / 'global' for folder in self.datatypeFolder]
+            if any([folder.exists() is False for folder in self.localfolder]):
                 raise Exception(f"The local folder {self.localfolder} does not exist. Please create it and extract your sandbox before using it.")
             self.schemaAPI = None
         if self.schemaAPI is not None:
@@ -99,20 +102,32 @@ class DataTypeManager:
                     self.dataType = self.schemaAPI.getDataType(dataType['$id'],full=False)
                     self.EDITABLE = True
                 elif self.localfolder is not None:
-                    for dataTypeFile in self.datatypeFolder.glob(f"*.json"):
-                        tmp_def = json.load(FileIO(dataTypeFile))
-                        if tmp_def.get('$id') == dataType.get('$id') or tmp_def.get('meta:altId') == dataType.get('meta:altId'):
-                            self.dataType = tmp_def
+                    found = False
+                    for folder in self.datatypeFolder:
+                        for dataTypeFile in folder.glob(f"*.json"):
+                            tmp_def = json.load(FileIO(dataTypeFile))
+                            if tmp_def.get('$id') == dataType.get('$id') or tmp_def.get('meta:altId') == dataType.get('meta:altId'):
+                                self.dataType = tmp_def
+                                found = True
+                                break
+                        if found:
+                            break
                     self.EDITABLE = False
             else:
                 if self.schemaAPI is not None:
                     self.dataType = self.schemaAPI.getDataType(dataType['$id'],full=True)
                     self.EDITABLE = True
                 elif self.localfolder is not None:
-                    for dataTypeFile in self.datatypeGlobalFolder.glob("*.json"):
-                        tmp_def = json.load(FileIO(dataTypeFile))
-                        if tmp_def.get('$id') == dataType.get('$id') or tmp_def.get('meta:altId') == dataType.get('meta:altId') or tmp_def.get('title') == dataType.get('title'):   
-                            self.dataType = tmp_def
+                    found = False
+                    for folder in self.datatypeGlobalFolder:
+                        for dataTypeFile in folder.glob("*.json"):
+                            tmp_def = json.load(FileIO(dataTypeFile))
+                            if tmp_def.get('$id') == dataType.get('$id') or tmp_def.get('meta:altId') == dataType.get('meta:altId') or tmp_def.get('title') == dataType.get('title'):   
+                                self.dataType = tmp_def
+                                found = True
+                                break
+                        if found:
+                            break
                     self.EDITABLE = False
         elif type(dataType) == str:
             if self.tenantId[1:] in dataType:
@@ -122,10 +137,16 @@ class DataTypeManager:
                         raise ValueError(f"Cannot find the data type with id {dataType} in the schema API.")
                     self.EDITABLE = True
                 elif self.localfolder is not None:
-                    for dataTypeFile in self.datatypeFolder.glob("*.json"):
-                        tmp_def = json.load(FileIO(dataTypeFile))
-                        if tmp_def.get('$id') == dataType or tmp_def.get('meta:altId') == dataType or tmp_def.get('title') == dataType:
-                            self.dataType = tmp_def
+                    found = False
+                    for folder in self.datatypeFolder:
+                        for dataTypeFile in folder.glob("*.json"):
+                            tmp_def = json.load(FileIO(dataTypeFile))
+                            if tmp_def.get('$id') == dataType or tmp_def.get('meta:altId') == dataType or tmp_def.get('title') == dataType:
+                                self.dataType = tmp_def
+                                found = True
+                                break
+                        if found:
+                            break
                     self.EDITABLE = False
                 else:
                     raise Exception("You try to retrieve the datatype definition from the id, but no API or localFolder has been passed as a parameter.")
@@ -136,10 +157,16 @@ class DataTypeManager:
                         raise ValueError(f"Cannot find the data type with id {dataType} in the schema API.")
                     self.EDITABLE = True
                 elif self.localfolder is not None:
-                    for dataTypeFile in self.datatypeGlobalFolder.glob("*.json"):
-                        tmp_def = json.load(FileIO(dataTypeFile))
-                        if tmp_def.get('$id') == dataType or tmp_def.get('meta:altId') == dataType or tmp_def.get('title') == dataType:
-                            self.dataType = tmp_def
+                    found = False
+                    for folder in self.datatypeGlobalFolder:
+                        for dataTypeFile in folder.glob("*.json"):
+                            tmp_def = json.load(FileIO(dataTypeFile))
+                            if tmp_def.get('$id') == dataType or tmp_def.get('meta:altId') == dataType or tmp_def.get('title') == dataType:
+                                self.dataType = tmp_def
+                                found = True
+                                break
+                        if found:
+                            break
                     self.EDITABLE = False
                 else:
                     raise Exception("You try to retrieve the datatype definition from the id, but no API or localFolder has been passed as a parameter.")
@@ -181,12 +208,18 @@ class DataTypeManager:
                     self.dataTypeManagers[dt_manager.title] = dt_manager
             elif self.localfolder is not None:
                 for dt in dataTypes: ## today only searching custom data types in local folder
-                    for dataTypeFile in self.datatypeFolder.glob("*.json"):
-                        tmp_def = json.load(FileIO(dataTypeFile))
-                        if tmp_def.get('$id') == dt or tmp_def.get('meta:altId') == dt or tmp_def.get('title') == dt:
-                            dt_manager = DataTypeManager(dataType=tmp_def,localFolder=self.localfolder,tenantId=self.tenantId,sandbox=self.sandbox)
-                            self.dataTypes[dt_manager.id] = dt_manager.title
-                            self.dataTypeManagers[dt_manager.title] = dt_manager
+                    found = False
+                    for folder in self.datatypeFolder:
+                        for dataTypeFile in folder.glob("*.json"):
+                            tmp_def = json.load(FileIO(dataTypeFile))
+                            if tmp_def.get('$id') == dt or tmp_def.get('meta:altId') == dt or tmp_def.get('title') == dt:
+                                dt_manager = DataTypeManager(dataType=tmp_def,localFolder=self.localfolder,tenantId=self.tenantId,sandbox=self.sandbox)
+                                self.dataTypes[dt_manager.id] = dt_manager.title
+                                self.dataTypeManagers[dt_manager.title] = dt_manager
+                                found = True
+                                break
+                        if found:
+                            break
         if title is not None:
             self.dataType['title'] = title
             self.title = title