aepp 0.4.3.post1__py3-none-any.whl → 0.4.3.post3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
aepp/cli/__main__.py ADDED
@@ -0,0 +1,1242 @@
1
+ from ast import arg
2
+ from matplotlib.pyplot import table
3
+ import aepp
4
+ from aepp import synchronizer, schema, schemamanager, fieldgroupmanager, datatypemanager, identity, queryservice,catalog,flowservice
5
+ import argparse, cmd, shlex, json
6
+ from functools import wraps
7
+ from rich.console import Console
8
+ from rich.table import Table
9
+ from rich.panel import Panel
10
+ from rich.markdown import Markdown
11
+ from pathlib import Path
12
+ from io import FileIO
13
+ import pandas as pd
14
+ from datetime import datetime
15
+ import urllib.parse
16
+
17
+ # --- 1. The Decorator (The Gatekeeper) ---
18
+ def login_required(f):
19
+ """Decorator to block commands if not logged in."""
20
+ @wraps(f)
21
+ def wrapper(self, *args, **kwargs):
22
+ if not hasattr(self, 'config') or self.config is None:
23
+ print("(!) Access Denied: You must setup config first.")
24
+ return
25
+ return f(self, *args, **kwargs)
26
+ return wrapper
27
+
28
+ console = Console()
29
+
30
+ # --- 2. The Interactive Shell ---
31
+ class ServiceShell(cmd.Cmd):
32
+ def __init__(self, **kwargs):
33
+ super().__init__()
34
+ self.config = None
35
+ self.connectInstance = True
36
+ config_path = Path(kwargs.get("config_file"))
37
+ if not config_path.is_absolute():
38
+ config_path = Path.cwd() / config_path
39
+ if config_path.exists() and kwargs.get("config_file") is not None:
40
+ print("Loading configuration from file...")
41
+ print(f"Config path: {config_path}")
42
+ dict_config = json.load(FileIO(config_path))
43
+ self.sandbox = kwargs.get("sandbox",dict_config.get("sandbox-name","prod"))
44
+ self.secret = dict_config.get("secret",kwargs.get("secret"))
45
+ self.org_id = dict_config.get("org_id",kwargs.get("org_id"))
46
+ self.client_id = dict_config.get("client_id",kwargs.get("client_id"))
47
+ self.scopes = dict_config.get("scopes",kwargs.get("scopes"))
48
+ else:
49
+ self.sandbox = kwargs.get("sandbox","prod")
50
+ self.secret = kwargs.get("secret")
51
+ self.org_id = kwargs.get("org_id")
52
+ self.client_id = kwargs.get("client_id")
53
+ self.scopes = kwargs.get("scopes")
54
+ print(f"Debug: sandbox={self.sandbox}, secret={self.secret}, org_id={self.org_id}, client_id={self.client_id}, scopes={self.scopes}")
55
+ self.connectInstance = True
56
+ if self.sandbox is not None and self.secret is not None and self.org_id is not None and self.client_id is not None and self.scopes is not None:
57
+ print("Auto-configuring connection...")
58
+ self.config = aepp.configure(
59
+ connectInstance=self.connectInstance,
60
+ sandbox=self.sandbox,
61
+ secret=self.secret,
62
+ org_id=self.org_id,
63
+ client_id=self.client_id,
64
+ scopes=self.scopes
65
+ )
66
+ self.prompt = f"{self.config.sandbox}> "
67
+ console.print(Panel(f"Connected to [bold green]{self.sandbox}[/bold green]", style="blue"))
68
+
69
+ # # --- Commands ---
70
+ def do_config(self, arg):
71
+ """connect to an AEP instance"""
72
+ parser = argparse.ArgumentParser(prog='config', add_help=True)
73
+ parser.add_argument("-sx", "--sandbox", help="Auto-login sandbox")
74
+ parser.add_argument("-s", "--secret", help="Secret")
75
+ parser.add_argument("-o", "--org_id", help="Auto-login org ID")
76
+ parser.add_argument("-sc", "--scopes", help="Scopes")
77
+ parser.add_argument("-cid", "--client_id", help="Auto-login client ID")
78
+ parser.add_argument("-cf", "--config_file", help="Path to config file", default=None)
79
+ args = parser.parse_args(shlex.split(arg))
80
+ if args.config_file:
81
+ mypath = Path.cwd()
82
+ dict_config = json.load(FileIO(mypath / Path(args.config_file)))
83
+ self.sandbox = args.sandbox if args.sandbox else dict_config.get("sandbox-name",args.sandbox)
84
+ self.secret = dict_config.get("secret",args.secret)
85
+ self.org_id = dict_config.get("org_id",args.org_id)
86
+ self.client_id = dict_config.get("client_id",args.client_id)
87
+ self.scopes = dict_config.get("scopes",args.scopes)
88
+ self.connectInstance = True
89
+ else:
90
+ if args.sandbox: self.sandbox = args.sandbox
91
+ if args.secret: self.secret = args.secret
92
+ if args.org_id: self.org_id = args.org_id
93
+ if args.scopes: self.scopes = args.scopes
94
+ if args.client_id: self.client_id = args.client_id
95
+ console.print("Configuring connection...", style="blue")
96
+ self.config = aepp.configure(
97
+ connectInstance=self.connectInstance,
98
+ sandbox=self.sandbox,
99
+ secret=self.secret,
100
+ org_id=self.org_id,
101
+ client_id=self.client_id,
102
+ scopes=self.scopes
103
+ )
104
+ console.print(Panel(f"Connected to [bold green]{self.sandbox}[/bold green]", style="blue"))
105
+ self.prompt = f"{self.config.sandbox}> "
106
+ return
107
+
108
+ def do_change_sandbox(self, args):
109
+ """Change the current sandbox after configuration"""
110
+ parser = argparse.ArgumentParser(prog='change sandbox', add_help=True)
111
+ parser.add_argument("sandbox", help="sandbox name to switch to")
112
+ args = parser.parse_args(shlex.split(args))
113
+ self.sandbox = args.sandbox if args.sandbox else console.print(Panel("(!) Please provide a sandbox name using -sx or --sandbox", style="red"))
114
+ if self.config is not None:
115
+ if args.sandbox:
116
+ self.config.setSandbox(args.sandbox)
117
+ self.prompt = f"{self.config.sandbox}> "
118
+ console.print(Panel(f"Sandbox changed to: {self.config.sandbox}", style="blue"))
119
+ else:
120
+ console.print(Panel("(!) You must configure the connection first using the 'config' command.", style="red"))
121
+
122
+
123
+ @login_required
124
+ def do_get_schemas(self, args):
125
+ """List all schemas in the current sandbox"""
126
+ parser = argparse.ArgumentParser(prog='get_schemas', add_help=True)
127
+ parser.add_argument("-sv", "--save",help="Save schemas to CSV file")
128
+ try:
129
+ args = parser.parse_args(shlex.split(args))
130
+ aepp_schema = schema.Schema(config=self.config)
131
+ schemas = aepp_schema.getSchemas()
132
+ if len(schemas) > 0:
133
+ if args.save:
134
+ df_schemas = pd.DataFrame(schemas)
135
+ df_schemas.to_csv(f"{self.config.sandbox}_schemas.csv", index=False)
136
+ console.print(f"Schemas exported to {self.config.sandbox}_schemas.csv", style="green")
137
+ table = Table(title=f"Schemas in Sandbox: {self.config.sandbox}")
138
+ table.add_column("ID", style="cyan")
139
+ table.add_column("Name", style="magenta")
140
+ table.add_column("Version", style="green")
141
+ for sch in schemas:
142
+ table.add_row(
143
+ sch.get("meta:altId","N/A"),
144
+ sch.get("title","N/A"),
145
+ str(sch.get("version","N/A")),
146
+ )
147
+ console.print(table)
148
+ else:
149
+ console.print("(!) No schemas found.", style="red")
150
+ except Exception as e:
151
+ console.print(f"(!) Error: {str(e)}", style="red")
152
+ except SystemExit:
153
+ return
154
+
155
+ @login_required
156
+ def do_get_ups_schemas(self, args):
157
+ """List all schemas enabled for Profile in the current sandbox"""
158
+ parser = argparse.ArgumentParser(prog='get_schemas_enabled', add_help=True)
159
+ parser.add_argument("-sv", "--save",help="Save enabled schemas to CSV file")
160
+ try:
161
+ args = parser.parse_args(shlex.split(args))
162
+ aepp_schema = schema.Schema(config=self.config)
163
+ union_schemas = aepp_schema.getUnions()
164
+ schemas = aepp_schema.getSchemas()
165
+ enabled_schemas = []
166
+ for union in union_schemas:
167
+ for member in union.get("meta:extends",[]):
168
+ if 'schema' in member:
169
+ enabled_schemas.append(member)
170
+ list_enabled_schemas = []
171
+ list_enabled_schemas = [sc for sc in schemas if sc.get("$id") in enabled_schemas]
172
+ if len(list_enabled_schemas) > 0:
173
+ if args.save:
174
+ df_schemas = pd.DataFrame(list_enabled_schemas)
175
+ df_schemas.to_csv(f"{self.config.sandbox}_enabled_schemas.csv", index=False)
176
+ console.print(f"Enabled Schemas exported to {self.config.sandbox}_enabled_schemas.csv", style="green")
177
+ table = Table(title=f"Enabled Schemas in Sandbox: {self.config.sandbox}")
178
+ table.add_column("ID", style="cyan")
179
+ table.add_column("Name", style="magenta")
180
+ table.add_column("Version", style="green")
181
+ for sch in list_enabled_schemas:
182
+ table.add_row(
183
+ sch.get("meta:altId","N/A"),
184
+ sch.get("title","N/A"),
185
+ str(sch.get("version","N/A")),
186
+ )
187
+ console.print(table)
188
+ else:
189
+ console.print("(!) No enabled schemas found.", style="red")
190
+ except Exception as e:
191
+ console.print(f"(!) Error: {str(e)}", style="red")
192
+ except SystemExit:
193
+ return
194
+ @login_required
195
+ def do_get_ups_fieldgroups(self, args):
196
+ """List all field groups enabled for Profile in the current sandbox"""
197
+ parser = argparse.ArgumentParser(prog='get_fieldgroups_enabled', add_help=True)
198
+ parser.add_argument("-sv", "--save",help="Save enabled field groups to CSV file")
199
+ try:
200
+ args = parser.parse_args(shlex.split(args))
201
+ aepp_schema = schema.Schema(config=self.config)
202
+ union_schemas = aepp_schema.getUnions()
203
+ fgs = aepp_schema.getFieldGroups()
204
+ enabled_fgs = []
205
+ for union in union_schemas:
206
+ for member in union.get("meta:extends",[]):
207
+ if 'mixins' in member:
208
+ enabled_fgs.append(member)
209
+ list_enabled_fgs = []
210
+ list_enabled_fgs = [f for f in fgs if f.get("$id") in enabled_fgs]
211
+ if len(list_enabled_fgs) > 0:
212
+ if args.save:
213
+ df_fgs = pd.DataFrame(list_enabled_fgs)
214
+ df_fgs.to_csv(f"{self.config.sandbox}_enabled_field_groups.csv", index=False)
215
+ console.print(f"Enabled Field Groups exported to {self.config.sandbox}_enabled_field_groups.csv", style="green")
216
+ table = Table(title=f"Enabled Field Groups in Sandbox: {self.config.sandbox}")
217
+ table.add_column("ID", style="cyan")
218
+ table.add_column("Name", style="magenta")
219
+ table.add_column("Version", style="green")
220
+ for sch in list_enabled_fgs:
221
+ table.add_row(
222
+ sch.get("meta:altId","N/A"),
223
+ sch.get("title","N/A"),
224
+ str(sch.get("version","N/A")),
225
+ )
226
+ console.print(table)
227
+ else:
228
+ console.print("(!) No enabled field groups found.", style="red")
229
+ except Exception as e:
230
+ console.print(f"(!) Error: {str(e)}", style="red")
231
+ except SystemExit:
232
+ return
233
+
234
+ @login_required
235
+ def do_get_profile_schemas(self,args):
236
+ """Get the current profile schema"""
237
+ parser = argparse.ArgumentParser(prog='get_schemas_enabled', add_help=True)
238
+ try:
239
+ args = parser.parse_args(shlex.split(args))
240
+ aepp_schema = schema.Schema(config=self.config)
241
+ profile_schemas = aepp_schema.getSchemas(classFilter="https://ns.adobe.com/xdm/context/profile")
242
+ if profile_schemas:
243
+ table = Table(title=f"Profile Schemas in Sandbox: {self.config.sandbox}")
244
+ table.add_column("ID", style="cyan")
245
+ table.add_column("Name", style="magenta")
246
+ table.add_column("Version", style="green")
247
+ for sch in profile_schemas:
248
+ table.add_row(
249
+ sch.get("meta:altId","N/A"),
250
+ sch.get("title","N/A"),
251
+ str(sch.get("version","N/A")),
252
+ )
253
+ console.print(table)
254
+ else:
255
+ console.print("(!) No profile schemas found.", style="red")
256
+ except Exception as e:
257
+ console.print(f"(!) Error: {str(e)}", style="red")
258
+ except SystemExit:
259
+ return
260
+
261
+ @login_required
262
+ def do_get_union_profile_json(self,args):
263
+ """Get the current Profile union schema"""
264
+ parser = argparse.ArgumentParser(prog='get_union_profile', add_help=True)
265
+ try:
266
+ args = parser.parse_args(shlex.split(args))
267
+ profile_union = schemamanager.SchemaManager('https://ns.adobe.com/xdm/context/profile__union',config=self.config)
268
+ data = profile_union.to_dict()
269
+ with open(f"{self.config.sandbox}_profile_union_schema.json", 'w') as f:
270
+ json.dump(data, f, indent=4)
271
+ console.print(f"Profile Union Schema exported to {self.config.sandbox}_profile_union_schema.json", style="green")
272
+ except Exception as e:
273
+ console.print(f"(!) Error: {str(e)}", style="red")
274
+ except SystemExit:
275
+ return
276
+
277
+ @login_required
278
+ def do_get_union_profile_csv(self,args):
279
+ """Get the current Profile union schema"""
280
+ parser = argparse.ArgumentParser(prog='get_union_profile', add_help=True)
281
+ parser.add_argument("-f","--full",default=False,help="Get full schema information with all details",type=bool)
282
+ try:
283
+ args = parser.parse_args(shlex.split(args))
284
+ profile_union = schemamanager.SchemaManager('https://ns.adobe.com/xdm/context/profile__union',config=self.config)
285
+ df = profile_union.to_dataframe(full=args.full)
286
+ df.to_csv(f"{self.config.sandbox}_profile_union_schema.csv", index=False)
287
+ console.print(f"Profile Union Schema exported to {self.config.sandbox}_profile_union_schema.csv", style="green")
288
+ except Exception as e:
289
+ console.print(f"(!) Error: {str(e)}", style="red")
290
+ except SystemExit:
291
+ return
292
+
293
+ @login_required
294
+ def do_get_union_event_json(self,args):
295
+ """Get the current Experience Event union schema"""
296
+ parser = argparse.ArgumentParser(prog='get_union_event', add_help=True)
297
+ try:
298
+ args = parser.parse_args(shlex.split(args))
299
+ event_union = schemamanager.SchemaManager('https://ns.adobe.com/xdm/context/experienceevent__union',config=self.config)
300
+ data = event_union.to_dict()
301
+ with open(f"{self.config.sandbox}_event_union_schema.json", 'w') as f:
302
+ json.dump(data, f, indent=4)
303
+ console.print(f"Event Union Schema exported to {self.config.sandbox}_event_union_schema.json", style="green")
304
+ except Exception as e:
305
+ console.print(f"(!) Error: {str(e)}", style="red")
306
+ except SystemExit:
307
+ return
308
+
309
+ @login_required
310
+ def do_get_union_event_csv(self,args):
311
+ """Get the current Experience Event union schema"""
312
+ parser = argparse.ArgumentParser(prog='get_union_event', add_help=True)
313
+ parser.add_argument("-f","--full",default=False,help="Get full schema information with all details",type=bool)
314
+ try:
315
+ args = parser.parse_args(shlex.split(args))
316
+ event_union = schemamanager.SchemaManager('https://ns.adobe.com/xdm/context/experienceevent__union',config=self.config)
317
+ df = event_union.to_dataframe(full=args.full)
318
+ df.to_csv(f"{self.config.sandbox}_event_union_schema.csv", index=False)
319
+ console.print(f"Event Union Schema exported to {self.config.sandbox}_event_union_schema.csv", style="green")
320
+ except Exception as e:
321
+ console.print(f"(!) Error: {str(e)}", style="red")
322
+ except SystemExit:
323
+ return
324
+
325
+ @login_required
326
+ def do_get_event_schemas(self,args):
327
+ """Get the current Experience Event schemas"""
328
+ parser = argparse.ArgumentParser(prog='get_event_schemas', add_help=True)
329
+ parser.add_argument("-sv", "--save",help="Save event schemas to CSV file")
330
+ try:
331
+ args = parser.parse_args(shlex.split(args))
332
+ aepp_schema = schema.Schema(config=self.config)
333
+ event_schemas = aepp_schema.getSchemas(classFilter="https://ns.adobe.com/xdm/context/experienceevent")
334
+ if args.save:
335
+ df_schemas = pd.DataFrame(event_schemas)
336
+ df_schemas.to_csv(f"{self.config.sandbox}_event_schemas.csv", index=False)
337
+ console.print(f"Event Schemas exported to {self.config.sandbox}_event_schemas.csv", style="green")
338
+ if event_schemas:
339
+ table = Table(title=f"Event Schemas in Sandbox: {self.config.sandbox}")
340
+ table.add_column("ID", style="cyan")
341
+ table.add_column("Name", style="magenta")
342
+ table.add_column("Version", style="green")
343
+ for sch in event_schemas:
344
+ table.add_row(
345
+ sch.get("meta:altId","N/A"),
346
+ sch.get("title","N/A"),
347
+ str(sch.get("version","N/A")),
348
+ )
349
+ console.print(table)
350
+ else:
351
+ console.print("(!) No event schemas found.", style="red")
352
+ except Exception as e:
353
+ console.print(f"(!) Error: {str(e)}", style="red")
354
+ except SystemExit:
355
+ return
356
+
357
+ @login_required
358
+ def do_get_union_event_json(self,args):
359
+ """Get the current Experience Event union schema"""
360
+ parser = argparse.ArgumentParser(prog='get_union_event', add_help=True)
361
+ try:
362
+ args = parser.parse_args(shlex.split(args))
363
+ event_union = schemamanager.SchemaManager('https://ns.adobe.com/xdm/context/experienceevent__union',config=self.config)
364
+ data = event_union.to_dict()
365
+ with open(f"{self.config.sandbox}_event_union_schema.json", 'w') as f:
366
+ json.dump(data, f, indent=4)
367
+ console.print(f"Event Union Schema exported to {self.config.sandbox}_event_union_schema.json", style="green")
368
+ except Exception as e:
369
+ console.print(f"(!) Error: {str(e)}", style="red")
370
+ except SystemExit:
371
+ return
372
+
373
+
374
+ @login_required
375
+ def do_get_schema_xdm(self, arg):
376
+ """Get schema JSON by name or ID"""
377
+ parser = argparse.ArgumentParser(prog='get_schema_xdm', add_help=True)
378
+ parser.add_argument("schema", help="Schema title, $id or alt:Id to retrieve")
379
+ parser.add_argument("-f","--full",default=False,help="Get full schema with all details",type=bool)
380
+ try:
381
+ args = parser.parse_args(shlex.split(arg))
382
+ aepp_schema = schema.Schema(config=self.config)
383
+ schemas = aepp_schema.getSchemas()
384
+ print(args.schema)
385
+ ## chech if schema title is found
386
+ if args.schema in [sch for sch in aepp_schema.data.schemas_altId.keys()]:
387
+ schema_json = aepp_schema.getSchema(
388
+ schemaId=aepp_schema.data.schemas_altId[args.schema],
389
+ )
390
+ else:
391
+
392
+ schema_json = aepp_schema.getSchema(
393
+ schemaId=args.schema
394
+ )
395
+ if 'title' in schema_json.keys():
396
+ filename = f"{schema_json['title']}_xdm.json"
397
+ with open(filename, 'w') as f:
398
+ json.dump(schema_json, f, indent=4)
399
+ console.print(f"Schema '{args.schema}' saved to {filename}.", style="green")
400
+ else:
401
+ console.print(f"(!) Schema '{args.schema}' not found.", style="red")
402
+ except Exception as e:
403
+ console.print(f"(!) Error: {str(e)}", style="red")
404
+ except SystemExit:
405
+ return
406
+
407
+ @login_required
408
+ def do_get_schema_csv(self, arg):
409
+ """Get schema CSV by name or ID"""
410
+ parser = argparse.ArgumentParser(prog='get_schema_csv', add_help=True)
411
+ parser.add_argument("schema", help="Schema $id or alt:Id to retrieve")
412
+ parser.add_argument("-f","--full",default=False,help="Get full schema information with all details",type=bool)
413
+ try:
414
+ args = parser.parse_args(shlex.split(arg))
415
+ aepp_schema = schema.Schema(config=self.config)
416
+ schemas = aepp_schema.getSchemas()
417
+ ## chech if schema title is found
418
+ if args.schema in [sch for sch in aepp_schema.data.schemas_altId.keys()]:
419
+ my_schema_manager = schemamanager.SchemaManager(
420
+ schema=aepp_schema.data.schemas_altId[args.schema],
421
+ config=self.config
422
+ )
423
+ df = my_schema_manager.to_dataframe(full=args.full)
424
+ else:
425
+ my_schema_manager = schemamanager.SchemaManager(
426
+ schema=args.schema,
427
+ config=self.config
428
+ )
429
+ df = my_schema_manager.to_dataframe(full=args.full)
430
+ df.to_csv(f"{my_schema_manager.title}_schema.csv", index=False)
431
+ console.print(f"Schema exported to {my_schema_manager.title}_schema.csv", style="green")
432
+ except Exception as e:
433
+ console.print(f"(!) Error: {str(e)}", style="red")
434
+ except SystemExit:
435
+ return
436
+
437
+ @login_required
438
+ def do_get_schema_json(self, args):
439
+ """Get schema JSON by name or ID"""
440
+ parser = argparse.ArgumentParser(prog='get_schema_json', add_help=True)
441
+ parser.add_argument("schema", help="Schema $id or alt:Id to retrieve")
442
+ try:
443
+ args = parser.parse_args(shlex.split(args))
444
+ aepp_schema = schema.Schema(config=self.config)
445
+ schemas = aepp_schema.getSchemas()
446
+ ## chech if schema title is found
447
+ if args.schema in [sch for sch in aepp_schema.data.schemas_altId.keys()]:
448
+ my_schema_manager = schemamanager.SchemaManager(
449
+ schema=aepp_schema.data.schemas_altId[args.schema],
450
+ config=self.config
451
+ )
452
+ else:
453
+ my_schema_manager = schemamanager.SchemaManager(
454
+ schema=args.schema,
455
+ config=self.config
456
+ )
457
+ data = my_schema_manager.to_dict()
458
+ with open(f"{my_schema_manager.title}.json", 'w') as f:
459
+ json.dump(data, f, indent=4)
460
+ console.print(f"Schema exported to {my_schema_manager.title}.json", style="green")
461
+ except Exception as e:
462
+ console.print(f"(!) Error: {str(e)}", style="red")
463
+ except SystemExit:
464
+ return
465
+
466
+ @login_required
467
+ def do_get_fieldgroups(self, args):
468
+ """List all field groups in the current sandbox"""
469
+ parser = argparse.ArgumentParser(prog='get_fieldgroups', add_help=True)
470
+ parser.add_argument("-sv", "--save",help="Save field groups to CSV file")
471
+ try:
472
+ args = parser.parse_args(shlex.split(args))
473
+ aepp_schema = schema.Schema(config=self.config)
474
+ fieldgroups = aepp_schema.getFieldGroups()
475
+ if args.save:
476
+ df_fgs = pd.DataFrame(fieldgroups)
477
+ df_fgs.to_csv(f"{self.config.sandbox}_fieldgroups.csv",index=False)
478
+ console.print(f"Field Groups exported to {self.config.sandbox}_fieldgroups.csv", style="green")
479
+ if fieldgroups:
480
+ table = Table(title=f"Field Groups in Sandbox: {self.config.sandbox}")
481
+ table.add_column("altId", style="cyan")
482
+ table.add_column("Title", style="magenta")
483
+ for fg in fieldgroups:
484
+ table.add_row(
485
+ fg.get("meta:altId","N/A"),
486
+ fg.get("title","N/A"),
487
+ )
488
+ console.print(table)
489
+ else:
490
+ console.print("(!) No field groups found.", style="red")
491
+ except Exception as e:
492
+ console.print(f"(!) Error: {str(e)}", style="red")
493
+ except SystemExit:
494
+ return
495
+
496
+ @login_required
497
+ def do_get_fieldgroup_json(self, args):
498
+ """Get field group JSON by name or ID"""
499
+ parser = argparse.ArgumentParser(prog='get_fieldgroup_json', add_help=True)
500
+ parser.add_argument("fieldgroup", help="Field Group name, $id or alt:Id to retrieve")
501
+ try:
502
+ args = parser.parse_args(shlex.split(args))
503
+ aepp_schema = schema.Schema(config=self.config)
504
+ fieldgroups = aepp_schema.getFieldGroups()
505
+ ## chech if schema title is found
506
+ if args.fieldgroup in [fg for fg in aepp_schema.data.fieldGroups_altId.keys()]:
507
+ my_fieldgroup_manager = fieldgroupmanager.FieldGroupManager(
508
+ fieldgroup=aepp_schema.data.fieldGroups_altId[args.fieldgroup],
509
+ config=self.config
510
+ )
511
+ else:
512
+ my_fieldgroup_manager = fieldgroupmanager.FieldGroupManager(
513
+ fieldgroup=args.fieldgroup,
514
+ config=self.config
515
+ )
516
+ data = my_fieldgroup_manager.to_dict()
517
+ with open(f"{my_fieldgroup_manager.title}_fieldgroup.json", 'w') as f:
518
+ json.dump(data, f, indent=4)
519
+ console.print(f"Field Group exported to {my_fieldgroup_manager.title}_fieldgroup.json", style="green")
520
+ except Exception as e:
521
+ console.print(f"(!) Error: {str(e)}", style="red")
522
+ except SystemExit:
523
+ return
524
+
525
+ @login_required
526
+ def do_get_fieldgroup_csv(self, args):
527
+ """Get field group CSV by name or ID"""
528
+ parser = argparse.ArgumentParser(prog='get_fieldgroup_csv', add_help=True)
529
+ parser.add_argument("fieldgroup", help="Field Group name, $id or alt:Id to retrieve")
530
+ parser.add_argument("-f","--full",default=False,help="Get full field group information with all details",type=bool)
531
+ try:
532
+ args = parser.parse_args(shlex.split(args))
533
+ aepp_schema = schema.Schema(config=self.config)
534
+ fieldgroups = aepp_schema.getFieldGroups()
535
+ ## chech if schema title is found
536
+ if args.fieldgroup in [fg for fg in aepp_schema.data.fieldGroups_altId.keys()]:
537
+ my_fieldgroup_manager = fieldgroupmanager.FieldGroupManager(
538
+ fieldgroup=aepp_schema.data.fieldGroups_altId[args.fieldgroup],
539
+ config=self.config
540
+ )
541
+ else:
542
+ my_fieldgroup_manager = fieldgroupmanager.FieldGroupManager(
543
+ fieldgroup=args.fieldgroup,
544
+ config=self.config
545
+ )
546
+ df = my_fieldgroup_manager.to_dataframe(full=args.full)
547
+ df.to_csv(f"{my_fieldgroup_manager.title}_fieldgroup.csv", index=False)
548
+ console.print(f"Field Group exported to {my_fieldgroup_manager.title}_fieldgroup.csv", style="green")
549
+ except Exception as e:
550
+ console.print(f"(!) Error: {str(e)}", style="red")
551
+ except SystemExit:
552
+ return
553
+
554
+ def do_get_datatypes(self, args):
555
+ """List all data types in the current sandbox"""
556
+ parser = argparse.ArgumentParser(prog='get_datatypes', add_help=True)
557
+ try:
558
+ args = parser.parse_args(shlex.split(args))
559
+ aepp_schema = schema.Schema(config=self.config)
560
+ datatypes = aepp_schema.getDataTypes()
561
+ if datatypes:
562
+ table = Table(title=f"Data Types in Sandbox: {self.config.sandbox}")
563
+ table.add_column("altId", style="cyan")
564
+ table.add_column("Title", style="magenta")
565
+ for dt in datatypes:
566
+ table.add_row(
567
+ dt.get("meta:altId","N/A"),
568
+ dt.get("title","N/A"),
569
+ )
570
+ console.print(table)
571
+ else:
572
+ console.print("(!) No data types found.", style="red")
573
+ except Exception as e:
574
+ console.print(f"(!) Error: {str(e)}", style="red")
575
+ except SystemExit:
576
+ return
577
+
578
+ @login_required
579
+ def do_get_datatype_csv(self, args):
580
+ """Get data type CSV by name or ID"""
581
+ parser = argparse.ArgumentParser(prog='get_datatype_csv', add_help=True)
582
+ parser.add_argument("datatype", help="Data Type name, $id or alt:Id to retrieve")
583
+ parser.add_argument("-f","--full",default=False,help="Get full data type information with all details",type=bool)
584
+ try:
585
+ args = parser.parse_args(shlex.split(args))
586
+ aepp_schema = schema.Schema(config=self.config)
587
+ datatypes = aepp_schema.getDataTypes()
588
+ ## chech if schema title is found
589
+ if args.datatype in [dt for dt in aepp_schema.data.dataTypes_altId.keys()]:
590
+ my_datatype_manager = datatypemanager.DataTypeManager(
591
+ datatype=aepp_schema.data.dataTypes_altId[args.datatype],
592
+ config=self.config
593
+ )
594
+ else:
595
+ my_datatype_manager = datatypemanager.DataTypeManager(
596
+ datatype=args.datatype,
597
+ config=self.config
598
+ )
599
+ df = my_datatype_manager.to_dataframe(full=args.full)
600
+ df.to_csv(f"{my_datatype_manager.title}_datatype.csv", index=False)
601
+ console.print(f"Data Type exported to {my_datatype_manager.title}_datatype.csv", style="green")
602
+ except Exception as e:
603
+ console.print(f"(!) Error: {str(e)}", style="red")
604
+ except SystemExit:
605
+ return
606
+
607
+ @login_required
608
+ def do_get_datatype_json(self, args):
609
+ """Get data type JSON by name or ID"""
610
+ parser = argparse.ArgumentParser(prog='get_datatype_json', add_help=True)
611
+ parser.add_argument("datatype", help="Data Type name, $id or alt:Id to retrieve")
612
+ parser.add_argument("-f","--full",default=False,help="Get full data type information with all details",type=bool)
613
+ try:
614
+ args = parser.parse_args(shlex.split(args))
615
+ aepp_schema = schema.Schema(config=self.config)
616
+ datatypes = aepp_schema.getDataTypes()
617
+ ## chech if schema title is found
618
+ if args.datatype in [dt for dt in aepp_schema.data.dataTypes_altId.keys()]:
619
+ my_datatype_manager = datatypemanager.DataTypeManager(
620
+ datatype=aepp_schema.data.dataTypes_altId[args.datatype],
621
+ config=self.config
622
+ )
623
+ else:
624
+ my_datatype_manager = datatypemanager.DataTypeManager(
625
+ datatype=args.datatype,
626
+ config=self.config
627
+ )
628
+ data = my_datatype_manager.to_dict()
629
+ with open(f"{my_datatype_manager.title}_datatype.json", 'w') as f:
630
+ json.dump(data, f, indent=4)
631
+ console.print(f"Data Type exported to {my_datatype_manager.title}_datatype.json", style="green")
632
+ except Exception as e:
633
+ console.print(f"(!) Error: {str(e)}", style="red")
634
+ except SystemExit:
635
+ return
636
+
637
+ @login_required
638
+ def do_enable_schema_for_ups(self, args):
639
+ """Enable a schema for Profile"""
640
+ parser = argparse.ArgumentParser(prog='enable_schema_for_ups', add_help=True)
641
+ parser.add_argument("schema_id", help="Schema ID to enable for Profile")
642
+ try:
643
+ args = parser.parse_args(shlex.split(args))
644
+ aepp_schema = schema.Schema(config=self.config)
645
+ result = aepp_schema.enableSchemaForUPS(schemaId=args.schema_id)
646
+ console.print(f"Schema '{args.schema_id}' enabled for Profile.", style="green")
647
+ except Exception as e:
648
+ console.print(f"(!) Error: {str(e)}", style="red")
649
+ except SystemExit:
650
+ return
651
+
652
+ @login_required
653
+ def do_upload_fieldgroup_definition_csv(self,args):
654
+ """Upload a field group definition from a CSV file"""
655
+ parser = argparse.ArgumentParser(prog='upload_fieldgroup_definition_csv', add_help=True)
656
+ parser.add_argument("csv_path", help="Path to the field group CSV file")
657
+ parser.add_argument("-ts","--test",help="Test upload without uploading it to AEP",default=False,type=bool)
658
+ try:
659
+ args = parser.parse_args(shlex.split(args))
660
+ myfg = fieldgroupmanager.FieldGroupManager(config=self.config)
661
+ myfg.importFieldGroupDefinition(fieldgroup=args.csv_path)
662
+ if args.test:
663
+ data = myfg.to_dict()
664
+ with open(f"test_{myfg.title}_fieldgroup.json", 'w') as f:
665
+ json.dump(data, f, indent=4)
666
+ console.print(f"Field Group definition test exported to test_{myfg.title}_fieldgroup.json", style="green")
667
+ console.print_json(data=data)
668
+ return
669
+ res = myfg.createFieldGroup()
670
+ console.print(f"Field Group uploaded with ID: {res.get('meta:altId')}", style="green")
671
+ except Exception as e:
672
+ console.print(f"(!) Error: {str(e)}", style="red")
673
+ except SystemExit:
674
+ return
675
+
676
+ @login_required
677
+ def do_upload_fieldgroup_definition_xdm(self,args):
678
+ """Upload a field group definition from a JSON XDM file"""
679
+ parser = argparse.ArgumentParser(prog='upload_fieldgroup_definition_xdm', add_help=True)
680
+ parser.add_argument("xdm_path", help="Path to the field group JSON XDM file")
681
+ parser.add_argument("-ts","--test",help="Test upload without uploading it to AEP",default=False,type=bool)
682
+ try:
683
+ args = parser.parse_args(shlex.split(args))
684
+ with open(args.xdm_path, 'r') as f:
685
+ xdm_data = json.load(f)
686
+ myfg = fieldgroupmanager.FieldGroupManager(xdm_data,config=self.config)
687
+ if args.test:
688
+ data = myfg.to_dict()
689
+ with open(f"test_{myfg.title}_fieldgroup.json", 'w') as f:
690
+ json.dump(data, f, indent=4)
691
+ console.print(f"Field Group definition test exported to test_{myfg.title}_fieldgroup.json", style="green")
692
+ console.print_json(data=data)
693
+ return
694
+ res = myfg.createFieldGroup()
695
+ console.print(f"Field Group uploaded with ID: {res.get('meta:altId')}", style="green")
696
+ except Exception as e:
697
+ console.print(f"(!) Error: {str(e)}", style="red")
698
+ except SystemExit:
699
+ return
700
+
701
+ @login_required
702
+ def do_get_datasets(self, args):
703
+ """List all datasets in the current sandbox"""
704
+ parser = argparse.ArgumentParser(prog='get_datasets', add_help=True)
705
+ try:
706
+ args = parser.parse_args(shlex.split(args))
707
+ aepp_cat = catalog.Catalog(config=self.config)
708
+ datasets = aepp_cat.getDataSets(output='list')
709
+ df_datasets = pd.DataFrame(datasets)
710
+ df_datasets.to_csv(f"{self.config.sandbox}_datasets.csv",index=False)
711
+ console.print(f"Datasets exported to {self.config.sandbox}_datasets.csv", style="green")
712
+ table = Table(title=f"Datasets in Sandbox: {self.config.sandbox}")
713
+ table.add_column("ID", style="white")
714
+ table.add_column("Name", style="white",no_wrap=True)
715
+ table.add_column("Created At", style="yellow")
716
+ table.add_column("Data Ingested", style="magenta")
717
+ table.add_column("Data Type", style="red")
718
+ for ds in datasets:
719
+ table.add_row(
720
+ ds.get("id","N/A"),
721
+ ds.get("name","N/A"),
722
+ datetime.fromtimestamp(ds.get("created",1000)/1000).isoformat().split('T')[0],
723
+ str(ds.get("dataIngested",False)),
724
+ ds.get("classification",{}).get("dataBehavior","unknown")
725
+ )
726
+ console.print(table)
727
+ except Exception as e:
728
+ console.print(f"(!) Error: {str(e)}", style="red")
729
+ except SystemExit:
730
+ return
731
+
732
+ @login_required
733
+ def do_get_datasets_infos(self, args):
734
+ """List all datasets in the current sandbox"""
735
+ parser = argparse.ArgumentParser(prog='get_datasets_infos', add_help=True)
736
+ try:
737
+ args = parser.parse_args(shlex.split(args))
738
+ aepp_cat = catalog.Catalog(config=self.config)
739
+ datasets = aepp_cat.getDataSets()
740
+ aepp_cat.data.infos.to_csv(f"{aepp_cat.sandbox}_datasets_infos.csv",index=False)
741
+ console.print(f"Datasets infos exported to {aepp_cat.sandbox}_datasets_infos.csv", style="green")
742
+ table = Table(title=f"Datasets in Sandbox: {self.config.sandbox}")
743
+ table.add_column("ID", style="white")
744
+ table.add_column("Name", style="white",no_wrap=True)
745
+ table.add_column("Datalake_rows", style="blue")
746
+ table.add_column("Datalake_storage", style="blue")
747
+ table.add_column("UPS_rows", style="magenta")
748
+ table.add_column("UPS_storage", style="magenta")
749
+ for _, ds in aepp_cat.data.infos.iterrows():
750
+ table.add_row(
751
+ ds.get("id","N/A"),
752
+ ds.get("name","N/A"),
753
+ str(ds.get("datalake_rows","N/A")),
754
+ str(ds.get("datalake_storageSize","N/A")),
755
+ str(ds.get("ups_rows","N/A")),
756
+ str(ds.get("ups_storageSize","N/A"))
757
+ )
758
+ console.print(table)
759
+ except Exception as e:
760
+ console.print(f"(!) Error: {str(e)}", style="red")
761
+ except SystemExit:
762
+ return
763
+
764
+ @login_required
765
+ def do_createDataset(self, args):
766
+ """Create a new dataset in the current sandbox"""
767
+ parser = argparse.ArgumentParser(prog='createDataset', add_help=True)
768
+ parser.add_argument("dataset_name", help="Name of the dataset to create")
769
+ parser.add_argument("schema_id", help="Schema ID to associate with the dataset")
770
+ try:
771
+ args = parser.parse_args(shlex.split(args))
772
+ aepp_cat = catalog.Catalog(config=self.config,region=args.region)
773
+ dataset_id = aepp_cat.createDataSet(dataset_name=args.dataset_name,schemaId=args.schema_id)
774
+ console.print(f"Dataset '{args.dataset_name}' created with ID: {dataset_id[0]}", style="green")
775
+ except Exception as e:
776
+ console.print(f"(!) Error: {str(e)}", style="red")
777
+ except SystemExit:
778
+ return
779
+
780
+ @login_required
781
+ def do_enable_dataset_for_ups(self, args):
782
+ """Enable a dataset for Profile"""
783
+ parser = argparse.ArgumentParser(prog='enable_dataset_for_ups', add_help=True)
784
+ parser.add_argument("dataset", help="Dataset ID or Dataset Name to enable for Profile")
785
+ try:
786
+ args = parser.parse_args(shlex.split(args))
787
+ aepp_cat = catalog.Catalog(config=self.config)
788
+ datasets = aepp_cat.getDataSets(output='list')
789
+ for ds in datasets:
790
+ if ds.get("name","") == args.dataset or ds.get("id","") == args.dataset:
791
+ datasetId = ds.get("id")
792
+ result = aepp_cat.enableDatasetProfile(datasetId=datasetId)
793
+ console.print(f"Dataset '{datasetId}' enabled for Profile.", style="green")
794
+ except Exception as e:
795
+ console.print(f"(!) Error: {str(e)}", style="red")
796
+ except SystemExit:
797
+ return
798
+
799
+ @login_required
800
+ def do_get_identities(self, args):
801
+ """List all identities in the current sandbox"""
802
+ parser = argparse.ArgumentParser(prog='get_identities', add_help=True)
803
+ parser.add_argument("-r","--region", help="Region to get identities from: 'ndl2' (default), 'va7', 'aus5', 'can2', 'ind2'", default='ndl2')
804
+ parser.add_argument("-co","--custom_only",help="Get only custom identities", default=False,type=bool)
805
+ try:
806
+ args = parser.parse_args(shlex.split(args))
807
+ aepp_identity = identity.Identity(config=self.config,region=args.region)
808
+ identities = aepp_identity.getIdentities(only_custom=args.custom_only)
809
+ df_identites = pd.DataFrame(identities)
810
+ df_identites.to_csv(f"{self.config.sandbox}_identities.csv",index=False)
811
+ console.print(f"Identities exported to {self.config.sandbox}_identities.csv", style="green")
812
+ table = Table(title=f"Identities in Sandbox: {self.config.sandbox}")
813
+ table.add_column("Code", style="cyan")
814
+ table.add_column("Name", style="magenta")
815
+ table.add_column("id", style="white")
816
+ table.add_column("namespaceType", style="green")
817
+ for _, iden in df_identites.iterrows():
818
+ table.add_row(
819
+ iden.get("code","N/A"),
820
+ iden.get("name","N/A"),
821
+ str(iden.get("id","N/A")),
822
+ iden.get("namespaceType","N/A"),
823
+ )
824
+ console.print(table)
825
+ except Exception as e:
826
+ console.print(f"(!) Error: {str(e)}", style="red")
827
+ except SystemExit:
828
+ return
829
+
830
+ @login_required
831
+ def do_get_flows(self, args):
832
+ """List flows in the current sandbox based on parameters provided. By default, list all sources and destinations."""
833
+ parser = argparse.ArgumentParser(prog='get_flows', add_help=True)
834
+ parser.add_argument("-i","--internal_flows",help="Get internal flows", default=False,type=bool)
835
+ parser.add_argument("-adv","--advanced",help="Get advanced information about runs", default=False,type=bool)
836
+ parser.add_argument("-ao","--active_only",help="Get only active flows during that time period", default=True,type=bool)
837
+ parser.add_argument("-mn","--minutes", help="Timeframe in minutes to check for errors, default 0", default=0,type=int)
838
+ parser.add_argument("-H","--hours", help="Timeframe in hours to check for errors, default 0", default=0,type=int)
839
+ parser.add_argument("-d","--days", help="Timeframe in days to check for errors, default 0", default=0,type=int)
840
+ try:
841
+ args = parser.parse_args(shlex.split(args))
842
+ timetotal_minutes = args.minutes + (args.hours * 60) + (args.days * 1440)
843
+ if timetotal_minutes == 0:
844
+ timetotal_minutes = 1440 # default to last 24 hours
845
+ timereference = int(datetime.now().timestamp()*1000) - (timetotal_minutes * 60 * 1000)
846
+ aepp_flow = flowservice.FlowService(config=self.config)
847
+ flows = aepp_flow.getFlows(n_results="inf")
848
+ runs = None
849
+ if args.active_only:
850
+ runs = aepp_flow.getRuns(n_results="inf",prop=[f'metrics.durationSummary.startedAtUTC>{timereference}'])
851
+ active_flow_ids = list(set([run.get("flowId") for run in runs]))
852
+ source_flows = aepp_flow.getFlows(onlySources=True)
853
+ destinations_flows = aepp_flow.getFlows(onlyDestinations=True)
854
+ list_source_ids = [f.get("id") for f in source_flows]
855
+ list_destination_ids = [f.get("id") for f in destinations_flows]
856
+ if args.internal_flows:
857
+ list_flows = flows
858
+ else:
859
+ list_flows = source_flows + destinations_flows
860
+ if args.active_only:
861
+ list_flows = [fl for fl in list_flows if fl.get("id") in active_flow_ids]
862
+ if args.advanced:
863
+ if runs is None:
864
+ runs = aepp_flow.getRuns(n_results="inf",prop=[f'metrics.durationSummary.startedAtUTC>{timereference}'])
865
+ runs_by_flow = {}
866
+ for run in runs:
867
+ flow_id = run.get("flowId")
868
+ if flow_id not in runs_by_flow:
869
+ runs_by_flow[flow_id] = {
870
+ "total_runs": 0,
871
+ "failed_runs": 0,
872
+ "success_runs": 0,
873
+ }
874
+ runs_by_flow[flow_id]["total_runs"] += 1
875
+ status = run.get("metrics",{}).get("statusSummary",{}).get("status","unknown")
876
+ if status == "failed":
877
+ runs_by_flow[flow_id]["failed_runs"] += 1
878
+ elif status == "success":
879
+ runs_by_flow[flow_id]["success_runs"] += 1
880
+ report_flows = []
881
+ for fl in list_flows:
882
+ obj = {
883
+ "id": fl.get("id","N/A"),
884
+ "name": fl.get("name","N/A"),
885
+ "created": fl.get("createdAt",1000),
886
+ "flowSpec": fl.get("flowSpec",{}).get('id','N/A'),
887
+ "sourceConnectionId": fl.get("sourceConnectionIds",["N/A"])[0],
888
+ "targetConnectionId": fl.get("targetConnectionIds",["N/A"])[0],
889
+ "connectionSpec": fl.get("inheritedAttributes",{}).get('sourceConnections',[{}])[0].get('connectionSpec',{}).get('id'),
890
+ "type": fl.get("inheritedAttributes",{}).get('properties','N/A'),
891
+ }
892
+ if obj.get("id") in list_source_ids:
893
+ obj["type"] = "Source"
894
+ elif obj.get("id") in list_destination_ids:
895
+ obj["type"] = "Destination"
896
+ else:
897
+ obj["type"] = "Internal"
898
+ if fl.get('transformations') and len(fl.get('transformations')) > 0:
899
+ obj["Transformation"] = True
900
+ else:
901
+ obj["Transformation"] = False
902
+ if args.advanced:
903
+ run_info = runs_by_flow.get(obj.get("id"),{"total_runs":0,"failed_runs":0,"success_runs":0})
904
+ obj["Total Runs"] = run_info.get("total_runs",0)
905
+ obj["Failed Runs"] = run_info.get("failed_runs",0)
906
+ obj["Successful Runs"] = run_info.get("success_runs",0)
907
+ report_flows.append(obj)
908
+ df_flows = pd.DataFrame(list_flows)
909
+ filename = f"{self.config.sandbox}_flows_{timereference/1000}"
910
+ if args.advanced:
911
+ filename = f"{filename}_advanced"
912
+ if args.active_only == False:
913
+ filename = f"{filename}_all"
914
+ if args.internal_flows:
915
+ filename = f"{filename}_internal"
916
+ df_flows.to_csv(f"{filename}.csv",index=False)
917
+ console.print(f"Flows exported to {filename}.csv", style="green")
918
+ table = Table(title=f"Flows in Sandbox: {self.config.sandbox}")
919
+ table.add_column("ID", style="cyan")
920
+ table.add_column("Name", style="magenta")
921
+ table.add_column("Created", style="white")
922
+ table.add_column("Type", style="white")
923
+ table.add_column("Transformation", style="white")
924
+ if args.advanced == False:
925
+ table.add_column("Flow Spec", style="white")
926
+ table.add_column("Source Conn ID", style="white")
927
+ table.add_column("Target Conn ID", style="white")
928
+ if args.advanced:
929
+ table.add_column("Total Runs", style="blue")
930
+ table.add_column("Failed Runs", style="red")
931
+ table.add_column("Successful Runs", style="green")
932
+ table.add_column("Success Rate", style="green")
933
+ table.add_column("Failure Rate", style="red")
934
+ for fl in report_flows:
935
+ row_data = []
936
+ if args.advanced:
937
+ if fl.get("Failed Runs",0) > 0:
938
+ colorStart = "[red]"
939
+ colorEnd = "[/red]"
940
+ else:
941
+ colorStart = "[green]"
942
+ colorEnd = "[/green]"
943
+ else:
944
+ colorStart = ""
945
+ colorEnd = ""
946
+ row_data = [
947
+ f"{colorStart}{fl.get('id','N/A')}{colorEnd}",
948
+ f"{colorStart}{fl.get('name','N/A')}{colorEnd}",
949
+ f"{colorStart}{datetime.fromtimestamp(fl.get('created',1000)/1000).isoformat().split('T')[0]}{colorEnd}",
950
+ f"{colorStart}{fl.get('type','N/A')}{colorEnd}",
951
+ f"{colorStart}{str(fl.get('Transformation', False))}{colorEnd}",
952
+ ]
953
+ if args.advanced == False:
954
+ row_data.extend([
955
+ f"{colorStart}{fl.get('flowSpec','N/A')}{colorEnd}",
956
+ f"{colorStart}{fl.get('sourceConnectionId','N/A')}{colorEnd}",
957
+ f"{colorStart}{fl.get('targetConnectionId','N/A')}{colorEnd}",
958
+ ])
959
+ if args.advanced:
960
+ total_runs = fl.get("Total Runs", 0)
961
+ failed_runs = fl.get("Failed Runs", 0)
962
+ successful_runs = fl.get("Successful Runs", 0)
963
+ success_rate = (successful_runs / total_runs * 100) if total_runs > 0 else 0
964
+ failure_rate = (failed_runs / total_runs * 100) if total_runs > 0 else 0
965
+ row_data.extend([
966
+ f"{colorStart}{str(total_runs)}{colorEnd}",
967
+ f"{colorStart}{str(failed_runs)}{colorEnd}",
968
+ f"{colorStart}{str(successful_runs)}{colorEnd}",
969
+ f"{colorStart}{success_rate:.0f}%{colorEnd}",
970
+ f"{colorStart}{failure_rate:.0f}%{colorEnd}"
971
+ ])
972
+ table.add_row(*row_data)
973
+ console.print(table)
974
+ except Exception as e:
975
+ console.print(f"(!) Error: {str(e)}", style="red")
976
+ except SystemExit:
977
+ return
978
+
979
+ @login_required
980
+ def do_get_flow_errors(self,args):
981
+ """Get errors for a specific flow, saving it in a JSON file for specific timeframe, default last 24 hours."""
982
+ parser = argparse.ArgumentParser(prog='get_flow_errors', add_help=True)
983
+ parser.add_argument("flow_id", help="Flow ID to get errors for")
984
+ parser.add_argument("-mn","--minutes", help="Timeframe in minutes to check for errors, default 0", default=0,type=int)
985
+ parser.add_argument("-H","--hours", help="Timeframe in hours to check for errors, default 0", default=0,type=int)
986
+ parser.add_argument("-d","--days", help="Timeframe in days to check for errors, default 0", default=0,type=int)
987
+ try:
988
+ args = parser.parse_args(shlex.split(args))
989
+ timetotal_minutes = args.minutes + (args.hours * 60) + (args.days * 1440)
990
+ if timetotal_minutes == 0:
991
+ timetotal_minutes = 1440 # default to last 24 hours
992
+ aepp_flow = flowservice.FlowService(config=self.config)
993
+ timereference = int(datetime.now().timestamp()*1000) - (timetotal_minutes * 60 * 1000)
994
+ failed_runs = aepp_flow.getRuns(prop=['metrics.statusSummary.status==failed',f'flowId=={args.flow_id}',f'metrics.durationSummary.startedAtUTC>{timereference}'],n_results="inf")
995
+ with open(f"flow_{args.flow_id}_errors.json", 'w') as f:
996
+ json.dump(failed_runs, f, indent=4)
997
+ console.print(f"Flow errors exported to flow_{args.flow_id}_errors.json", style="green")
998
+ except Exception as e:
999
+ console.print(f"(!) Error: {str(e)}", style="red")
1000
+ except SystemExit:
1001
+ return
1002
+
1003
+ @login_required
1004
+ def do_create_dataset_http_source(self,args):
1005
+ """Create an HTTP Source connection for a specific dataset, XDM compatible data only."""
1006
+ parser = argparse.ArgumentParser(prog='do_create_dataset_http_source', add_help=True)
1007
+ parser.add_argument("dataset", help="Name or ID of the Dataset Source connection to create")
1008
+ try:
1009
+ args = parser.parse_args(shlex.split(args))
1010
+ aepp_cat = catalog.Catalog(config=self.config)
1011
+ datasets = aepp_cat.getDataSets(output='list')
1012
+ if args.dataset in [ds.get("name","") for ds in datasets]:
1013
+ for ds in datasets:
1014
+ if ds.get("name","") == args.dataset:
1015
+ datasetId = ds.get("id")
1016
+ else:
1017
+ datasetId = args.dataset
1018
+ flw = flowservice.FlowService(config=self.config)
1019
+ res = flw.createFlowStreaming(datasetId=datasetId)
1020
+ console.print(f"HTTP Source connection created with Flow ID: {res.get('flow',{}).get('id')}", style="green")
1021
+ source_id = res.get('source_connection_id',{}).get('id')
1022
+ sourceConnection = flw.getSourceConnection(sourceConnectionId=source_id)
1023
+ console.print(f"Endpoint URL: {sourceConnection.get('params',{}).get('inletUrl')}", style="green")
1024
+ except Exception as e:
1025
+ console.print(f"(!) Error: {str(e)}", style="red")
1026
+ except SystemExit:
1027
+ return
1028
+
1029
+ @login_required
1030
+ def do_get_DLZ_credential(self,args):
1031
+ """Get Data Lake Zone credential for the current sandbox"""
1032
+ parser = argparse.ArgumentParser(prog='get_DLZ_credential', add_help=True)
1033
+ parser.add_argument("type",nargs='?',help="Type of credential to retrieve: 'user_drop_zone' or 'dlz_destination'",default="user_drop_zone")
1034
+ try:
1035
+ args = parser.parse_args(shlex.split(args))
1036
+ flw = flowservice.FlowService(config=self.config)
1037
+ cred = flw.getLandingZoneCredential(dlz_type=args.type)
1038
+ console.print(f"Data Lake Zone Credential for sandbox '{self.config.sandbox}':", style="green")
1039
+ console.print_json(data=cred)
1040
+ except Exception as e:
1041
+ console.print(f"(!) Error: {str(e)}", style="red")
1042
+ except SystemExit:
1043
+ return
1044
+
1045
+ @login_required
1046
+ def do_get_queries(self, args):
1047
+ """List top 1000 queries in the current sandbox for the last 24 hours by default, optionally filtered by dataset ID"""
1048
+ parser = argparse.ArgumentParser(prog='get_queries', add_help=True)
1049
+ parser.add_argument("-ds","--dataset", help="Dataset ID to filter queries", default=None)
1050
+ parser.add_argument("-st","--state", help="State to filter queries (running, completed, failed)", default=None)
1051
+ parser.add_argument("-H","--hours", help="Timeframe in hours to check for errors, default 0", default=0,type=int)
1052
+ parser.add_argument("-d","--days", help="Timeframe in days to check for errors, default 0", default=0,type=int)
1053
+ parser.add_argument("-mn","--minutes", help="Timeframe in minutes to check for errors, default 0", default=0,type=int)
1054
+ try:
1055
+ args = parser.parse_args(shlex.split(args))
1056
+ timetotal_minutes = args.minutes + (args.hours * 60) + (args.days * 1440)
1057
+ if timetotal_minutes == 0:
1058
+ timetotal_minutes = 1440 # default to last 24 hours
1059
+ time_reference = int(datetime.now().timestamp()) - (timetotal_minutes * 60)
1060
+ time_reference_z = datetime.fromtimestamp(time_reference).isoformat() + 'Z'
1061
+ params = {'property':f'created>={time_reference_z}','orderBy':'-created'}
1062
+ if args.dataset:
1063
+ if params['property'] == '':
1064
+ params['property'] = f'referenced_datasets=={args.dataset}'
1065
+ else:
1066
+ params['property'] += f',referenced_datasets=={args.dataset}'
1067
+ if params['property'] == '':
1068
+ params = None
1069
+ else:
1070
+ params['property'] = urllib.parse.quote(params['property'])
1071
+ aepp_query = queryservice.QueryService(config=self.config)
1072
+ queries = aepp_query.getQueries(property=params['property'] if params else None, orderby=params['orderBy'])
1073
+ list_queries = []
1074
+ for q in queries:
1075
+ if q['client'] == "Adobe Query Service UI" or q["client"] == 'Generic PostgreSQL':
1076
+ list_queries.append(q)
1077
+ for q in list_queries:
1078
+ obj = {
1079
+ "id": q.get("id","N/A"),
1080
+ "created": q.get("created"),
1081
+ "client": q.get("client","N/A"),
1082
+ "elapsedTime": q.get("elapsedTime","N/A"),
1083
+ "userId": q.get("userId","N/A"),
1084
+ }
1085
+ list_queries.append(obj)
1086
+ df_queries = pd.DataFrame(list_queries)
1087
+ df_queries.to_csv(f"{self.config.sandbox}_queries.csv",index=False)
1088
+ console.print(f"Queries exported to {self.config.sandbox}_queries.csv", style="green")
1089
+ table = Table(title=f"Queries in Sandbox: {self.config.sandbox}")
1090
+ table.add_column("ID", style="cyan")
1091
+ table.add_column("Created", style="yellow")
1092
+ table.add_column("Client", style="white")
1093
+ table.add_column("Elapsed Time (ms)", style="white")
1094
+ for q in queries:
1095
+ table.add_row(
1096
+ q.get("id","N/A"),
1097
+ q.get("created","N/A"),
1098
+ q.get("client","N/A"),
1099
+ str(q.get("elapsedTime","N/A"))
1100
+ )
1101
+ console.print(table)
1102
+ except Exception as e:
1103
+ console.print(f"(!) Error: {str(e)}", style="red")
1104
+ except SystemExit:
1105
+ return
1106
+
1107
+ @login_required
1108
+ def do_query(self,args):
1109
+ """Execute a SQL query against the current sandbox"""
1110
+ parser = argparse.ArgumentParser(prog='query', add_help=True)
1111
+ parser.add_argument("sql_query", help="SQL query to execute",type=str)
1112
+ try:
1113
+ args = parser.parse_args(shlex.split(args))
1114
+ aepp_query = queryservice.QueryService(config=self.config)
1115
+ conn = aepp_query.connection()
1116
+ iqs2 = queryservice.InteractiveQuery2(conn)
1117
+ result:pd.DataFrame = iqs2.query(sql=args.sql_query)
1118
+ result.to_csv(f"query_result_{int(datetime.now().timestamp())}.csv", index=False)
1119
+ console.print(f"Query result exported to query_result_{int(datetime.now().timestamp())}.csv", style="green")
1120
+ console.print(result)
1121
+ except Exception as e:
1122
+ console.print(f"(!) Error: {str(e)}", style="red")
1123
+ except SystemExit:
1124
+ return
1125
+
1126
+
1127
+ @login_required
1128
+ def do_extractArtefacts(self,args):
1129
+ """extractArtefacts localfolder"""
1130
+ console.print("Extracting artefacts...", style="blue")
1131
+ parser = argparse.ArgumentParser(prog='extractArtefacts', description='Extract artefacts from AEP')
1132
+ parser.add_argument('-lf','--localfolder', help='Local folder to extract artefacts to', default='./extractions')
1133
+ parser.add_argument('-rg','--region', help='Region to extract artefacts from: "ndl2" (default), "va7", "aus5", "can2", "ind2"',default='ndl2')
1134
+ try:
1135
+ args = parser.parse_args(shlex.split(args))
1136
+ aepp.extractSandboxArtefacts(
1137
+ sandbox=self.config,
1138
+ localFolder=args.localfolder,
1139
+ region=args.region
1140
+ )
1141
+ console.print(Panel("Extraction completed!", style="green"))
1142
+ except SystemExit:
1143
+ return
1144
+
1145
+ @login_required
1146
+ def do_extractArtefact(self,args):
1147
+ """extractArtefacts localfolder"""
1148
+ console.print("Extracting artefact...", style="blue")
1149
+ parser = argparse.ArgumentParser(prog='extractArtefact', description='Extract artefacts from AEP')
1150
+ parser.add_argument('artefact', help='artefact to extract (name or id): "schema","fieldgroup","datatype","descriptor","dataset","identity","mergepolicy","audience"')
1151
+ parser.add_argument('-at','--artefactType', help='artefact type ')
1152
+ parser.add_argument('-lf','--localfolder', help='Local folder to extract artefacts to',default='extractions')
1153
+ parser.add_argument('-rg','--region', help='Region to extract artefacts from: "ndl2" (default), "va7", "aus5", "can2", "ind2"',default='ndl2')
1154
+
1155
+ try:
1156
+ args = parser.parse_args(shlex.split(args))
1157
+ aepp.extractSandboxArtefact(
1158
+ artefact=args.artefact,
1159
+ artefactType=args.artefactType,
1160
+ sandbox=self.config,
1161
+ localFolder=args.localfolder
1162
+ )
1163
+ console.print("Extraction completed!", style="green")
1164
+ except SystemExit:
1165
+ return
1166
+
1167
+ @login_required
1168
+ def do_sync(self,args):
1169
+ """extractArtefacts localfolder"""
1170
+ console.print("Syncing artefact...", style="blue")
1171
+ parser = argparse.ArgumentParser(prog='extractArtefact', description='Extract artefacts from AEP')
1172
+ parser.add_argument('artefact', help='artefact to extract (name or id): "schema","fieldgroup","datatype","descriptor","dataset","identity","mergepolicy","audience"')
1173
+ parser.add_argument('-at','--artefactType', help='artefact type ')
1174
+ parser.add_argument('-t','--targets', help='target sandboxes')
1175
+ parser.add_argument('-lf','--localfolder', help='Local folder to extract artefacts to',default='extractions')
1176
+ parser.add_argument('-b','--baseSandbox', help='Base sandbox for synchronization')
1177
+ parser.add_argument('-rg','--region', help='Region to extract artefacts from: "ndl2" (default), "va7", "aus5", "can2", "ind2"',default='ndl2')
1178
+ parser.add_argument('-v','--verbose', help='Enable verbose output',default=True)
1179
+ try:
1180
+ args = parser.parse_args(shlex.split(args))
1181
+ if ',' in args.targets:
1182
+ args.targets = args.targets.split(',')
1183
+ else:
1184
+ args.targets = [args.targets]
1185
+ console.print("Initializing Synchronizor...", style="blue")
1186
+ if args.baseSandbox:
1187
+ synchronizor = synchronizer.Synchronizer(
1188
+ config=self.config,
1189
+ targets=args.targets,
1190
+ region=args.region,
1191
+ baseSandbox=args.baseSandbox,
1192
+ )
1193
+ elif args.localfolder:
1194
+ synchronizor = synchronizer.Synchronizer(
1195
+ config=self.config,
1196
+ targets=args.targets,
1197
+ region=args.region,
1198
+ localFolder=args.localfolder,
1199
+ )
1200
+ console.print("Starting Sync...", style="blue")
1201
+ synchronizor.syncComponent(
1202
+ component=args.artefact,
1203
+ componentType=args.artefactType,
1204
+ verbose=args.verbose
1205
+ )
1206
+ console.print("Sync completed!", style="green")
1207
+ except SystemExit:
1208
+ return
1209
+
1210
+
1211
+ def do_exit(self, args):
1212
+ """Exit the application"""
1213
+ console.print(Panel("Exiting...", style="blue"))
1214
+ return True # Stops the loop
1215
+
1216
+ def do_EOF(self, args):
1217
+ """Handle Ctrl+D"""
1218
+ console.print(Panel("Exiting...", style="blue"))
1219
+ return True
1220
+
1221
+ # --- 3. The Entry Point ---#
1222
+
1223
+ def main():
1224
+ # ARGPARSE: Handles the initial setup flags
1225
+ parser = argparse.ArgumentParser(description="Interactive Client Tool",add_help=True)
1226
+
1227
+ # Optional: Allow passing user/pass via flags to skip the interactive login step
1228
+ parser.add_argument("-sx", "--sandbox", help="Auto-login sandbox")
1229
+ parser.add_argument("-s", "--secret", help="Secret")
1230
+ parser.add_argument("-o", "--org_id", help="Auto-login org ID")
1231
+ parser.add_argument("-sc", "--scopes", help="Scopes")
1232
+ parser.add_argument("-cid", "--client_id", help="Auto-login client ID")
1233
+ parser.add_argument("-cf", "--config_file", help="Path to config file", default=None)
1234
+ args = parser.parse_args()
1235
+ shell = ServiceShell(**vars(args))
1236
+ try:
1237
+ shell.cmdloop()
1238
+ except KeyboardInterrupt:
1239
+ console.print(Panel("\nForce closing...", style="red"))
1240
+
1241
+ if __name__ == "__main__":
1242
+ main()