kaqing 2.0.145__py3-none-any.whl → 2.0.172__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kaqing might be problematic. Click here for more details.

Files changed (172) hide show
  1. adam/__init__.py +0 -2
  2. adam/app_session.py +8 -11
  3. adam/batch.py +3 -3
  4. adam/checks/check_utils.py +14 -46
  5. adam/checks/cpu.py +7 -1
  6. adam/checks/cpu_metrics.py +52 -0
  7. adam/checks/disk.py +2 -3
  8. adam/columns/columns.py +3 -1
  9. adam/columns/cpu.py +3 -1
  10. adam/columns/cpu_metrics.py +22 -0
  11. adam/columns/memory.py +3 -4
  12. adam/commands/__init__.py +18 -0
  13. adam/commands/alter_tables.py +43 -47
  14. adam/commands/audit/audit.py +22 -23
  15. adam/commands/audit/audit_repair_tables.py +14 -17
  16. adam/commands/audit/audit_run.py +15 -23
  17. adam/commands/audit/show_last10.py +10 -13
  18. adam/commands/audit/show_slow10.py +10 -13
  19. adam/commands/audit/show_top10.py +10 -13
  20. adam/commands/audit/utils_show_top10.py +2 -3
  21. adam/commands/bash/__init__.py +5 -0
  22. adam/commands/bash/bash.py +7 -104
  23. adam/commands/bash/utils_bash.py +16 -0
  24. adam/commands/cat.py +13 -19
  25. adam/commands/cd.py +8 -10
  26. adam/commands/check.py +20 -21
  27. adam/commands/cli_commands.py +2 -3
  28. adam/commands/code.py +20 -23
  29. adam/commands/command.py +120 -39
  30. adam/commands/commands_utils.py +8 -17
  31. adam/commands/cp.py +33 -39
  32. adam/commands/cql/cql_completions.py +9 -4
  33. adam/commands/cql/cqlsh.py +10 -30
  34. adam/commands/cql/{cql_utils.py → utils_cql.py} +149 -15
  35. adam/commands/deploy/code_start.py +7 -10
  36. adam/commands/deploy/code_stop.py +4 -21
  37. adam/commands/deploy/code_utils.py +3 -3
  38. adam/commands/deploy/deploy.py +4 -27
  39. adam/commands/deploy/deploy_frontend.py +14 -17
  40. adam/commands/deploy/deploy_pg_agent.py +2 -5
  41. adam/commands/deploy/deploy_pod.py +64 -68
  42. adam/commands/deploy/undeploy.py +4 -27
  43. adam/commands/deploy/undeploy_frontend.py +4 -7
  44. adam/commands/deploy/undeploy_pg_agent.py +4 -7
  45. adam/commands/deploy/undeploy_pod.py +9 -12
  46. adam/commands/devices/device.py +93 -2
  47. adam/commands/devices/device_app.py +37 -10
  48. adam/commands/devices/device_auit_log.py +8 -2
  49. adam/commands/devices/device_cass.py +47 -7
  50. adam/commands/devices/device_export.py +2 -2
  51. adam/commands/devices/device_postgres.py +41 -6
  52. adam/commands/exit.py +1 -4
  53. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  54. adam/commands/export/clean_up_export_sessions.py +18 -7
  55. adam/commands/export/drop_export_database.py +15 -18
  56. adam/commands/export/drop_export_databases.py +6 -9
  57. adam/commands/export/export.py +8 -38
  58. adam/commands/export/export_databases.py +16 -12
  59. adam/commands/export/export_handlers.py +71 -0
  60. adam/commands/export/export_select.py +33 -24
  61. adam/commands/export/export_use.py +12 -15
  62. adam/commands/export/exporter.py +37 -48
  63. adam/commands/export/import_session.py +4 -32
  64. adam/commands/export/importer_athena.py +4 -7
  65. adam/commands/export/importer_sqlite.py +19 -27
  66. adam/commands/export/show_column_counts.py +13 -22
  67. adam/commands/export/show_export_databases.py +3 -6
  68. adam/commands/export/show_export_session.py +10 -13
  69. adam/commands/export/show_export_sessions.py +8 -11
  70. adam/commands/export/utils_export.py +24 -1
  71. adam/commands/intermediate_command.py +49 -0
  72. adam/commands/issues.py +11 -43
  73. adam/commands/kubectl.py +3 -6
  74. adam/commands/login.py +22 -24
  75. adam/commands/logs.py +3 -6
  76. adam/commands/ls.py +8 -9
  77. adam/commands/medusa/medusa.py +4 -22
  78. adam/commands/medusa/medusa_backup.py +20 -24
  79. adam/commands/medusa/medusa_restore.py +29 -33
  80. adam/commands/medusa/medusa_show_backupjobs.py +14 -18
  81. adam/commands/medusa/medusa_show_restorejobs.py +11 -18
  82. adam/commands/nodetool.py +6 -15
  83. adam/commands/param_get.py +11 -12
  84. adam/commands/param_set.py +9 -10
  85. adam/commands/postgres/postgres.py +29 -37
  86. adam/commands/postgres/postgres_context.py +47 -23
  87. adam/commands/postgres/postgres_ls.py +4 -8
  88. adam/commands/postgres/postgres_preview.py +5 -9
  89. adam/commands/postgres/psql_completions.py +1 -1
  90. adam/commands/postgres/utils_postgres.py +66 -0
  91. adam/commands/preview_table.py +5 -44
  92. adam/commands/pwd.py +13 -16
  93. adam/commands/reaper/reaper.py +4 -27
  94. adam/commands/reaper/reaper_forward.py +48 -55
  95. adam/commands/reaper/reaper_forward_session.py +6 -0
  96. adam/commands/reaper/reaper_forward_stop.py +10 -16
  97. adam/commands/reaper/reaper_restart.py +7 -14
  98. adam/commands/reaper/reaper_run_abort.py +11 -30
  99. adam/commands/reaper/reaper_runs.py +42 -57
  100. adam/commands/reaper/reaper_runs_abort.py +29 -49
  101. adam/commands/reaper/reaper_schedule_activate.py +11 -30
  102. adam/commands/reaper/reaper_schedule_start.py +10 -29
  103. adam/commands/reaper/reaper_schedule_stop.py +10 -29
  104. adam/commands/reaper/reaper_schedules.py +4 -14
  105. adam/commands/reaper/reaper_status.py +8 -16
  106. adam/commands/reaper/utils_reaper.py +196 -0
  107. adam/commands/repair/repair.py +4 -22
  108. adam/commands/repair/repair_log.py +5 -11
  109. adam/commands/repair/repair_run.py +27 -34
  110. adam/commands/repair/repair_scan.py +32 -38
  111. adam/commands/repair/repair_stop.py +5 -11
  112. adam/commands/report.py +27 -29
  113. adam/commands/restart.py +25 -26
  114. adam/commands/rollout.py +19 -24
  115. adam/commands/shell.py +10 -4
  116. adam/commands/show/show.py +10 -26
  117. adam/commands/show/show_cassandra_repairs.py +35 -0
  118. adam/commands/show/show_cassandra_status.py +32 -43
  119. adam/commands/show/show_cassandra_version.py +5 -18
  120. adam/commands/show/show_commands.py +19 -24
  121. adam/commands/show/show_host.py +1 -1
  122. adam/commands/show/show_login.py +20 -27
  123. adam/commands/show/show_processes.py +15 -19
  124. adam/commands/show/show_storage.py +10 -20
  125. adam/commands/watch.py +26 -29
  126. adam/config.py +4 -16
  127. adam/embedded_params.py +1 -1
  128. adam/log.py +4 -4
  129. adam/pod_exec_result.py +3 -3
  130. adam/repl.py +29 -32
  131. adam/repl_commands.py +11 -11
  132. adam/repl_state.py +52 -26
  133. adam/sql/sql_completer.py +4 -6
  134. adam/sql/sql_state_machine.py +21 -14
  135. adam/sso/authn_ad.py +6 -8
  136. adam/sso/authn_okta.py +4 -6
  137. adam/sso/cred_cache.py +3 -5
  138. adam/sso/idp.py +9 -12
  139. adam/utils.py +393 -33
  140. adam/utils_athena.py +14 -13
  141. adam/utils_audits.py +12 -12
  142. adam/utils_issues.py +32 -0
  143. adam/utils_k8s/app_clusters.py +13 -18
  144. adam/utils_k8s/app_pods.py +2 -0
  145. adam/utils_k8s/cassandra_clusters.py +21 -18
  146. adam/utils_k8s/custom_resources.py +16 -17
  147. adam/utils_k8s/ingresses.py +2 -2
  148. adam/utils_k8s/jobs.py +7 -11
  149. adam/utils_k8s/k8s.py +87 -0
  150. adam/utils_k8s/pods.py +14 -76
  151. adam/utils_k8s/secrets.py +4 -4
  152. adam/utils_k8s/service_accounts.py +5 -4
  153. adam/utils_k8s/services.py +2 -2
  154. adam/utils_k8s/statefulsets.py +1 -12
  155. adam/utils_repl/state_machine.py +3 -3
  156. adam/utils_sqlite.py +78 -42
  157. adam/version.py +1 -1
  158. {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/METADATA +1 -1
  159. kaqing-2.0.172.dist-info/RECORD +230 -0
  160. adam/commands/app.py +0 -67
  161. adam/commands/app_ping.py +0 -44
  162. adam/commands/export/clean_up_export_session.py +0 -53
  163. adam/commands/postgres/postgres_utils.py +0 -31
  164. adam/commands/reaper/reaper_session.py +0 -159
  165. adam/commands/show/show_app_actions.py +0 -56
  166. adam/commands/show/show_app_id.py +0 -47
  167. adam/commands/show/show_app_queues.py +0 -45
  168. adam/commands/show/show_repairs.py +0 -47
  169. kaqing-2.0.145.dist-info/RECORD +0 -227
  170. {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/WHEEL +0 -0
  171. {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/entry_points.txt +0 -0
  172. {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/top_level.txt +0 -0
@@ -27,26 +27,23 @@ class ExportUse(Command):
27
27
  if not(args := self.args(cmd)):
28
28
  return super().run(cmd, state)
29
29
 
30
- state, args = self.apply_state(args, state)
31
- if not self.validate_state(state):
32
- return state
33
-
34
- if not args:
35
- state.export_session = None
30
+ with self.validate(args, state) as (args, state):
31
+ if not args:
32
+ state.export_session = None
36
33
 
37
- log2('Export database is unset.')
34
+ log2('Export database is unset.')
38
35
 
39
- return state
36
+ return state
40
37
 
41
- state.export_session = args[0]
42
- if state.export_session.startswith('e'):
43
- Athena.clear_cache()
44
- else:
45
- SQLite.clear_cache()
38
+ state.export_session = args[0]
39
+ if state.export_session.startswith('e'):
40
+ Athena.clear_cache()
41
+ else:
42
+ SQLite.clear_cache()
46
43
 
47
- ExportDatabases.display_export_db(state.export_session)
44
+ ExportDatabases.display_export_db(state.export_session)
48
45
 
49
- return state
46
+ return state
50
47
 
51
48
  def completion(self, state: ReplState):
52
49
  return super().completion(state, {n: None for n in ExportDatabases.database_names()})
@@ -1,11 +1,9 @@
1
- from concurrent.futures import ThreadPoolExecutor, as_completed
2
1
  from datetime import datetime
3
2
  import functools
4
3
  import re
5
4
  import time
6
- import traceback
7
5
 
8
- from adam.commands.cql.cql_utils import cassandra_table_names, run_cql, table_spec
6
+ from adam.commands.cql.utils_cql import cassandra_table_names, run_cql, table_spec
9
7
  from adam.commands.export.export_databases import ExportDatabases
10
8
  from adam.commands.export.importer import Importer
11
9
  from adam.commands.export.importer_athena import AthenaImporter
@@ -14,7 +12,7 @@ from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, Exp
14
12
  from adam.config import Config
15
13
  from adam.pod_exec_result import PodExecResult
16
14
  from adam.repl_state import ReplState
17
- from adam.utils import elapsed_time, log2, ing
15
+ from adam.utils import debug, parallelize, log2, ing, log_exc
18
16
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
19
17
  from adam.utils_k8s.pods import log_prefix
20
18
  from adam.utils_k8s.statefulsets import StatefulSets
@@ -25,7 +23,7 @@ class Exporter:
25
23
  log2('export-only for testing')
26
24
 
27
25
  spec: ExportSpec = None
28
- try:
26
+ with log_exc(True):
29
27
  spec = Exporter.export_spec(' '.join(args), state)
30
28
 
31
29
  statuses, spec = Exporter._export_tables(spec, state, max_workers=max_workers, export_state='init')
@@ -33,8 +31,6 @@ class Exporter:
33
31
  return statuses, spec
34
32
 
35
33
  return Exporter._export_tables(spec, state, export_only, max_workers, 'pending_export')
36
- except Exception as e:
37
- log2(e)
38
34
 
39
35
  return [], None
40
36
 
@@ -53,12 +49,19 @@ class Exporter:
53
49
  raise Exception(f"You're currently using {importer_from_session} export database. You cannot export tables with {spec.importer} type database.")
54
50
  else:
55
51
  spec.importer = Importer.importer_from_session(session)
52
+
53
+ if spec.importer == 'athena' and not AthenaImporter.ping():
54
+ raise Exception('Credentials for Athena is not present.')
56
55
  else:
57
56
  if not spec.importer:
58
57
  spec.importer = Config().get('export.default-importer', 'sqlite')
59
58
 
60
59
  prefix = Importer.prefix_from_importer(spec.importer)
61
60
  session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
61
+
62
+ if spec.importer == 'athena' and not AthenaImporter.ping():
63
+ raise Exception('Credentials for Athena is not present.')
64
+
62
65
  if spec.importer != 'csv':
63
66
  state.export_session = session
64
67
 
@@ -68,7 +71,7 @@ class Exporter:
68
71
 
69
72
  def import_session(args: list[str], state: ReplState, max_workers = 0) -> tuple[list[str], ExportSpec]:
70
73
  import_spec: ImportSpec = None
71
- try:
74
+ with log_exc(True):
72
75
  import_spec = Exporter.import_spec(' '.join(args), state)
73
76
  tables, status_in_whole = ExportTableStatus.from_session(state.sts, state.pod, state.namespace, import_spec.session)
74
77
  if status_in_whole == 'done':
@@ -77,12 +80,7 @@ class Exporter:
77
80
 
78
81
  spec = ExportSpec(None, None, importer=import_spec.importer, tables=[ExportTableSpec.from_status(table) for table in tables], session=import_spec.session)
79
82
 
80
- return Exporter._export_tables(spec, state, max_workers=max_workers)
81
- except Exception as e:
82
- if Config().is_debug():
83
- traceback.print_exception(e)
84
- else:
85
- log2(e)
83
+ return Exporter._export_tables(spec, state, max_workers=max_workers, export_state = 'import')
86
84
 
87
85
  return [], None
88
86
 
@@ -99,16 +97,19 @@ class Exporter:
99
97
  spec.importer = Importer.importer_from_session(state.export_session)
100
98
  if not spec.importer:
101
99
  spec.importer = Config().get('export.default-importer', 'sqlite')
100
+
101
+ if spec.importer == 'athena' and not AthenaImporter.ping():
102
+ raise Exception('Credentials for Athena is not present.')
102
103
  else:
103
- if spec.importer:
104
- if not AthenaImporter.ping():
105
- raise Exception('Credentials for Athena are not present.')
106
- else:
104
+ if not spec.importer:
107
105
  spec.importer = Importer.importer_from_session(spec.session)
108
106
 
109
107
  if spec.importer == 'csv':
110
108
  spec.importer = Config().get('export.default-importer', 'sqlite')
111
109
 
110
+ if spec.importer == 'athena' and not AthenaImporter.ping():
111
+ raise Exception('Credentials for Athena is not present.')
112
+
112
113
  prefix = Importer.prefix_from_importer(spec.importer)
113
114
  session = f'{prefix}{spec.session[1:]}'
114
115
  state.export_session = session
@@ -128,20 +129,14 @@ class Exporter:
128
129
  if export_state == 'init':
129
130
  CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_dir()}/{spec.session}_*', show_out=Config().is_debug(), shell='bash')
130
131
 
131
- if max_workers > 1 and len(spec.tables) > 1:
132
- log2(f'Executing on {len(spec.tables)} Cassandra tables in parallel...')
133
- start_time = time.time()
134
- try:
135
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
136
- futures = [executor.submit(Exporter.export_table, table, state, spec.session, spec.importer, export_only, True, consistency=spec.consistency, export_state=export_state) for table in spec.tables]
137
- if len(futures) == 0:
138
- return [], spec
139
-
140
- return [future.result() for future in as_completed(futures)], spec
141
- finally:
142
- log2(f"{len(spec.tables)} parallel table export elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
143
- else:
144
- return [Exporter.export_table(table, state, spec.session, spec.importer, export_only, multi_tables=len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state) for table in spec.tables], spec
132
+ action = f'[{spec.session}] Exporting|Exported'
133
+ if export_state == 'init':
134
+ action = f'[{spec.session}] Preparing|Prepared'
135
+ elif export_state == 'import':
136
+ action = f'[{spec.session}] Importing|Imported'
137
+
138
+ with parallelize(spec.tables, max_workers, msg=action + ' {size} Cassandra tables') as exec:
139
+ return exec.map(lambda table: Exporter.export_table(table, state, spec.session, spec.importer, export_only, len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state)), spec
145
140
 
146
141
  def export_table(spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, consistency: str = None, export_state=None):
147
142
  s: str = None
@@ -167,8 +162,7 @@ class Exporter:
167
162
  status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
168
163
  while status.status != 'done':
169
164
  if status.status == 'export_in_pregress':
170
- if Config().is_debug():
171
- log2('Exporting to CSV is still in progess, sleeping for 1 sec...')
165
+ debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
172
166
  time.sleep(1)
173
167
  elif status.status == 'exported':
174
168
  log_file = Exporter.rename_to_pending_import(spec, state, session, target_table)
@@ -283,20 +277,15 @@ class Exporter:
283
277
  if not max_workers:
284
278
  max_workers = Config().action_workers('export', 8)
285
279
 
286
- if max_workers > 1 and len(sessions) > 1:
287
- log2(f'Executing on {len(sessions)} export session clean ups in parallel...')
288
- start_time = time.time()
289
- try:
290
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
291
- futures = [executor.submit(Exporter.clean_up_session, sts, pod, namespace, session, True) for session in sessions]
292
- if len(futures) == 0:
293
- return []
294
-
295
- return [future.result() for future in as_completed(futures)]
296
- finally:
297
- log2(f"{len(sessions)} parallel session clean ups elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
298
- else:
299
- return [Exporter.clean_up_session(sts, pod, namespace, session) for session in sessions]
280
+ with parallelize(sessions, max_workers, msg='Cleaning|Cleaned up {size} export sessions') as exec:
281
+ cnt_tuples = exec.map(lambda session: Exporter.clean_up_session(sts, pod, namespace, session, True))
282
+ csv_cnt = 0
283
+ log_cnt = 0
284
+ for (csv, log) in cnt_tuples:
285
+ csv_cnt += csv
286
+ log_cnt += log
287
+
288
+ return csv_cnt, log_cnt
300
289
 
301
290
  def clean_up_session(sts: str, pod: str, namespace: str, session: str, multi_tables = True):
302
291
  if not sts or not namespace:
@@ -1,9 +1,7 @@
1
1
  from adam.commands.command import Command
2
- from adam.commands.export.export_databases import ExportDatabases
2
+ from adam.commands.export.export_handlers import export
3
3
  from adam.commands.export.exporter import Exporter
4
4
  from adam.repl_state import ReplState, RequiredState
5
- from adam.utils import log, log2
6
- from adam.utils_k8s.statefulsets import StatefulSets
7
5
 
8
6
  class ImportSession(Command):
9
7
  COMMAND = 'import session'
@@ -27,35 +25,9 @@ class ImportSession(Command):
27
25
  if not(args := self.args(cmd)):
28
26
  return super().run(cmd, state)
29
27
 
30
- state, args = self.apply_state(args, state)
31
- if not self.validate_state(state):
32
- return state
33
-
34
- if not args:
35
- if state.in_repl:
36
- log2('Specify export session name.')
37
- else:
38
- log2('* Export session name is missing.')
39
-
40
- Command.display_help()
41
-
42
- return 'command-missing'
43
-
44
- if not state.pod:
45
- state.push()
46
- state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
47
-
48
- try:
49
- tables, _ = Exporter.import_session(args, state)
50
- if tables:
51
- Exporter.clear_export_session_cache()
52
-
53
- log()
54
- ExportDatabases.display_export_db(state.export_session)
55
- finally:
56
- state.pop()
57
-
58
- return state
28
+ with self.validate(args, state) as (args, state):
29
+ with export(state) as exporter:
30
+ return exporter.import_sesion(args)
59
31
 
60
32
  def completion(self, state: ReplState):
61
33
  # warm up cache
@@ -3,7 +3,7 @@ import boto3
3
3
  from adam.commands.export.importer import Importer
4
4
  from adam.commands.export.utils_export import GeneratorStream
5
5
  from adam.config import Config
6
- from adam.utils import log2, ing
6
+ from adam.utils import debug, log2, ing
7
7
  from adam.utils_athena import Athena
8
8
  from adam.utils_k8s.pods import Pods
9
9
 
@@ -38,13 +38,11 @@ class AthenaImporter(Importer):
38
38
  msg = f"[{to_session}] Creating table {target_table}"
39
39
  with ing(msg, suppress_log=multi_tables):
40
40
  query = f'CREATE DATABASE IF NOT EXISTS {db};'
41
- if Config().is_debug():
42
- log2(query)
41
+ debug(query)
43
42
  Athena.query(query, 'default')
44
43
 
45
44
  query = f'DROP TABLE IF EXISTS {target_table};'
46
- if Config().is_debug():
47
- log2(query)
45
+ debug(query)
48
46
  Athena.query(query, db)
49
47
 
50
48
  athena_columns = ', '.join([f'{c} string' for c in columns.split(',')])
@@ -56,8 +54,7 @@ class AthenaImporter(Importer):
56
54
  ' "quoteChar" = "\\"")\n' + \
57
55
  f"LOCATION 's3://{bucket}/export/{db}/{keyspace}/{target_table}'\n" + \
58
56
  'TBLPROPERTIES ("skip.header.line.count"="1");'
59
- if Config().is_debug():
60
- log2(query)
57
+ debug(query)
61
58
  try:
62
59
  Athena.query(query, db)
63
60
  except Exception as e:
@@ -1,12 +1,10 @@
1
- import os
2
- import sqlite3
3
1
  import pandas
4
2
 
5
3
  from adam.commands.export.importer import Importer
6
4
  from adam.commands.export.utils_export import GeneratorStream
7
5
  from adam.utils import log2, ing
8
6
  from adam.utils_k8s.pods import Pods
9
- from adam.utils_sqlite import SQLite
7
+ from adam.utils_sqlite import SQLite, sqlite
10
8
 
11
9
  class SqliteImporter(Importer):
12
10
  def prefix(self):
@@ -14,34 +12,28 @@ class SqliteImporter(Importer):
14
12
 
15
13
  def import_from_csv(self, pod: str, namespace: str, to_session: str, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
16
14
  csv_file = self.csv_file(from_session, table, target_table)
17
- db = self.db(to_session, keyspace)
18
15
 
19
16
  succeeded = False
20
- conn = None
21
- try:
22
- os.makedirs(SQLite.local_db_dir(), exist_ok=True)
23
- conn = sqlite3.connect(f'{SQLite.local_db_dir()}/{db}.db')
17
+ with sqlite(to_session, keyspace) as conn:
18
+ try:
19
+ with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
20
+ bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
21
+ df = pandas.read_csv(GeneratorStream(bytes))
24
22
 
25
- with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
26
- bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
27
- df = pandas.read_csv(GeneratorStream(bytes))
23
+ df.to_sql(target_table, conn, index=False, if_exists='replace')
28
24
 
29
- df.to_sql(target_table, conn, index=False, if_exists='replace')
25
+ to, _ = self.move_to_done(pod, namespace, to_session, from_session, keyspace, target_table)
30
26
 
31
- to, _ = self.move_to_done(pod, namespace, to_session, from_session, keyspace, target_table)
27
+ succeeded = True
32
28
 
33
- succeeded = True
29
+ return to, to_session
30
+ finally:
31
+ if succeeded:
32
+ self.remove_csv(pod, namespace, from_session, table, target_table, multi_tables)
33
+ SQLite.clear_cache()
34
34
 
35
- return to, to_session
36
- finally:
37
- if succeeded:
38
- self.remove_csv(pod, namespace, from_session, table, target_table, multi_tables)
39
- SQLite.clear_cache()
40
-
41
- if not multi_tables:
42
- query = f'select * from {target_table} limit 10'
43
- log2(query)
44
- SQLite.run_query(query, conn_passed=conn)
45
-
46
- if conn:
47
- conn.close()
35
+ if not multi_tables:
36
+ with sqlite(to_session) as conn:
37
+ query = f'select * from {keyspace}.{target_table} limit 10'
38
+ log2(query)
39
+ SQLite.run_query_with_conn(conn, query)
@@ -3,8 +3,6 @@ from adam.commands.export.export_databases import ExportDatabases
3
3
  from adam.config import Config
4
4
  from adam.repl_state import ReplState, RequiredState
5
5
  from adam.utils import log2
6
- from adam.utils_athena import Athena
7
- from adam.utils_sqlite import SQLite
8
6
 
9
7
  class ShowColumnCounts(Command):
10
8
  COMMAND = 'show column counts on'
@@ -28,30 +26,23 @@ class ShowColumnCounts(Command):
28
26
  if not(args := self.args(cmd)):
29
27
  return super().run(cmd, state)
30
28
 
31
- state, args = self.apply_state(args, state)
32
- if not self.validate_state(state):
33
- return state
34
-
35
- if not args:
36
- if state.in_repl:
37
- log2('Use a SQL statement.')
38
- else:
39
- log2('* SQL statement is missing.')
29
+ with self.validate(args, state) as (args, state):
30
+ if not args:
31
+ if state.in_repl:
32
+ log2('Use a SQL statement.')
33
+ else:
34
+ log2('* SQL statement is missing.')
40
35
 
41
- Command.display_help()
36
+ Command.display_help()
42
37
 
43
- return 'command-missing'
38
+ return 'command-missing'
44
39
 
45
- copy_or_export = 'copy'
46
- if state.export_session.startswith('e'):
47
- copy_or_export = 'export'
40
+ table = args[0]
41
+ query = Config().get(f'export.column_counts_query', 'select id, count(id) as columns from {table} group by id')
42
+ query = query.replace('{table}', table)
43
+ ExportDatabases.run_query(query, state.export_session)
48
44
 
49
- table = args[0]
50
- query = Config().get(f'{copy_or_export}.column_counts_query', 'select id, count(id) as columns from {table} group by id')
51
- query = query.replace('{table}', table)
52
- ExportDatabases.run_query(query, state.export_session)
53
-
54
- return state
45
+ return state
55
46
 
56
47
  def completion(self, state: ReplState):
57
48
  if not state.export_session:
@@ -24,13 +24,10 @@ class ShowExportDatabases(Command):
24
24
  if not(args := self.args(cmd)):
25
25
  return super().run(cmd, state)
26
26
 
27
- state, args = self.apply_state(args, state)
28
- if not self.validate_state(state):
29
- return state
30
-
31
- DeviceExport().show_export_databases()
27
+ with self.validate(args, state) as (args, state):
28
+ DeviceExport().show_export_databases()
32
29
 
33
- return state
30
+ return state
34
31
 
35
32
  def completion(self, state: ReplState):
36
33
  return DeviceExport().ls_completion(ShowExportDatabases.COMMAND, state, default = super().completion(state))
@@ -26,23 +26,20 @@ class ShowExportSession(Command):
26
26
  if not(args := self.args(cmd)):
27
27
  return super().run(cmd, state)
28
28
 
29
- state, args = self.apply_state(args, state)
30
- if not self.validate_state(state):
31
- return state
32
-
33
- if not args:
34
- if state.in_repl:
35
- log2('Specify export database name.')
36
- else:
37
- log2('* Database name is missing.')
29
+ with self.validate(args, state) as (args, state):
30
+ if not args:
31
+ if state.in_repl:
32
+ log2('Specify export database name.')
33
+ else:
34
+ log2('* Database name is missing.')
38
35
 
39
- Command.display_help()
36
+ Command.display_help()
40
37
 
41
- return 'command-missing'
38
+ return 'command-missing'
42
39
 
43
- ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
40
+ ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
44
41
 
45
- return state
42
+ return state
46
43
 
47
44
  def completion(self, state: ReplState):
48
45
  return super().completion(state, {session: None for session in Exporter.export_session_names(state.sts, state.pod, state.namespace)})
@@ -26,19 +26,16 @@ class ShowExportSessions(Command):
26
26
  if not(args := self.args(cmd)):
27
27
  return super().run(cmd, state)
28
28
 
29
- state, args = self.apply_state(args, state)
30
- if not self.validate_state(state):
31
- return state
32
-
33
- pod = state.pod
34
- if not pod:
35
- pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
29
+ with self.validate(args, state) as (args, state):
30
+ pod = state.pod
31
+ if not pod:
32
+ pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
36
33
 
37
- sessions: dict[str, str] = Exporter.find_export_sessions(pod, state.namespace)
38
- log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
39
- header='EXPORT_SESSION\tSTATUS', separator='\t'))
34
+ sessions: dict[str, str] = Exporter.find_export_sessions(pod, state.namespace)
35
+ log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
36
+ header='EXPORT_SESSION\tSTATUS', separator='\t'))
40
37
 
41
- return state
38
+ return state
42
39
 
43
40
  def completion(self, state: ReplState):
44
41
  return super().completion(state)
@@ -3,8 +3,10 @@ import re
3
3
 
4
4
  from adam.config import Config
5
5
  from adam.pod_exec_result import PodExecResult
6
+ from adam.repl_state import ReplState
6
7
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
7
8
  from adam.utils_k8s.pods import log_prefix
9
+ from adam.utils_k8s.statefulsets import StatefulSets
8
10
 
9
11
  class ImportSpec:
10
12
  def __init__(self, session: str, importer: str):
@@ -288,4 +290,25 @@ class GeneratorStream(io.RawIOBase):
288
290
 
289
291
  data = self._buffer[:size]
290
292
  self._buffer = self._buffer[size:]
291
- return data
293
+ return data
294
+
295
+ class PodHandler:
296
+ def __init__(self, state: ReplState):
297
+ self.state = state
298
+
299
+ def __enter__(self):
300
+ state = self.state
301
+
302
+ if not state.pod:
303
+ state.push()
304
+ state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
305
+
306
+ return state
307
+
308
+ def __exit__(self, exc_type, exc_val, exc_tb):
309
+ self.state.pop()
310
+
311
+ return False
312
+
313
+ def state_with_pod(state: ReplState):
314
+ return PodHandler(state)
@@ -0,0 +1,49 @@
1
+ from abc import abstractmethod
2
+
3
+ from adam.commands.command import Command
4
+ from adam.commands.command_helpers import ClusterCommandHelper
5
+ from adam.repl_state import ReplState
6
+ from adam.utils import lines_to_tabular, log, log2
7
+
8
+ class IntermediateCommand(Command):
9
+ def run(self, cmd: str, state: ReplState):
10
+ if not(args := self.args(cmd)):
11
+ return super().run(cmd, state)
12
+
13
+ return self.intermediate_run(cmd, state, args, self.cmd_list())
14
+
15
+ @abstractmethod
16
+ def cmd_list(self):
17
+ pass
18
+
19
+ def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
20
+ state, _ = self.apply_state(args, state)
21
+
22
+ if state.in_repl:
23
+ if display_help:
24
+ log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
25
+
26
+ return 'command-missing'
27
+ else:
28
+ # head with the Chain of Responsibility pattern
29
+ if not self.run_subcommand(cmd, state):
30
+ if display_help:
31
+ log2('* Command is missing.')
32
+ Command.display_help()
33
+ return 'command-missing'
34
+
35
+ return state
36
+
37
+ def run_subcommand(self, cmd: str, state: ReplState):
38
+ cmds = Command.chain(self.cmd_list())
39
+ return cmds.run(cmd, state)
40
+
41
+ def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
42
+ log(super_help)
43
+ log()
44
+ log('Sub-Commands:')
45
+
46
+ log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', ' ', 1) for c in cmd_list], separator=separator))
47
+ if show_cluster_help:
48
+ log()
49
+ ClusterCommandHelper.cluster_help()
adam/commands/issues.py CHANGED
@@ -1,10 +1,9 @@
1
1
  from adam.checks.check_result import CheckResult
2
2
  from adam.checks.check_utils import run_checks
3
- from adam.checks.issue import Issue
3
+ from adam.commands import extract_options
4
4
  from adam.commands.command import Command
5
- from adam.repl_session import ReplSession
6
5
  from adam.repl_state import ReplState
7
- from adam.utils import lines_to_tabular, log, log2
6
+ from adam.utils_issues import IssuesUtils
8
7
 
9
8
  class Issues(Command):
10
9
  COMMAND = 'issues'
@@ -28,48 +27,17 @@ class Issues(Command):
28
27
  if not(args := self.args(cmd)):
29
28
  return super().run(cmd, state)
30
29
 
31
- state, args = self.apply_state(args, state)
32
- if not self.validate_state(state):
33
- return state
30
+ with self.validate(args, state) as (args, state):
31
+ with extract_options(args, ['-s', '--show']) as (args, show_out):
32
+ results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
34
33
 
35
- args, show = Command.extract_options(args, ['-s', '--show'])
34
+ issues = CheckResult.collect_issues(results)
35
+ IssuesUtils.show_issues(issues, in_repl=state.in_repl)
36
36
 
37
- results = run_checks(state.sts, state.namespace, state.pod, show_output=show)
37
+ return issues if issues else 'issues'
38
38
 
39
- issues = CheckResult.collect_issues(results)
40
- Issues.show_issues(issues, in_repl=state.in_repl)
41
-
42
- return issues if issues else 'issues'
43
-
44
- def show(check_results: list[CheckResult], in_repl = False):
45
- Issues.show_issues(CheckResult.collect_issues(check_results), in_repl=in_repl)
46
-
47
- def show_issues(issues: list[Issue], in_repl = False):
48
- if not issues:
49
- log2('No issues found.')
50
- else:
51
- suggested = 0
52
- log2(f'* {len(issues)} issues found.')
53
- lines = []
54
- for i, issue in enumerate(issues, start=1):
55
- lines.append(f"{i}||{issue.category}||{issue.desc}")
56
- lines.append(f"||statefulset||{issue.statefulset}@{issue.namespace}")
57
- lines.append(f"||pod||{issue.pod}@{issue.namespace}")
58
- if issue.details:
59
- lines.append(f"||details||{issue.details}")
60
-
61
- if issue.suggestion:
62
- lines.append(f'||suggestion||{issue.suggestion}')
63
- if in_repl:
64
- ReplSession().prompt_session.history.append_string(issue.suggestion)
65
- suggested += 1
66
- log(lines_to_tabular(lines, separator='||'))
67
- if suggested:
68
- log2()
69
- log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')
70
-
71
- def completion(self, _: ReplState):
72
- return {Issues.COMMAND: None}
39
+ def completion(self, state: ReplState):
40
+ return super().completion(state, {'-s': None})
73
41
 
74
42
  def help(self, _: ReplState):
75
- return f'{Issues.COMMAND}\t find all issues'
43
+ return f'{Issues.COMMAND} [-s]\t find all issues'