kaqing 2.0.115__py3-none-any.whl → 2.0.172__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kaqing might be problematic. Click here for more details.

Files changed (187) hide show
  1. adam/__init__.py +0 -2
  2. adam/app_session.py +8 -11
  3. adam/batch.py +3 -3
  4. adam/checks/check_utils.py +14 -46
  5. adam/checks/cpu.py +7 -1
  6. adam/checks/cpu_metrics.py +52 -0
  7. adam/checks/disk.py +2 -3
  8. adam/columns/columns.py +3 -1
  9. adam/columns/cpu.py +3 -1
  10. adam/columns/cpu_metrics.py +22 -0
  11. adam/columns/memory.py +3 -4
  12. adam/commands/__init__.py +18 -0
  13. adam/commands/alter_tables.py +43 -47
  14. adam/commands/audit/audit.py +24 -25
  15. adam/commands/audit/audit_repair_tables.py +14 -17
  16. adam/commands/audit/audit_run.py +15 -23
  17. adam/commands/audit/show_last10.py +10 -13
  18. adam/commands/audit/show_slow10.py +10 -13
  19. adam/commands/audit/show_top10.py +10 -14
  20. adam/commands/audit/utils_show_top10.py +2 -3
  21. adam/commands/bash/__init__.py +5 -0
  22. adam/commands/bash/bash.py +8 -96
  23. adam/commands/bash/utils_bash.py +16 -0
  24. adam/commands/cat.py +14 -19
  25. adam/commands/cd.py +12 -100
  26. adam/commands/check.py +20 -21
  27. adam/commands/cli_commands.py +2 -3
  28. adam/commands/code.py +20 -23
  29. adam/commands/command.py +123 -39
  30. adam/commands/commands_utils.py +8 -17
  31. adam/commands/cp.py +33 -39
  32. adam/commands/cql/cql_completions.py +28 -10
  33. adam/commands/cql/cqlsh.py +10 -30
  34. adam/commands/cql/utils_cql.py +343 -0
  35. adam/commands/deploy/code_start.py +7 -10
  36. adam/commands/deploy/code_stop.py +4 -21
  37. adam/commands/deploy/code_utils.py +3 -3
  38. adam/commands/deploy/deploy.py +4 -27
  39. adam/commands/deploy/deploy_frontend.py +14 -17
  40. adam/commands/deploy/deploy_pg_agent.py +2 -5
  41. adam/commands/deploy/deploy_pod.py +65 -73
  42. adam/commands/deploy/deploy_utils.py +14 -24
  43. adam/commands/deploy/undeploy.py +4 -27
  44. adam/commands/deploy/undeploy_frontend.py +4 -7
  45. adam/commands/deploy/undeploy_pg_agent.py +5 -7
  46. adam/commands/deploy/undeploy_pod.py +11 -12
  47. adam/commands/devices/__init__.py +0 -0
  48. adam/commands/devices/device.py +118 -0
  49. adam/commands/devices/device_app.py +173 -0
  50. adam/commands/devices/device_auit_log.py +49 -0
  51. adam/commands/devices/device_cass.py +185 -0
  52. adam/commands/devices/device_export.py +86 -0
  53. adam/commands/devices/device_postgres.py +144 -0
  54. adam/commands/devices/devices.py +25 -0
  55. adam/commands/exit.py +1 -4
  56. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  57. adam/commands/export/clean_up_export_sessions.py +51 -0
  58. adam/commands/export/drop_export_database.py +55 -0
  59. adam/commands/export/drop_export_databases.py +43 -0
  60. adam/commands/export/export.py +19 -26
  61. adam/commands/export/export_databases.py +174 -0
  62. adam/commands/export/export_handlers.py +71 -0
  63. adam/commands/export/export_select.py +48 -22
  64. adam/commands/export/export_select_x.py +54 -0
  65. adam/commands/export/export_use.py +19 -23
  66. adam/commands/export/exporter.py +353 -0
  67. adam/commands/export/import_session.py +40 -0
  68. adam/commands/export/importer.py +67 -0
  69. adam/commands/export/importer_athena.py +77 -0
  70. adam/commands/export/importer_sqlite.py +39 -0
  71. adam/commands/export/show_column_counts.py +54 -0
  72. adam/commands/export/show_export_databases.py +36 -0
  73. adam/commands/export/show_export_session.py +48 -0
  74. adam/commands/export/show_export_sessions.py +44 -0
  75. adam/commands/export/utils_export.py +223 -162
  76. adam/commands/help.py +1 -1
  77. adam/commands/intermediate_command.py +49 -0
  78. adam/commands/issues.py +11 -43
  79. adam/commands/kubectl.py +3 -6
  80. adam/commands/login.py +22 -24
  81. adam/commands/logs.py +3 -6
  82. adam/commands/ls.py +11 -128
  83. adam/commands/medusa/medusa.py +4 -22
  84. adam/commands/medusa/medusa_backup.py +20 -24
  85. adam/commands/medusa/medusa_restore.py +29 -33
  86. adam/commands/medusa/medusa_show_backupjobs.py +14 -18
  87. adam/commands/medusa/medusa_show_restorejobs.py +11 -18
  88. adam/commands/nodetool.py +6 -15
  89. adam/commands/param_get.py +11 -12
  90. adam/commands/param_set.py +9 -10
  91. adam/commands/postgres/postgres.py +41 -34
  92. adam/commands/postgres/postgres_context.py +57 -24
  93. adam/commands/postgres/postgres_ls.py +4 -8
  94. adam/commands/postgres/postgres_preview.py +5 -9
  95. adam/commands/postgres/psql_completions.py +1 -1
  96. adam/commands/postgres/utils_postgres.py +66 -0
  97. adam/commands/preview_table.py +5 -44
  98. adam/commands/pwd.py +14 -47
  99. adam/commands/reaper/reaper.py +4 -27
  100. adam/commands/reaper/reaper_forward.py +48 -55
  101. adam/commands/reaper/reaper_forward_session.py +6 -0
  102. adam/commands/reaper/reaper_forward_stop.py +10 -16
  103. adam/commands/reaper/reaper_restart.py +7 -14
  104. adam/commands/reaper/reaper_run_abort.py +11 -30
  105. adam/commands/reaper/reaper_runs.py +42 -57
  106. adam/commands/reaper/reaper_runs_abort.py +29 -49
  107. adam/commands/reaper/reaper_schedule_activate.py +11 -30
  108. adam/commands/reaper/reaper_schedule_start.py +10 -29
  109. adam/commands/reaper/reaper_schedule_stop.py +10 -29
  110. adam/commands/reaper/reaper_schedules.py +4 -14
  111. adam/commands/reaper/reaper_status.py +8 -16
  112. adam/commands/reaper/utils_reaper.py +196 -0
  113. adam/commands/repair/repair.py +4 -22
  114. adam/commands/repair/repair_log.py +5 -11
  115. adam/commands/repair/repair_run.py +27 -34
  116. adam/commands/repair/repair_scan.py +32 -38
  117. adam/commands/repair/repair_stop.py +5 -11
  118. adam/commands/report.py +27 -29
  119. adam/commands/restart.py +25 -26
  120. adam/commands/rollout.py +19 -24
  121. adam/commands/shell.py +10 -4
  122. adam/commands/show/show.py +10 -25
  123. adam/commands/show/show_cassandra_repairs.py +35 -0
  124. adam/commands/show/show_cassandra_status.py +32 -43
  125. adam/commands/show/show_cassandra_version.py +5 -18
  126. adam/commands/show/show_commands.py +19 -24
  127. adam/commands/show/show_host.py +1 -1
  128. adam/commands/show/show_login.py +20 -27
  129. adam/commands/show/show_processes.py +15 -19
  130. adam/commands/show/show_storage.py +10 -20
  131. adam/commands/watch.py +26 -29
  132. adam/config.py +5 -14
  133. adam/embedded_params.py +1 -1
  134. adam/log.py +4 -4
  135. adam/pod_exec_result.py +3 -3
  136. adam/repl.py +40 -103
  137. adam/repl_commands.py +32 -16
  138. adam/repl_state.py +57 -28
  139. adam/sql/sql_completer.py +44 -28
  140. adam/sql/sql_state_machine.py +89 -28
  141. adam/sso/authn_ad.py +6 -8
  142. adam/sso/authn_okta.py +4 -6
  143. adam/sso/cred_cache.py +3 -5
  144. adam/sso/idp.py +9 -12
  145. adam/utils.py +435 -6
  146. adam/utils_athena.py +57 -37
  147. adam/utils_audits.py +12 -14
  148. adam/utils_issues.py +32 -0
  149. adam/utils_k8s/app_clusters.py +13 -18
  150. adam/utils_k8s/app_pods.py +2 -0
  151. adam/utils_k8s/cassandra_clusters.py +22 -19
  152. adam/utils_k8s/cassandra_nodes.py +2 -2
  153. adam/utils_k8s/custom_resources.py +16 -17
  154. adam/utils_k8s/ingresses.py +2 -2
  155. adam/utils_k8s/jobs.py +7 -11
  156. adam/utils_k8s/k8s.py +87 -0
  157. adam/utils_k8s/pods.py +40 -77
  158. adam/utils_k8s/secrets.py +4 -4
  159. adam/utils_k8s/service_accounts.py +5 -4
  160. adam/utils_k8s/services.py +2 -2
  161. adam/utils_k8s/statefulsets.py +1 -12
  162. adam/utils_net.py +4 -4
  163. adam/utils_repl/__init__.py +0 -0
  164. adam/utils_repl/automata_completer.py +48 -0
  165. adam/utils_repl/repl_completer.py +46 -0
  166. adam/utils_repl/state_machine.py +173 -0
  167. adam/utils_sqlite.py +137 -0
  168. adam/version.py +1 -1
  169. {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/METADATA +1 -1
  170. kaqing-2.0.172.dist-info/RECORD +230 -0
  171. adam/commands/app.py +0 -67
  172. adam/commands/app_ping.py +0 -44
  173. adam/commands/cql/cql_utils.py +0 -204
  174. adam/commands/devices.py +0 -147
  175. adam/commands/export/export_on_x.py +0 -76
  176. adam/commands/export/export_rmdbs.py +0 -65
  177. adam/commands/postgres/postgres_utils.py +0 -31
  178. adam/commands/reaper/reaper_session.py +0 -159
  179. adam/commands/show/show_app_actions.py +0 -56
  180. adam/commands/show/show_app_id.py +0 -47
  181. adam/commands/show/show_app_queues.py +0 -45
  182. adam/commands/show/show_repairs.py +0 -47
  183. adam/utils_export.py +0 -42
  184. kaqing-2.0.115.dist-info/RECORD +0 -203
  185. {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/WHEEL +0 -0
  186. {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/entry_points.txt +0 -0
  187. {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,48 @@
1
+ from adam.commands.command import Command
2
+ from adam.commands.export.export_databases import ExportDatabases
3
+ from adam.commands.export.exporter import Exporter
4
+ from adam.repl_state import ReplState, RequiredState
5
+ from adam.utils import log2
6
+
7
+ class ShowExportSession(Command):
8
+ COMMAND = 'show export session'
9
+
10
+ # the singleton pattern
11
+ def __new__(cls, *args, **kwargs):
12
+ if not hasattr(cls, 'instance'): cls.instance = super(ShowExportSession, cls).__new__(cls)
13
+
14
+ return cls.instance
15
+
16
+ def __init__(self, successor: Command=None):
17
+ super().__init__(successor)
18
+
19
+ def command(self):
20
+ return ShowExportSession.COMMAND
21
+
22
+ def required(self):
23
+ return RequiredState.CLUSTER_OR_POD
24
+
25
+ def run(self, cmd: str, state: ReplState):
26
+ if not(args := self.args(cmd)):
27
+ return super().run(cmd, state)
28
+
29
+ with self.validate(args, state) as (args, state):
30
+ if not args:
31
+ if state.in_repl:
32
+ log2('Specify export database name.')
33
+ else:
34
+ log2('* Database name is missing.')
35
+
36
+ Command.display_help()
37
+
38
+ return 'command-missing'
39
+
40
+ ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
41
+
42
+ return state
43
+
44
+ def completion(self, state: ReplState):
45
+ return super().completion(state, {session: None for session in Exporter.export_session_names(state.sts, state.pod, state.namespace)})
46
+
47
+ def help(self, _: ReplState):
48
+ return f'{ShowExportSession.COMMAND} <export-session-name>\t show export session'
@@ -0,0 +1,44 @@
1
+ from adam.commands.command import Command
2
+ from adam.commands.export.exporter import Exporter
3
+ from adam.repl_state import ReplState, RequiredState
4
+ from adam.utils import lines_to_tabular, log
5
+ from adam.utils_k8s.statefulsets import StatefulSets
6
+
7
+ class ShowExportSessions(Command):
8
+ COMMAND = 'show export sessions'
9
+
10
+ # the singleton pattern
11
+ def __new__(cls, *args, **kwargs):
12
+ if not hasattr(cls, 'instance'): cls.instance = super(ShowExportSessions, cls).__new__(cls)
13
+
14
+ return cls.instance
15
+
16
+ def __init__(self, successor: Command=None):
17
+ super().__init__(successor)
18
+
19
+ def command(self):
20
+ return ShowExportSessions.COMMAND
21
+
22
+ def required(self):
23
+ return RequiredState.CLUSTER_OR_POD
24
+
25
+ def run(self, cmd: str, state: ReplState):
26
+ if not(args := self.args(cmd)):
27
+ return super().run(cmd, state)
28
+
29
+ with self.validate(args, state) as (args, state):
30
+ pod = state.pod
31
+ if not pod:
32
+ pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
33
+
34
+ sessions: dict[str, str] = Exporter.find_export_sessions(pod, state.namespace)
35
+ log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
36
+ header='EXPORT_SESSION\tSTATUS', separator='\t'))
37
+
38
+ return state
39
+
40
+ def completion(self, state: ReplState):
41
+ return super().completion(state)
42
+
43
+ def help(self, _: ReplState):
44
+ return f'{ShowExportSessions.COMMAND}\t list export sessions'
@@ -1,54 +1,122 @@
1
- from concurrent.futures import ThreadPoolExecutor, as_completed
2
- from datetime import datetime
3
1
  import io
4
2
  import re
5
- import time
6
- from typing import Callable
7
- import boto3
8
3
 
9
- from adam.commands.cql.cql_utils import cassandra_table_names, run_cql, table_spec
10
4
  from adam.config import Config
5
+ from adam.pod_exec_result import PodExecResult
11
6
  from adam.repl_state import ReplState
12
- from adam.utils import elapsed_time, log2
13
- from adam.utils_athena import Athena
14
7
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
15
- from adam.utils_k8s.pods import Pods
8
+ from adam.utils_k8s.pods import log_prefix
9
+ from adam.utils_k8s.statefulsets import StatefulSets
16
10
 
17
- def export_tables(args: list[str], state: ReplState, max_workers = 0):
18
- consistency = None
19
- specs = None
11
+ class ImportSpec:
12
+ def __init__(self, session: str, importer: str):
13
+ self.session = session
14
+ self.importer = importer
20
15
 
21
- if args:
22
- consistency, specs = ExportSpec.parse_multiple(' '.join(args))
16
+ def parse_specs(specs_str: str):
17
+ session: str = None
18
+ importer: str = None
23
19
 
24
- if not specs:
25
- specs = [ExportSpec(t) for t in cassandra_table_names(state, keyspace=f'{state.namespace}_db')]
20
+ if specs_str:
21
+ importer, session = ImportSpec._extract_importer(specs_str.strip(' '))
26
22
 
27
- if not max_workers:
28
- max_workers = Config().action_workers('export', 8)
23
+ return ImportSpec(session, importer)
29
24
 
30
- if max_workers > 1 and len(specs) > 1:
31
- log2(f'Executing on {len(specs)} Cassandra tables in parallel...')
32
- start_time = time.time()
33
- try:
34
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
35
- futures = [executor.submit(export_table, spec, state, True, consistency=consistency) for spec in specs]
36
- if len(futures) == 0:
37
- return []
38
-
39
- return [future.result() for future in as_completed(futures)]
40
- finally:
41
- log2(f"{len(specs)} parallel table export elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
42
- else:
43
- return [export_table(spec, state, multi_tables=len(specs) > 1, consistency=consistency) for spec in specs]
25
+ def _extract_importer(spec_str: str) -> tuple[str, str]:
26
+ importer = None
27
+ rest = spec_str
28
+
29
+ p = re.compile(r"(.*?)to\s+(.*)", re.IGNORECASE)
30
+ match = p.match(spec_str)
31
+ if match:
32
+ rest = match.group(1).strip(' ')
33
+ importer = match.group(2).strip(' ')
34
+
35
+ return importer, rest
36
+
37
+ class ExportSpec(ImportSpec):
38
+ def __init__(self, keyspace: str, consistency: str, importer: str, tables: list['ExportTableSpec'], session: str = None):
39
+ super().__init__(None, importer)
40
+
41
+ self.keyspace = keyspace
42
+ self.consistency = consistency
43
+ self.tables = tables
44
+ self.session = session
45
+
46
+ def __str__(self):
47
+ return f'keyspace: {self.keyspace}, consistency: {self.consistency}, importer: {self.importer}, tables: {",".join([t.table for t in self.tables])}, session: {self.session}'
48
+
49
+ def __eq__(self, other):
50
+ if not isinstance(other, ExportSpec):
51
+ return NotImplemented
52
+
53
+ return self.keyspace == other.keyspace and self.tables == other.tables and self.consistency == other.consistency and self.importer == other.importer and self.session == other.session
54
+
55
+ def parse_specs(specs_str: str):
56
+ keyspace: str = None
57
+ consistency: str = None
58
+ importer: str = None
59
+ specs: list[ExportTableSpec] = None
60
+
61
+ if specs_str:
62
+ importer, specs_str = ExportSpec._extract_importer(specs_str.strip(' '))
63
+ keyspace, specs_str = ExportSpec._extract_keyspace(specs_str)
64
+ consistency, specs = ExportSpec._extract_consisteny(specs_str)
65
+
66
+ return ExportSpec(keyspace, consistency, importer, specs)
67
+
68
+ def _extract_keyspace(spec_str: str) -> tuple[str, str]:
69
+ keyspace = None
70
+ rest = spec_str
71
+
72
+ p = re.compile(r"\s*\*\s+in\s+(\S+)(.*)", re.IGNORECASE)
73
+ match = p.match(spec_str)
74
+ if match:
75
+ keyspace = match.group(1).strip(' ')
76
+ rest = match.group(2).strip(' ')
77
+ elif spec_str.startswith('*'):
78
+ keyspace = '*'
79
+ rest = spec_str[1:].strip(' ')
80
+
81
+ return keyspace, rest
82
+
83
+ def _extract_consisteny(spec_str: str) -> tuple[str, list['ExportTableSpec']]:
84
+ consistency = None
85
+
86
+ p = re.compile(r"(.*?)with\s+consistency\s+(.*)", re.IGNORECASE)
87
+ match = p.match(spec_str)
88
+ if match:
89
+ spec_str = match.group(1).strip(' ')
90
+ consistency = match.group(2)
91
+
92
+ if spec_str:
93
+ p = r",\s*(?![^()]*\))"
94
+ specs = re.split(p, spec_str)
95
+
96
+ return consistency, [ExportTableSpec.parse(spec) for spec in specs]
97
+
98
+ return consistency, None
44
99
 
45
- class ExportSpec:
46
- def __init__(self, table: str, columns: str = None, target_table: str = None):
100
+ class ExportTableSpec:
101
+ def __init__(self, keyspace: str, table: str, columns: str = None, target_table: str = None):
102
+ self.keyspace = keyspace
47
103
  self.table = table
48
104
  self.columns = columns
49
105
  self.target_table = target_table
50
106
 
51
- def parse(spec_str: str) -> 'ExportSpec':
107
+ def __str__(self):
108
+ return f'{self.keyspace}.{self.table}({self.columns}) AS {self.target_table}'
109
+
110
+ def __eq__(self, other):
111
+ if not isinstance(other, ExportTableSpec):
112
+ return NotImplemented
113
+
114
+ return self.keyspace == other.keyspace and self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
115
+
116
+ def from_status(status: 'ExportTableStatus'):
117
+ return ExportTableSpec(status.keyspace, status.table, target_table=status.target_table)
118
+
119
+ def parse(spec_str: str) -> 'ExportTableSpec':
52
120
  target = None
53
121
 
54
122
  p = re.compile(r"(.*?)\s+as\s+(.*)", re.IGNORECASE)
@@ -57,148 +125,120 @@ class ExportSpec:
57
125
  spec_str = match.group(1)
58
126
  target = match.group(2)
59
127
 
128
+ keyspace = None
60
129
  table = spec_str
61
130
  columns = None
62
131
 
63
- p = re.compile('(.*?)\((.*)\)')
132
+ p = re.compile('(.*?)\.(.*?)\((.*)\)')
64
133
  match = p.match(spec_str)
65
134
  if match:
66
- table = match.group(1)
67
- columns = match.group(2)
135
+ keyspace = match.group(1)
136
+ table = match.group(2)
137
+ columns = match.group(3)
138
+ else:
139
+ p = re.compile('(.*?)\.(.*)')
140
+ match = p.match(spec_str)
141
+ if match:
142
+ keyspace = match.group(1)
143
+ table = match.group(2)
68
144
 
69
- return ExportSpec(table, columns, target)
145
+ return ExportTableSpec(keyspace, table, columns, target)
70
146
 
71
147
  def __eq__(self, other):
72
- if isinstance(other, ExportSpec):
73
- return self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
148
+ if isinstance(other, ExportTableSpec):
149
+ return self.keyspace == other.keyspace and self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
74
150
 
75
151
  return False
76
152
 
77
153
  def __str__(self):
78
- return f'{self.table}, {self.columns}, {self.target_table}'
154
+ return f'{self.keyspace}.{self.table}({self.columns}) as {self.target_table}'
79
155
 
80
- def parse_multiple(spec_str: str) -> tuple[str, list['ExportSpec']]:
81
- consistency = None
156
+ class ExportTableStatus:
157
+ def __init__(self, keyspace: str, target_table: str, status: str, table: str = None):
158
+ self.keyspace = keyspace
159
+ self.target_table = target_table
160
+ self.status = status
161
+ self.table = table
82
162
 
83
- p = re.compile(r"(.*?)with\s+consistency\s+(.*)", re.IGNORECASE)
84
- match = p.match(spec_str)
85
- if match:
86
- spec_str = match.group(1).strip(' ')
87
- consistency = match.group(2)
163
+ def __str__(self):
164
+ return f'{self.keyspace}.{self.table} as {self.target_table} = {self.status}'
88
165
 
89
- if spec_str:
90
- p = r",\s*(?![^()]*\))"
91
- specs = re.split(p, spec_str)
166
+ def __eq__(self, other):
167
+ if isinstance(other, ExportTableStatus):
168
+ return self.keyspace == other.keyspace and self.table == other.table and self.status == other.status and self.target_table == other.target_table
92
169
 
93
- return consistency, [ExportSpec.parse(spec) for spec in specs]
94
-
95
- return consistency, []
96
-
97
- def export_table(spec: ExportSpec, state: ReplState, multi_tables = True, consistency: str = None):
98
- table = spec.table
99
- columns = spec.columns
100
- if not columns:
101
- columns = Config().get('export.columns', f'<keys>')
102
-
103
- if columns == '<keys>':
104
- columns = ','.join(table_spec(state, table, on_any=True).keys())
105
- elif columns == '<row-key>':
106
- columns = table_spec(state, table, on_any=True).row_key()
107
- elif columns == '*':
108
- columns = ','.join([c.name for c in table_spec(state, table, on_any=True).columns])
109
-
110
- if not columns:
111
- log2(f'ERROR: Empty columns on {table}.')
112
- return table
113
-
114
- athena_table = spec.target_table if spec.target_table else table
115
- if '.' in athena_table:
116
- athena_table = athena_table.split('.')[-1]
117
-
118
- temp_dir = Config().get('export.temp_dir', '/c3/cassandra/tmp')
119
- session = state.export_session
120
- create_db = not session
121
- if create_db:
122
- session = datetime.now().strftime("%Y%m%d%H%M%S")
123
- state.export_session = session
124
- db = f'export_{session}'
125
-
126
- CassandraNodes.exec(state.pod, state.namespace, f'mkdir -p {temp_dir}/{session}', show_out=not multi_tables, shell='bash')
127
- csv_file = f'{temp_dir}/{session}/{table}.csv'
128
- succeeded = False
129
- try:
130
- suppress_ing_log = Config().is_debug() or multi_tables
131
- queries = []
132
- if consistency:
133
- queries.append(f'CONSISTENCY {consistency}')
134
- queries.append(f"COPY {table}({columns}) TO '{csv_file}' WITH HEADER = TRUE")
135
- ing(f'Dumping table {table}{f" with consistency {consistency}" if consistency else ""}',
136
- lambda: run_cql(state, ';'.join(queries), show_out=False),
137
- suppress_log=suppress_ing_log)
138
-
139
- def upload_to_s3():
140
- bytes = Pods.read_file(state.pod, 'cassandra', state.namespace, csv_file)
141
-
142
- s3 = boto3.client('s3')
143
- s3.upload_fileobj(GeneratorStream(bytes), 'c3.ops--qing', f'export/{session}/{athena_table}/{table}.csv')
144
-
145
- ing(f'Uploading to S3', upload_to_s3, suppress_log=suppress_ing_log)
146
-
147
- def create_schema():
148
- query = f'CREATE DATABASE IF NOT EXISTS {db};'
149
- if Config().is_debug():
150
- log2(query)
151
- Athena.query(query, 'default')
152
-
153
- query = f'DROP TABLE IF EXISTS {athena_table};'
154
- if Config().is_debug():
155
- log2(query)
156
- Athena.query(query, db)
157
-
158
- # columns = ', '.join([f'{h.strip(" ")} string' for h in header[0].split(',')])
159
- athena_columns = ', '.join([f'{c} string' for c in columns.split(',')])
160
- query = f'CREATE EXTERNAL TABLE IF NOT EXISTS {athena_table}(\n' + \
161
- f' {athena_columns})\n' + \
162
- "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n" + \
163
- 'WITH SERDEPROPERTIES (\n' + \
164
- ' "separatorChar" = ",",\n' + \
165
- ' "quoteChar" = "\\"")\n' + \
166
- f"LOCATION 's3://c3.ops--qing/export/{session}/{athena_table}'\n" + \
167
- 'TBLPROPERTIES ("skip.header.line.count"="1");'
168
- if Config().is_debug():
169
- log2(query)
170
- try:
171
- Athena.query(query, db)
172
- except Exception as e:
173
- log2(f'*** Failed query:\n{query}')
174
- raise e
175
-
176
- ing(f"Creating database {db}" if create_db else f"Creating table {athena_table}", create_schema, suppress_log=suppress_ing_log)
177
-
178
- succeeded = True
179
- except Exception as e:
180
- log2(e)
181
- finally:
182
- ing('Cleaning up temporary files',
183
- lambda: CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_file}', show_out=False, shell='bash'),
184
- suppress_log=suppress_ing_log)
185
-
186
- if succeeded:
187
- Athena.clear_cache()
188
-
189
- if not suppress_ing_log:
190
- query = f'select * from {athena_table} limit 10'
191
- log2(query)
192
- Athena.run_query(query, db)
193
-
194
- return table
195
-
196
- def ing(msg: str, body: Callable[[], None], suppress_log=False):
197
- if not suppress_log:
198
- log2(f'{msg}...', nl=False)
199
- body()
200
- if not suppress_log:
201
- log2(' OK')
170
+ return False
171
+
172
+ def from_session(sts: str, pod: str, namespace: str, export_session: str):
173
+ statuses: list[ExportTableStatus] = []
174
+
175
+ status_in_whole = 'done'
176
+ log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
177
+
178
+ for log_file in log_files:
179
+ status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
180
+ statuses.append(status)
181
+
182
+ if status.status != 'done':
183
+ status_in_whole = status.status
184
+
185
+ return statuses, status_in_whole
186
+
187
+ def from_log_file(pod: str, namespace: str, copy_session: str, log_file: str):
188
+ def get_csv_files_n_table(target_table: str):
189
+ db = f'{copy_session}_{target_table}'
190
+ csv_file = f'{csv_dir()}/{db}/*.csv'
191
+ csv_files: list[str] = find_files(pod, namespace, csv_file)
192
+ if csv_files:
193
+ table = target_table
194
+ m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
195
+ if m:
196
+ table = m.group(1)
197
+ return csv_files, table
198
+
199
+ return csv_files, target_table
200
+
201
+ m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
202
+ if m:
203
+ keyspace = m.group(1)
204
+ target_table = m.group(2)
205
+ state = m.group(3)
206
+ if state == '.pending_import':
207
+ _, table = get_csv_files_n_table(target_table)
208
+ return ExportTableStatus(keyspace, target_table, 'pending_import', table)
209
+ elif state == '.done':
210
+ return ExportTableStatus(keyspace, target_table, 'done', target_table)
211
+
212
+ # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
213
+ pattern = 'rows exported to'
214
+ r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
215
+ if r.exit_code() == 0:
216
+ csv_files, table = get_csv_files_n_table(target_table)
217
+ if csv_files:
218
+ return ExportTableStatus(keyspace, target_table, 'exported', table)
219
+ else:
220
+ return ExportTableStatus(keyspace, target_table, 'imported', target_table)
221
+ else:
222
+ return ExportTableStatus(keyspace, target_table, 'export_in_pregress')
223
+
224
+ return ExportTableStatus(None, None, 'unknown')
225
+
226
+ def csv_dir():
227
+ return Config().get('export.csv_dir', '/c3/cassandra/tmp')
228
+
229
+ def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
230
+ if mmin:
231
+ r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
232
+ else:
233
+ r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
234
+
235
+ log_files = []
236
+ for line in r.stdout.split('\n'):
237
+ line = line.strip(' \r')
238
+ if line:
239
+ log_files.append(line)
240
+
241
+ return log_files
202
242
 
203
243
  class GeneratorStream(io.RawIOBase):
204
244
  def __init__(self, generator):
@@ -250,4 +290,25 @@ class GeneratorStream(io.RawIOBase):
250
290
 
251
291
  data = self._buffer[:size]
252
292
  self._buffer = self._buffer[size:]
253
- return data
293
+ return data
294
+
295
+ class PodHandler:
296
+ def __init__(self, state: ReplState):
297
+ self.state = state
298
+
299
+ def __enter__(self):
300
+ state = self.state
301
+
302
+ if not state.pod:
303
+ state.push()
304
+ state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
305
+
306
+ return state
307
+
308
+ def __exit__(self, exc_type, exc_val, exc_tb):
309
+ self.state.pop()
310
+
311
+ return False
312
+
313
+ def state_with_pod(state: ReplState):
314
+ return PodHandler(state)
adam/commands/help.py CHANGED
@@ -28,7 +28,7 @@ class Help(Command):
28
28
 
29
29
  lines = []
30
30
  lines.append('NAVIGATION')
31
- lines.append(' a: | c: | l: | p:\t switch to another operational device: App, Cassandra, Audit or Postgres')
31
+ lines.append(' a: | c: | l: | p: | x:\t switch to another operational device: App, Cassandra, Audit, Postgres or Export')
32
32
  lines.extend(section(ReplCommands.navigation()))
33
33
  lines.append('CASSANDRA')
34
34
  lines.extend(section(ReplCommands.cassandra_ops()))
@@ -0,0 +1,49 @@
1
+ from abc import abstractmethod
2
+
3
+ from adam.commands.command import Command
4
+ from adam.commands.command_helpers import ClusterCommandHelper
5
+ from adam.repl_state import ReplState
6
+ from adam.utils import lines_to_tabular, log, log2
7
+
8
+ class IntermediateCommand(Command):
9
+ def run(self, cmd: str, state: ReplState):
10
+ if not(args := self.args(cmd)):
11
+ return super().run(cmd, state)
12
+
13
+ return self.intermediate_run(cmd, state, args, self.cmd_list())
14
+
15
+ @abstractmethod
16
+ def cmd_list(self):
17
+ pass
18
+
19
+ def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
20
+ state, _ = self.apply_state(args, state)
21
+
22
+ if state.in_repl:
23
+ if display_help:
24
+ log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
25
+
26
+ return 'command-missing'
27
+ else:
28
+ # head with the Chain of Responsibility pattern
29
+ if not self.run_subcommand(cmd, state):
30
+ if display_help:
31
+ log2('* Command is missing.')
32
+ Command.display_help()
33
+ return 'command-missing'
34
+
35
+ return state
36
+
37
+ def run_subcommand(self, cmd: str, state: ReplState):
38
+ cmds = Command.chain(self.cmd_list())
39
+ return cmds.run(cmd, state)
40
+
41
+ def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
42
+ log(super_help)
43
+ log()
44
+ log('Sub-Commands:')
45
+
46
+ log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', ' ', 1) for c in cmd_list], separator=separator))
47
+ if show_cluster_help:
48
+ log()
49
+ ClusterCommandHelper.cluster_help()
adam/commands/issues.py CHANGED
@@ -1,10 +1,9 @@
1
1
  from adam.checks.check_result import CheckResult
2
2
  from adam.checks.check_utils import run_checks
3
- from adam.checks.issue import Issue
3
+ from adam.commands import extract_options
4
4
  from adam.commands.command import Command
5
- from adam.repl_session import ReplSession
6
5
  from adam.repl_state import ReplState
7
- from adam.utils import lines_to_tabular, log, log2
6
+ from adam.utils_issues import IssuesUtils
8
7
 
9
8
  class Issues(Command):
10
9
  COMMAND = 'issues'
@@ -28,48 +27,17 @@ class Issues(Command):
28
27
  if not(args := self.args(cmd)):
29
28
  return super().run(cmd, state)
30
29
 
31
- state, args = self.apply_state(args, state)
32
- if not self.validate_state(state):
33
- return state
30
+ with self.validate(args, state) as (args, state):
31
+ with extract_options(args, ['-s', '--show']) as (args, show_out):
32
+ results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
34
33
 
35
- args, show = Command.extract_options(args, ['-s', '--show'])
34
+ issues = CheckResult.collect_issues(results)
35
+ IssuesUtils.show_issues(issues, in_repl=state.in_repl)
36
36
 
37
- results = run_checks(state.sts, state.namespace, state.pod, show_output=show)
37
+ return issues if issues else 'issues'
38
38
 
39
- issues = CheckResult.collect_issues(results)
40
- Issues.show_issues(issues, in_repl=state.in_repl)
41
-
42
- return issues if issues else 'issues'
43
-
44
- def show(check_results: list[CheckResult], in_repl = False):
45
- Issues.show_issues(CheckResult.collect_issues(check_results), in_repl=in_repl)
46
-
47
- def show_issues(issues: list[Issue], in_repl = False):
48
- if not issues:
49
- log2('No issues found.')
50
- else:
51
- suggested = 0
52
- log2(f'* {len(issues)} issues found.')
53
- lines = []
54
- for i, issue in enumerate(issues, start=1):
55
- lines.append(f"{i}||{issue.category}||{issue.desc}")
56
- lines.append(f"||statefulset||{issue.statefulset}@{issue.namespace}")
57
- lines.append(f"||pod||{issue.pod}@{issue.namespace}")
58
- if issue.details:
59
- lines.append(f"||details||{issue.details}")
60
-
61
- if issue.suggestion:
62
- lines.append(f'||suggestion||{issue.suggestion}')
63
- if in_repl:
64
- ReplSession().prompt_session.history.append_string(issue.suggestion)
65
- suggested += 1
66
- log(lines_to_tabular(lines, separator='||'))
67
- if suggested:
68
- log2()
69
- log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')
70
-
71
- def completion(self, _: ReplState):
72
- return {Issues.COMMAND: None}
39
+ def completion(self, state: ReplState):
40
+ return super().completion(state, {'-s': None})
73
41
 
74
42
  def help(self, _: ReplState):
75
- return f'{Issues.COMMAND}\t find all issues'
43
+ return f'{Issues.COMMAND} [-s]\t find all issues'