kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. adam/__init__.py +1 -0
  2. adam/app_session.py +182 -0
  3. {walker → adam}/apps.py +8 -24
  4. {walker → adam}/batch.py +54 -97
  5. {walker → adam}/checks/check.py +3 -3
  6. {walker → adam}/checks/check_result.py +1 -1
  7. adam/checks/check_utils.py +65 -0
  8. {walker → adam}/checks/compactionstats.py +6 -6
  9. {walker → adam}/checks/cpu.py +14 -8
  10. adam/checks/cpu_metrics.py +52 -0
  11. {walker → adam}/checks/disk.py +6 -6
  12. {walker → adam}/checks/gossip.py +5 -5
  13. {walker → adam}/checks/memory.py +7 -7
  14. {walker → adam}/checks/status.py +5 -5
  15. {walker → adam}/cli.py +3 -3
  16. {walker → adam}/columns/column.py +1 -1
  17. adam/columns/columns.py +45 -0
  18. {walker → adam}/columns/compactions.py +5 -5
  19. {walker → adam}/columns/cpu.py +6 -4
  20. adam/columns/cpu_metrics.py +22 -0
  21. {walker → adam}/columns/dir_data.py +3 -3
  22. {walker → adam}/columns/dir_snapshots.py +3 -3
  23. {walker → adam}/columns/gossip.py +5 -5
  24. {walker → adam}/columns/host_id.py +3 -3
  25. {walker → adam}/columns/memory.py +3 -3
  26. {walker → adam}/columns/node_address.py +3 -3
  27. {walker → adam}/columns/node_load.py +3 -3
  28. {walker → adam}/columns/node_owns.py +3 -3
  29. {walker → adam}/columns/node_status.py +3 -3
  30. {walker → adam}/columns/node_tokens.py +3 -3
  31. {walker → adam}/columns/node_utils.py +2 -2
  32. {walker → adam}/columns/pod_name.py +2 -2
  33. {walker → adam}/columns/volume_cassandra.py +4 -4
  34. {walker → adam}/columns/volume_root.py +3 -3
  35. adam/commands/__init__.py +15 -0
  36. adam/commands/alter_tables.py +81 -0
  37. adam/commands/app_cmd.py +38 -0
  38. {walker → adam}/commands/app_ping.py +10 -16
  39. adam/commands/audit/audit.py +84 -0
  40. adam/commands/audit/audit_repair_tables.py +74 -0
  41. adam/commands/audit/audit_run.py +50 -0
  42. adam/commands/audit/show_last10.py +48 -0
  43. adam/commands/audit/show_slow10.py +47 -0
  44. adam/commands/audit/show_top10.py +45 -0
  45. adam/commands/audit/utils_show_top10.py +59 -0
  46. adam/commands/bash/__init__.py +5 -0
  47. adam/commands/bash/bash.py +36 -0
  48. adam/commands/bash/bash_completer.py +93 -0
  49. adam/commands/bash/utils_bash.py +16 -0
  50. adam/commands/cat.py +50 -0
  51. adam/commands/cd.py +43 -0
  52. adam/commands/check.py +73 -0
  53. {walker → adam}/commands/cli_commands.py +7 -8
  54. adam/commands/code.py +57 -0
  55. adam/commands/command.py +190 -0
  56. {walker → adam}/commands/command_helpers.py +1 -1
  57. {walker → adam}/commands/commands_utils.py +15 -25
  58. adam/commands/cp.py +89 -0
  59. adam/commands/cql/cql_completions.py +33 -0
  60. {walker/commands → adam/commands/cql}/cqlsh.py +20 -35
  61. adam/commands/cql/utils_cql.py +343 -0
  62. {walker/commands/frontend → adam/commands/deploy}/code_start.py +11 -14
  63. adam/commands/deploy/code_stop.py +40 -0
  64. {walker/commands/frontend → adam/commands/deploy}/code_utils.py +7 -9
  65. adam/commands/deploy/deploy.py +25 -0
  66. adam/commands/deploy/deploy_frontend.py +49 -0
  67. adam/commands/deploy/deploy_pg_agent.py +35 -0
  68. adam/commands/deploy/deploy_pod.py +108 -0
  69. adam/commands/deploy/deploy_utils.py +29 -0
  70. adam/commands/deploy/undeploy.py +25 -0
  71. adam/commands/deploy/undeploy_frontend.py +38 -0
  72. adam/commands/deploy/undeploy_pg_agent.py +39 -0
  73. adam/commands/deploy/undeploy_pod.py +48 -0
  74. adam/commands/devices/device.py +118 -0
  75. adam/commands/devices/device_app.py +173 -0
  76. adam/commands/devices/device_auit_log.py +49 -0
  77. adam/commands/devices/device_cass.py +185 -0
  78. adam/commands/devices/device_export.py +86 -0
  79. adam/commands/devices/device_postgres.py +144 -0
  80. adam/commands/devices/devices.py +25 -0
  81. {walker → adam}/commands/exit.py +3 -6
  82. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  83. adam/commands/export/clean_up_export_sessions.py +51 -0
  84. adam/commands/export/drop_export_database.py +55 -0
  85. adam/commands/export/drop_export_databases.py +43 -0
  86. adam/commands/export/export.py +53 -0
  87. adam/commands/export/export_databases.py +170 -0
  88. adam/commands/export/export_handlers.py +71 -0
  89. adam/commands/export/export_select.py +81 -0
  90. adam/commands/export/export_select_x.py +54 -0
  91. adam/commands/export/export_use.py +52 -0
  92. adam/commands/export/exporter.py +352 -0
  93. adam/commands/export/import_session.py +40 -0
  94. adam/commands/export/importer.py +67 -0
  95. adam/commands/export/importer_athena.py +80 -0
  96. adam/commands/export/importer_sqlite.py +47 -0
  97. adam/commands/export/show_column_counts.py +54 -0
  98. adam/commands/export/show_export_databases.py +36 -0
  99. adam/commands/export/show_export_session.py +48 -0
  100. adam/commands/export/show_export_sessions.py +44 -0
  101. adam/commands/export/utils_export.py +314 -0
  102. {walker → adam}/commands/help.py +17 -12
  103. adam/commands/intermediate_command.py +49 -0
  104. adam/commands/issues.py +43 -0
  105. adam/commands/kubectl.py +38 -0
  106. adam/commands/login.py +70 -0
  107. {walker → adam}/commands/logs.py +8 -10
  108. adam/commands/ls.py +41 -0
  109. adam/commands/medusa/medusa.py +27 -0
  110. adam/commands/medusa/medusa_backup.py +57 -0
  111. adam/commands/medusa/medusa_restore.py +83 -0
  112. adam/commands/medusa/medusa_show_backupjobs.py +51 -0
  113. adam/commands/medusa/medusa_show_restorejobs.py +47 -0
  114. {walker → adam}/commands/nodetool.py +17 -21
  115. {walker → adam}/commands/param_get.py +15 -16
  116. adam/commands/param_set.py +43 -0
  117. adam/commands/postgres/postgres.py +104 -0
  118. adam/commands/postgres/postgres_context.py +274 -0
  119. {walker → adam}/commands/postgres/postgres_ls.py +7 -11
  120. {walker → adam}/commands/postgres/postgres_preview.py +8 -13
  121. adam/commands/postgres/psql_completions.py +10 -0
  122. adam/commands/postgres/utils_postgres.py +66 -0
  123. adam/commands/preview_table.py +37 -0
  124. adam/commands/pwd.py +47 -0
  125. adam/commands/reaper/reaper.py +35 -0
  126. adam/commands/reaper/reaper_forward.py +93 -0
  127. adam/commands/reaper/reaper_forward_session.py +6 -0
  128. {walker → adam}/commands/reaper/reaper_forward_stop.py +13 -19
  129. {walker → adam}/commands/reaper/reaper_restart.py +10 -17
  130. adam/commands/reaper/reaper_run_abort.py +46 -0
  131. adam/commands/reaper/reaper_runs.py +82 -0
  132. adam/commands/reaper/reaper_runs_abort.py +63 -0
  133. adam/commands/reaper/reaper_schedule_activate.py +45 -0
  134. adam/commands/reaper/reaper_schedule_start.py +45 -0
  135. adam/commands/reaper/reaper_schedule_stop.py +45 -0
  136. {walker → adam}/commands/reaper/reaper_schedules.py +6 -16
  137. {walker → adam}/commands/reaper/reaper_status.py +11 -19
  138. adam/commands/reaper/utils_reaper.py +196 -0
  139. adam/commands/repair/repair.py +26 -0
  140. {walker → adam}/commands/repair/repair_log.py +7 -10
  141. adam/commands/repair/repair_run.py +70 -0
  142. adam/commands/repair/repair_scan.py +71 -0
  143. {walker → adam}/commands/repair/repair_stop.py +8 -11
  144. adam/commands/report.py +61 -0
  145. adam/commands/restart.py +60 -0
  146. {walker → adam}/commands/rollout.py +25 -30
  147. adam/commands/shell.py +34 -0
  148. adam/commands/show/show.py +39 -0
  149. walker/commands/show/show_version.py → adam/commands/show/show_adam.py +14 -10
  150. adam/commands/show/show_app_actions.py +57 -0
  151. {walker → adam}/commands/show/show_app_id.py +12 -15
  152. {walker → adam}/commands/show/show_app_queues.py +9 -12
  153. adam/commands/show/show_cassandra_repairs.py +38 -0
  154. adam/commands/show/show_cassandra_status.py +124 -0
  155. {walker → adam}/commands/show/show_cassandra_version.py +6 -16
  156. adam/commands/show/show_commands.py +59 -0
  157. walker/commands/show/show_storage.py → adam/commands/show/show_host.py +11 -13
  158. adam/commands/show/show_login.py +62 -0
  159. {walker → adam}/commands/show/show_params.py +4 -4
  160. adam/commands/show/show_processes.py +51 -0
  161. adam/commands/show/show_storage.py +42 -0
  162. adam/commands/watch.py +82 -0
  163. {walker → adam}/config.py +10 -22
  164. {walker → adam}/embedded_apps.py +1 -1
  165. adam/embedded_params.py +2 -0
  166. adam/log.py +47 -0
  167. {walker → adam}/pod_exec_result.py +10 -2
  168. adam/repl.py +182 -0
  169. adam/repl_commands.py +124 -0
  170. adam/repl_state.py +458 -0
  171. adam/sql/__init__.py +0 -0
  172. adam/sql/sql_completer.py +120 -0
  173. adam/sql/sql_state_machine.py +618 -0
  174. adam/sql/term_completer.py +76 -0
  175. adam/sso/__init__.py +0 -0
  176. {walker → adam}/sso/authenticator.py +5 -1
  177. adam/sso/authn_ad.py +170 -0
  178. {walker → adam}/sso/authn_okta.py +39 -22
  179. adam/sso/cred_cache.py +60 -0
  180. adam/sso/id_token.py +23 -0
  181. adam/sso/idp.py +143 -0
  182. adam/sso/idp_login.py +50 -0
  183. adam/sso/idp_session.py +55 -0
  184. adam/sso/sso_config.py +63 -0
  185. adam/utils.py +679 -0
  186. adam/utils_app.py +98 -0
  187. adam/utils_athena.py +145 -0
  188. adam/utils_audits.py +106 -0
  189. adam/utils_issues.py +32 -0
  190. adam/utils_k8s/__init__.py +0 -0
  191. adam/utils_k8s/app_clusters.py +28 -0
  192. adam/utils_k8s/app_pods.py +33 -0
  193. adam/utils_k8s/cassandra_clusters.py +36 -0
  194. adam/utils_k8s/cassandra_nodes.py +33 -0
  195. adam/utils_k8s/config_maps.py +34 -0
  196. {walker/k8s_utils → adam/utils_k8s}/custom_resources.py +7 -2
  197. adam/utils_k8s/deployment.py +56 -0
  198. {walker/k8s_utils → adam/utils_k8s}/ingresses.py +3 -4
  199. {walker/k8s_utils → adam/utils_k8s}/jobs.py +3 -3
  200. adam/utils_k8s/k8s.py +87 -0
  201. {walker/k8s_utils → adam/utils_k8s}/kube_context.py +4 -4
  202. adam/utils_k8s/pods.py +290 -0
  203. {walker/k8s_utils → adam/utils_k8s}/secrets.py +8 -4
  204. adam/utils_k8s/service_accounts.py +170 -0
  205. {walker/k8s_utils → adam/utils_k8s}/services.py +3 -4
  206. {walker/k8s_utils → adam/utils_k8s}/statefulsets.py +6 -16
  207. {walker/k8s_utils → adam/utils_k8s}/volumes.py +10 -1
  208. adam/utils_net.py +24 -0
  209. adam/utils_repl/__init__.py +0 -0
  210. adam/utils_repl/automata_completer.py +48 -0
  211. adam/utils_repl/repl_completer.py +46 -0
  212. adam/utils_repl/state_machine.py +173 -0
  213. adam/utils_sqlite.py +109 -0
  214. adam/version.py +5 -0
  215. {kaqing-1.77.0.dist-info → kaqing-2.0.171.dist-info}/METADATA +1 -1
  216. kaqing-2.0.171.dist-info/RECORD +236 -0
  217. kaqing-2.0.171.dist-info/entry_points.txt +3 -0
  218. kaqing-2.0.171.dist-info/top_level.txt +1 -0
  219. kaqing-1.77.0.dist-info/RECORD +0 -159
  220. kaqing-1.77.0.dist-info/entry_points.txt +0 -3
  221. kaqing-1.77.0.dist-info/top_level.txt +0 -1
  222. walker/__init__.py +0 -3
  223. walker/app_session.py +0 -168
  224. walker/checks/check_utils.py +0 -97
  225. walker/columns/columns.py +0 -43
  226. walker/commands/add_user.py +0 -68
  227. walker/commands/app.py +0 -67
  228. walker/commands/bash.py +0 -87
  229. walker/commands/cd.py +0 -115
  230. walker/commands/check.py +0 -68
  231. walker/commands/command.py +0 -104
  232. walker/commands/cp.py +0 -95
  233. walker/commands/cql_utils.py +0 -53
  234. walker/commands/devices.py +0 -89
  235. walker/commands/frontend/code_stop.py +0 -57
  236. walker/commands/frontend/setup.py +0 -60
  237. walker/commands/frontend/setup_frontend.py +0 -58
  238. walker/commands/frontend/teardown.py +0 -61
  239. walker/commands/frontend/teardown_frontend.py +0 -42
  240. walker/commands/issues.py +0 -69
  241. walker/commands/login.py +0 -72
  242. walker/commands/ls.py +0 -145
  243. walker/commands/medusa/medusa.py +0 -69
  244. walker/commands/medusa/medusa_backup.py +0 -61
  245. walker/commands/medusa/medusa_restore.py +0 -86
  246. walker/commands/medusa/medusa_show_backupjobs.py +0 -52
  247. walker/commands/medusa/medusa_show_restorejobs.py +0 -52
  248. walker/commands/param_set.py +0 -44
  249. walker/commands/postgres/postgres.py +0 -113
  250. walker/commands/postgres/postgres_session.py +0 -225
  251. walker/commands/preview_table.py +0 -98
  252. walker/commands/processes.py +0 -53
  253. walker/commands/pwd.py +0 -64
  254. walker/commands/reaper/reaper.py +0 -78
  255. walker/commands/reaper/reaper_forward.py +0 -100
  256. walker/commands/reaper/reaper_run_abort.py +0 -65
  257. walker/commands/reaper/reaper_runs.py +0 -97
  258. walker/commands/reaper/reaper_runs_abort.py +0 -83
  259. walker/commands/reaper/reaper_schedule_activate.py +0 -64
  260. walker/commands/reaper/reaper_schedule_start.py +0 -64
  261. walker/commands/reaper/reaper_schedule_stop.py +0 -64
  262. walker/commands/reaper/reaper_session.py +0 -159
  263. walker/commands/repair/repair.py +0 -68
  264. walker/commands/repair/repair_run.py +0 -72
  265. walker/commands/repair/repair_scan.py +0 -79
  266. walker/commands/report.py +0 -57
  267. walker/commands/restart.py +0 -61
  268. walker/commands/show/show.py +0 -72
  269. walker/commands/show/show_app_actions.py +0 -53
  270. walker/commands/show/show_cassandra_status.py +0 -35
  271. walker/commands/show/show_commands.py +0 -58
  272. walker/commands/show/show_processes.py +0 -35
  273. walker/commands/show/show_repairs.py +0 -47
  274. walker/commands/status.py +0 -128
  275. walker/commands/storage.py +0 -52
  276. walker/commands/user_entry.py +0 -69
  277. walker/commands/watch.py +0 -85
  278. walker/embedded_params.py +0 -2
  279. walker/k8s_utils/cassandra_clusters.py +0 -48
  280. walker/k8s_utils/cassandra_nodes.py +0 -26
  281. walker/k8s_utils/pods.py +0 -211
  282. walker/repl.py +0 -165
  283. walker/repl_commands.py +0 -58
  284. walker/repl_state.py +0 -211
  285. walker/sso/authn_ad.py +0 -94
  286. walker/sso/idp.py +0 -150
  287. walker/sso/idp_login.py +0 -29
  288. walker/sso/sso_config.py +0 -45
  289. walker/utils.py +0 -194
  290. walker/version.py +0 -5
  291. {walker → adam}/checks/__init__.py +0 -0
  292. {walker → adam}/checks/check_context.py +0 -0
  293. {walker → adam}/checks/issue.py +0 -0
  294. {walker → adam}/cli_group.py +0 -0
  295. {walker → adam}/columns/__init__.py +0 -0
  296. {walker/commands → adam/commands/audit}/__init__.py +0 -0
  297. {walker/commands/frontend → adam/commands/cql}/__init__.py +0 -0
  298. {walker/commands/medusa → adam/commands/deploy}/__init__.py +0 -0
  299. {walker/commands/postgres → adam/commands/devices}/__init__.py +0 -0
  300. {walker/commands/reaper → adam/commands/export}/__init__.py +0 -0
  301. {walker/commands/repair → adam/commands/medusa}/__init__.py +0 -0
  302. {walker → adam}/commands/nodetool_commands.py +0 -0
  303. {walker/commands/show → adam/commands/postgres}/__init__.py +0 -0
  304. {walker/k8s_utils → adam/commands/reaper}/__init__.py +0 -0
  305. {walker/sso → adam/commands/repair}/__init__.py +0 -0
  306. /walker/medusa_show_restorejobs.py → /adam/commands/show/__init__.py +0 -0
  307. {walker → adam}/repl_session.py +0 -0
  308. {kaqing-1.77.0.dist-info → kaqing-2.0.171.dist-info}/WHEEL +0 -0
adam/utils_app.py ADDED
@@ -0,0 +1,98 @@
1
+ import json
2
+ from typing import Union
3
+
4
+ from adam.app_session import AppSession
5
+ from adam.apps import Apps
6
+ from adam.pod_exec_result import PodExecResult
7
+ from adam.repl_state import ReplState
8
+ from adam.utils import log2
9
+ from adam.utils_k8s.app_clusters import AppClusters
10
+ from adam.utils_k8s.app_pods import AppPods
11
+
12
+ class AppRestHandler:
13
+ def __init__(self, state: ReplState, forced = False):
14
+ self.state = state
15
+ self.forced = forced
16
+
17
+ def __enter__(self):
18
+ return self.post
19
+
20
+ def __exit__(self, exc_type, exc_val, exc_tb):
21
+ return False
22
+
23
+ def post(self, args: list[str]) -> Union[ReplState, str]:
24
+ if not args:
25
+ return 'arg missing'
26
+
27
+ t_f = args[0].split('.')
28
+ if len(t_f) < 2:
29
+ return 'arg missing'
30
+
31
+ state = self.state
32
+
33
+ payload, valid = Apps().payload(t_f[0], t_f[1], args[1:] if len(args) > 1 else [])
34
+ if not valid:
35
+ log2('Missing one or more action arguments.')
36
+ return state
37
+
38
+ if payload:
39
+ try:
40
+ payload = json.loads(payload)
41
+ except json.decoder.JSONDecodeError as e:
42
+ log2(f'Invalid json argument: {e}')
43
+ return state
44
+
45
+ AppSession.run(state.app_env, state.app_app, state.namespace, t_f[0], t_f[1], payload=payload, forced=self.forced)
46
+
47
+ return state
48
+
49
+ class AppPodService:
50
+ def __init__(self, handler: 'AppHandler'):
51
+ self.handler = handler
52
+
53
+ def exec(self, command: str, show_out = True) -> Union[PodExecResult, list[PodExecResult]]:
54
+ state = self.handler.state
55
+
56
+ if state.app_pod:
57
+ return AppPods.exec(state.app_pod, state.namespace, command, show_out=show_out, shell='bash')
58
+ elif state.app_app:
59
+ pods = AppPods.pod_names(state.namespace, state.app_env, state.app_app)
60
+ return AppClusters.exec(pods, state.namespace, command, action='bash', show_out=show_out, shell='bash')
61
+
62
+ return []
63
+
64
+ def post(self, args: list[str], forced=False) -> Union[ReplState, str]:
65
+ state = self.handler.state
66
+
67
+ if not args:
68
+ return 'arg missing'
69
+
70
+ t_f = args[0].split('.')
71
+ if len(t_f) < 2:
72
+ return 'arg missing'
73
+
74
+ payload, valid = Apps().payload(t_f[0], t_f[1], args[1:] if len(args) > 1 else [])
75
+ if not valid:
76
+ log2('Missing one or more action arguments.')
77
+ return state
78
+
79
+ if payload:
80
+ try:
81
+ payload = json.loads(payload)
82
+ except json.decoder.JSONDecodeError as e:
83
+ log2(f'Invalid json argument: {e}')
84
+ return state
85
+
86
+ AppSession.run(state.app_env, state.app_app, state.namespace, t_f[0], t_f[1], payload=payload, forced=forced)
87
+
88
+ return state
89
+
90
+ class AppHandler:
91
+ def __init__(self, state: ReplState):
92
+ self.state = state
93
+
94
+ def __enter__(self):
95
+ return AppPodService(self)
96
+
97
+ def __exit__(self, exc_type, exc_val, exc_tb):
98
+ return False
adam/utils_athena.py ADDED
@@ -0,0 +1,145 @@
1
+ import functools
2
+ import time
3
+ import boto3
4
+ import botocore
5
+
6
+ from adam.config import Config
7
+ from adam.utils import lines_to_tabular, log, log2, wait_log
8
+
9
+ # no state utility class
10
+ class Athena:
11
+ @functools.lru_cache()
12
+ def database_names(like: str = None):
13
+ # this function is called only from export currently
14
+ wait_log(f'Inspecting export database schema...')
15
+
16
+ query = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name <> 'information_schema'"
17
+ if like:
18
+ query = f"{query} AND schema_name like '{like}'"
19
+
20
+ try:
21
+ state, reason, rs = Athena.query(query)
22
+ if rs:
23
+ names = []
24
+ for row in rs[1:]:
25
+ row_data = [col.get('VarCharValue') if col else '' for col in row['Data']]
26
+ names.append(row_data[0])
27
+
28
+ return names
29
+ except:
30
+ pass
31
+
32
+ return []
33
+
34
+ def clear_cache(cache: str = None):
35
+ if not cache or cache == 'databases':
36
+ Athena.database_names.cache_clear()
37
+ if not cache or cache == 'tables':
38
+ Athena.table_names.cache_clear()
39
+ if not cache or cache == 'columns':
40
+ Athena.column_names.cache_clear()
41
+
42
+ @functools.lru_cache()
43
+ def table_names(database: str = 'audit', function: str = 'audit'):
44
+ table_names = []
45
+ try:
46
+ region_name = Config().get(f'{function}.athena.region', 'us-west-2')
47
+ database_name = Config().get(f'{function}.athena.database', database)
48
+ catalog_name = Config().get(f'{function}.athena.catalog', 'AwsDataCatalog')
49
+
50
+ athena_client = boto3.client('athena', region_name=region_name)
51
+ paginator = athena_client.get_paginator('list_table_metadata')
52
+
53
+ for page in paginator.paginate(CatalogName=catalog_name, DatabaseName=database_name):
54
+ for table_metadata in page.get('TableMetadataList', []):
55
+ table_names.append(table_metadata['Name'])
56
+ except botocore.exceptions.NoCredentialsError as e:
57
+ # aws credentials not found
58
+ if function == 'audit':
59
+ log2(f'Please configure AWS credentials to Audit Log Database.')
60
+ except:
61
+ pass
62
+
63
+ return table_names
64
+
65
+ @functools.lru_cache()
66
+ def column_names(tables: list[str] = [], database: str = None, function: str = 'audit', partition_cols_only = False):
67
+ try:
68
+ if not database:
69
+ database = Config().get(f'{function}.athena.database', 'audit')
70
+
71
+ if not tables:
72
+ tables = Config().get(f'{function}.athena.tables', 'audit').split(',')
73
+
74
+ table_names = "'" + "','".join([table.strip() for table in tables]) + "'"
75
+
76
+ query = f"select column_name from information_schema.columns where table_name in ({table_names}) and table_schema = '{database}'"
77
+ if partition_cols_only:
78
+ query = f"{query} and extra_info = 'partition key'"
79
+
80
+ _, _, rs = Athena.query(query)
81
+ if rs:
82
+ return [row['Data'][0].get('VarCharValue') for row in rs[1:]]
83
+ except:
84
+ # aws credentials not found
85
+ pass
86
+
87
+ return []
88
+
89
+ def run_query(sql: str, database: str = None):
90
+ state, reason, rs = Athena.query(sql, database)
91
+
92
+ if state == 'SUCCEEDED':
93
+ if rs:
94
+ column_info = rs[0]['Data']
95
+ columns = [col.get('VarCharValue') for col in column_info]
96
+ lines = []
97
+ for row in rs[1:]:
98
+ row_data = [col.get('VarCharValue') if col else '' for col in row['Data']]
99
+ lines.append('\t'.join(row_data))
100
+
101
+ log(lines_to_tabular(lines, header='\t'.join(columns), separator='\t'))
102
+
103
+ return len(lines)
104
+ else:
105
+ log2(f"Query failed or was cancelled. State: {state}")
106
+ log2(f"Reason: {reason}")
107
+
108
+ return 0
109
+
110
+ def query(sql: str, database: str = None, function: str = 'audit') -> tuple[str, str, list]:
111
+ region_name = Config().get(f'{function}.athena.region', 'us-west-2')
112
+ athena_client = boto3.client('athena', region_name=region_name)
113
+
114
+ if not database:
115
+ database = Config().get(f'{function}.athena.database', 'audit')
116
+
117
+ s3_output_location = Config().get(f'{function}.athena.output', f's3://s3.ops--{function}/ddl/results')
118
+
119
+ response = athena_client.start_query_execution(
120
+ QueryString=sql,
121
+ QueryExecutionContext={
122
+ 'Database': database
123
+ },
124
+ ResultConfiguration={
125
+ 'OutputLocation': s3_output_location
126
+ }
127
+ )
128
+
129
+ query_execution_id = response['QueryExecutionId']
130
+
131
+ while True:
132
+ query_status = athena_client.get_query_execution(QueryExecutionId=query_execution_id)
133
+ state = query_status['QueryExecution']['Status']['State']
134
+ if state in ['SUCCEEDED', 'FAILED', 'CANCELLED']:
135
+ break
136
+ time.sleep(1)
137
+
138
+ if state == 'SUCCEEDED':
139
+ results_response = athena_client.get_query_results(QueryExecutionId=query_execution_id)
140
+ if results_response['ResultSet']['Rows']:
141
+ return (state, None, results_response['ResultSet']['Rows'])
142
+
143
+ return (state, None, [])
144
+ else:
145
+ return (state, query_status['QueryExecution']['Status'].get('StateChangeReason'), [])
adam/utils_audits.py ADDED
@@ -0,0 +1,106 @@
1
+ from datetime import datetime
2
+ import getpass
3
+ import time
4
+ import requests
5
+
6
+ from adam.config import Config
7
+ from adam.utils import OffloadHandler, debug, log2, offload
8
+ from adam.utils_athena import Athena
9
+ from adam.utils_net import get_my_host
10
+
11
+ class AuditMeta:
12
+ def __init__(self, partitions_last_checked: float, cluster_last_checked: float):
13
+ self.partitions_last_checked = partitions_last_checked
14
+ self.cluster_last_checked = cluster_last_checked
15
+
16
+ # no state utility class
17
+ class Audits:
18
+ PARTITIONS_ADDED = 'partitions-added'
19
+ ADD_CLUSTERS = 'add-clusters'
20
+
21
+ def log(cmd: str, cluster = 'NA', drive: str = 'NA', duration: float = 0.0, audit_extra = None):
22
+ payload = {
23
+ 'cluster': cluster if cluster else 'NA',
24
+ 'ts': time.time(),
25
+ 'host': get_my_host(),
26
+ 'user': getpass.getuser(),
27
+ 'line': cmd.replace('"', '""').replace('\n', ' '),
28
+ 'drive': drive,
29
+ 'duration': duration,
30
+ 'audit_extra': audit_extra if audit_extra else '',
31
+ }
32
+ audit_endpoint = Config().get("audit.endpoint", "https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/")
33
+ try:
34
+ response = requests.post(audit_endpoint, json=payload, timeout=Config().get("audit.timeout", 10))
35
+ if response.status_code in [200, 201]:
36
+ debug(response.text)
37
+ else:
38
+ log2(f"Error: {response.status_code} {response.text}")
39
+ except requests.exceptions.Timeout as e:
40
+ log2(f"Timeout occurred: {e}")
41
+
42
+ def get_meta() -> AuditMeta:
43
+ checked_in = 0.0
44
+ cluster_last_checked = 0.0
45
+
46
+ state, _, rs = Athena.query(f'select partitions_last_checked, clusters_last_checked from meta')
47
+ if state == 'SUCCEEDED':
48
+ if len(rs) > 1:
49
+ try:
50
+ row = rs[1]['Data']
51
+ checked_in = float(row[0]['VarCharValue'])
52
+ cluster_last_checked = float(row[1]['VarCharValue'])
53
+ except:
54
+ pass
55
+
56
+ return AuditMeta(checked_in, cluster_last_checked)
57
+
58
+ def put_meta(action: str, meta: AuditMeta, clusters: list[str] = None):
59
+ payload = {
60
+ 'action': action,
61
+ 'partitions-last-checked': meta.partitions_last_checked,
62
+ 'clusters-last-checked': meta.cluster_last_checked
63
+ }
64
+ if clusters:
65
+ payload['clusters'] = clusters
66
+
67
+ audit_endpoint = Config().get("audit.endpoint", "https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/")
68
+ try:
69
+ response = requests.post(audit_endpoint, json=payload, timeout=Config().get("audit.timeout", 10))
70
+ if response.status_code in [200, 201]:
71
+ debug(response.text)
72
+ else:
73
+ log2(f"Error: {response.status_code} {response.text}")
74
+ except requests.exceptions.Timeout as e:
75
+ log2(f"Timeout occurred: {e}")
76
+
77
+ def find_new_clusters(cluster_last_checked: float) -> list[str]:
78
+ dt_object = datetime.fromtimestamp(cluster_last_checked)
79
+
80
+ # select distinct c2.name from cluster as c1 right outer join
81
+ # (select distinct c as name from audit where y = '1969' and m = '12' and d >= '31' or y = '1969' and m > '12' or y > '1969') as c2
82
+ # on c1.name = c2.name where c1.name is null
83
+ query = '\n '.join([
84
+ 'select distinct c2.name from cluster as c1 right outer join',
85
+ f'(select distinct c as name from audit where {Audits.date_from(dt_object)}) as c2',
86
+ 'on c1.name = c2.name where c1.name is null'])
87
+ log2(query)
88
+ state, _, rs = Athena.query(query)
89
+ if state == 'SUCCEEDED':
90
+ if len(rs) > 1:
91
+ try:
92
+ return [r['Data'][0]['VarCharValue'] for r in rs[1:]]
93
+ except:
94
+ pass
95
+
96
+ return []
97
+
98
+ def date_from(dt_object: datetime):
99
+ y = dt_object.strftime("%Y")
100
+ m = dt_object.strftime("%m")
101
+ d = dt_object.strftime("%d")
102
+
103
+ return f"y = '{y}' and m = '{m}' and d >= '{d}' or y = '{y}' and m > '{m}' or y > '{y}'"
104
+
105
+ def offload() -> OffloadHandler:
106
+ return offload(max_workers=Config().get('audit.workers', 3))
adam/utils_issues.py ADDED
@@ -0,0 +1,32 @@
1
+ from adam.checks.check_result import CheckResult
2
+ from adam.checks.issue import Issue
3
+ from adam.repl_session import ReplSession
4
+ from adam.utils import lines_to_tabular, log, log2
5
+
6
+ class IssuesUtils:
7
+ def show(check_results: list[CheckResult], in_repl = False):
8
+ IssuesUtils.show_issues(CheckResult.collect_issues(check_results), in_repl=in_repl)
9
+
10
+ def show_issues(issues: list[Issue], in_repl = False):
11
+ if not issues:
12
+ log2('No issues found.')
13
+ else:
14
+ suggested = 0
15
+ log2(f'* {len(issues)} issues found.')
16
+ lines = []
17
+ for i, issue in enumerate(issues, start=1):
18
+ lines.append(f"{i}||{issue.category}||{issue.desc}")
19
+ lines.append(f"||statefulset||{issue.statefulset}@{issue.namespace}")
20
+ lines.append(f"||pod||{issue.pod}@{issue.namespace}")
21
+ if issue.details:
22
+ lines.append(f"||details||{issue.details}")
23
+
24
+ if issue.suggestion:
25
+ lines.append(f'||suggestion||{issue.suggestion}')
26
+ if in_repl:
27
+ ReplSession().prompt_session.history.append_string(issue.suggestion)
28
+ suggested += 1
29
+ log(lines_to_tabular(lines, separator='||'))
30
+ if suggested:
31
+ log2()
32
+ log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')
File without changes
@@ -0,0 +1,28 @@
1
+ import sys
2
+ from typing import TypeVar
3
+
4
+ from adam.utils_k8s.app_pods import AppPods
5
+ from adam.pod_exec_result import PodExecResult
6
+ from adam.utils import log, log2
7
+ from adam.utils_k8s.pods import Pods
8
+ from .kube_context import KubeContext
9
+
10
+ T = TypeVar('T')
11
+
12
+ # utility collection on app clusters; methods are all static
13
+ class AppClusters:
14
+ def exec(pods: list[str], namespace: str, command: str, action: str = 'action',
15
+ max_workers=0, show_out=True, on_any = False, shell = '/bin/sh', background = False) -> list[PodExecResult]:
16
+ samples = 1 if on_any else sys.maxsize
17
+ msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
18
+ with Pods.parallelize(pods, max_workers, samples, msg, action=action) as exec:
19
+ results: list[PodExecResult] = exec.map(lambda pod: AppPods.exec(pod, namespace, command, False, False, shell, background))
20
+ for result in results:
21
+ if KubeContext.show_out(show_out):
22
+ log(result.command)
23
+ if result.stdout:
24
+ log(result.stdout)
25
+ if result.stderr:
26
+ log2(result.stderr, file=sys.stderr)
27
+
28
+ return results
@@ -0,0 +1,33 @@
1
+ import functools
2
+ from typing import List
3
+ from kubernetes import client
4
+
5
+ from adam.config import Config
6
+ from adam.utils_k8s.pods import Pods
7
+ from adam.pod_exec_result import PodExecResult
8
+ from adam.repl_session import ReplSession
9
+
10
+ # utility collection on app pods; methods are all static
11
+ class AppPods:
12
+ @functools.lru_cache()
13
+ def pod_names(namespace: str, env: str, app: str):
14
+ return [pod.metadata.name for pod in AppPods.app_pods(namespace, env, app)]
15
+
16
+ def app_pods(namespace: str, env: str, app: str) -> List[client.V1Pod]:
17
+ v1 = client.CoreV1Api()
18
+
19
+ env_key = Config().get('app.env', 'c3__env-0')
20
+ app_key = Config().get('app.app', 'c3__app-0')
21
+ label_selector = f'applicationGroup=c3,{env_key}=0{env}0,{app_key}=0{app}0'
22
+
23
+ return v1.list_namespaced_pod(namespace, label_selector=label_selector).items
24
+
25
+ def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', background = False) -> PodExecResult:
26
+ container = Config().get('app.container-name', 'c3-server')
27
+ r = Pods.exec(pod_name, container, namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, background = background)
28
+
29
+ if r and Config().get('repl.history.push-cat-remote-log-file', True):
30
+ if r.log_file and ReplSession().prompt_session:
31
+ ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
32
+
33
+ return r
@@ -0,0 +1,36 @@
1
+ import sys
2
+ from typing import TypeVar
3
+
4
+ from adam.config import Config
5
+ from adam.utils_k8s.cassandra_nodes import CassandraNodes
6
+ from adam.pod_exec_result import PodExecResult
7
+ from adam.utils import log, log2
8
+ from adam.utils_k8s.pods import Pods
9
+ from adam.utils_k8s.statefulsets import StatefulSets
10
+
11
+ T = TypeVar('T')
12
+
13
+ # utility collection on cassandra clusters; methods are all static
14
+ class CassandraClusters:
15
+ def exec(sts: str, namespace: str, command: str, action: str = 'action',
16
+ max_workers=0, show_out=True, on_any = False, shell = '/bin/sh', background = False, log_file = None) -> list[PodExecResult]:
17
+
18
+ pods = StatefulSets.pod_names(sts, namespace)
19
+ samples = 1 if on_any else sys.maxsize
20
+ msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
21
+ with Pods.parallelize(pods, max_workers, samples, msg, action=action) as exec:
22
+ results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, background, log_file))
23
+ for result in results:
24
+ if show_out and not Config().is_debug():
25
+ log(result.command)
26
+ if result.stdout:
27
+ log(result.stdout)
28
+ if result.stderr:
29
+ log2(result.stderr, file=sys.stderr)
30
+
31
+ return results
32
+
33
+ def pod_names_by_host_id(sts: str, ns: str):
34
+ pods = StatefulSets.pods(sts, ns)
35
+
36
+ return {CassandraNodes.get_host_id(pod.metadata.name, ns): pod.metadata.name for pod in pods}
@@ -0,0 +1,33 @@
1
+ from adam.config import Config
2
+ from adam.utils_k8s.pods import Pods
3
+ from adam.utils_k8s.secrets import Secrets
4
+ from adam.pod_exec_result import PodExecResult
5
+ from adam.repl_session import ReplSession
6
+
7
+ # utility collection on cassandra nodes; methods are all static
8
+ class CassandraNodes:
9
+ def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', background = False, log_file = None) -> PodExecResult:
10
+ r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, background = background, log_file=log_file)
11
+
12
+ if r and Config().get('repl.history.push-cat-remote-log-file', True):
13
+ if r.log_file and ReplSession().prompt_session:
14
+ ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
15
+
16
+ return r
17
+
18
+ def get_host_id(pod_name: str, ns: str):
19
+ try:
20
+ user, pw = Secrets.get_user_pass(pod_name, ns)
21
+ command = f'echo "SELECT host_id FROM system.local; exit" | cqlsh --no-color -u {user} -p {pw}'
22
+ result: PodExecResult = CassandraNodes.exec(pod_name, ns, command, show_out=Config().is_debug())
23
+ next = False
24
+ for line in result.stdout.splitlines():
25
+ if next:
26
+ return line.strip(' ')
27
+ if line.startswith('----------'):
28
+ next = True
29
+ continue
30
+ except Exception as e:
31
+ return str(e)
32
+
33
+ return 'Unknown'
@@ -0,0 +1,34 @@
1
+ from kubernetes import client
2
+
3
+ # utility collection on config maps; methods are all static
4
+ class ConfigMaps:
5
+ def create(name: str, namespace: str, data: dict[str, str], labels: dict[str, str] = {}):
6
+ v1 = client.CoreV1Api()
7
+
8
+ metadata = client.V1ObjectMeta(
9
+ name=name,
10
+ namespace=namespace,
11
+ labels=labels
12
+ )
13
+
14
+ configmap = client.V1ConfigMap(
15
+ api_version="v1",
16
+ kind="ConfigMap",
17
+ metadata=metadata,
18
+ data=data
19
+ )
20
+
21
+ try:
22
+ api_response = v1.create_namespaced_config_map(body=configmap, namespace=namespace)
23
+ # print(f"ConfigMap '{name}' created successfully in namespace '{namespace}'.")
24
+ # print(api_response)
25
+ except client.ApiException as e:
26
+ # print(f"Error creating ConfigMap: {e}")
27
+ raise e
28
+
29
+ def delete_with_selector(namespace: str, label_selector: str):
30
+ v1 = client.CoreV1Api()
31
+
32
+ ret = v1.list_namespaced_config_map(namespace=namespace, label_selector=label_selector)
33
+ for i in ret.items:
34
+ v1.delete_namespaced_config_map(name=i.metadata.name, namespace=namespace)
@@ -1,10 +1,11 @@
1
+ import functools
1
2
  import re
2
3
  import time
3
4
  from kubernetes import client
4
5
 
5
- from walker.config import Config
6
+ from adam.config import Config
6
7
  from .kube_context import KubeContext
7
- from walker.utils import elapsed_time, lines_to_tabular, log2
8
+ from adam.utils import elapsed_time, lines_to_tabular, log2
8
9
 
9
10
 
10
11
  # utility collection; methods are all static
@@ -171,6 +172,10 @@ class CustomResources:
171
172
 
172
173
  return None
173
174
 
175
+ def clear_caches():
176
+ CustomResources.medusa_show_backupjobs.cache_clear()
177
+
178
+ @functools.lru_cache()
174
179
  def medusa_show_backupjobs(dc: str, ns: str) -> list[dict]:
175
180
  api_instance = client.CustomObjectsApi()
176
181
  group = 'medusa.k8ssandra.io'
@@ -0,0 +1,56 @@
1
+ from kubernetes import client
2
+
3
+ from adam.utils_k8s.pods import Pods
4
+ from adam.utils_k8s.volumes import ConfigMapMount
5
+
6
+ # utility collection on deployments; methods are all static
7
+ class Deployments:
8
+ def delete_with_selector(namespace: str, label_selector: str, grace_period_seconds: int = None):
9
+ v1 = client.AppsV1Api()
10
+
11
+ ret = v1.list_namespaced_deployment(namespace=namespace, label_selector=label_selector)
12
+ for i in ret.items:
13
+ v1.delete_namespaced_deployment(name=i.metadata.name, namespace=namespace, grace_period_seconds=grace_period_seconds)
14
+
15
+ def create_deployment_spec(name: str, image: str, image_pull_secret: str,
16
+ envs: list, container_security_context: client.V1SecurityContext,
17
+ volume_name: str, pvc_name:str, mount_path:str,
18
+ command: list[str]=None, sa_name=None, labels: dict[str, str] = {},
19
+ config_map_mount: ConfigMapMount = None):
20
+ return client.V1DeploymentSpec(
21
+ replicas=1,
22
+ selector=client.V1LabelSelector(match_labels=labels),
23
+ template=client.V1PodTemplateSpec(
24
+ metadata=client.V1ObjectMeta(labels=labels),
25
+ spec=Pods.create_pod_spec(name, image, image_pull_secret, envs, container_security_context,
26
+ volume_name, pvc_name, mount_path, command=command, sa_name=sa_name,
27
+ restart_policy="Always", config_map_mount=config_map_mount),
28
+ ),
29
+ )
30
+
31
+ def create(namespace: str, deployment_name: str, image: str,
32
+ command: list[str] = None,
33
+ secret: str = None,
34
+ env: dict[str, any] = {},
35
+ container_security_context: client.V1SecurityContext = None,
36
+ labels: dict[str, str] = {},
37
+ volume_name: str = None,
38
+ pvc_name: str = None,
39
+ mount_path: str = None,
40
+ sa_name=None,
41
+ config_map_mount: ConfigMapMount = None):
42
+ v1 = client.AppsV1Api()
43
+ envs = []
44
+ for k, v in env.items():
45
+ envs.append(client.V1EnvVar(name=str(k), value=str(v)))
46
+ deployment = Deployments.create_deployment_spec(deployment_name, image, secret, envs,
47
+ container_security_context, volume_name, pvc_name,
48
+ mount_path, command=command, sa_name=sa_name, labels=labels,
49
+ config_map_mount=config_map_mount)
50
+ return v1.create_namespaced_deployment(
51
+ namespace=namespace,
52
+ body=client.V1Deployment(spec=deployment, metadata=client.V1ObjectMeta(
53
+ name=deployment_name,
54
+ labels=labels
55
+ ))
56
+ )
@@ -1,7 +1,7 @@
1
1
  from kubernetes import client
2
2
 
3
- from walker.config import Config
4
- from walker.utils import log2
3
+ from adam.config import Config
4
+ from adam.utils import debug, log2
5
5
 
6
6
  # utility collection on ingresses; methods are all static
7
7
  class Ingresses:
@@ -63,8 +63,7 @@ class Ingresses:
63
63
  log2(f"200 Ingress '{name}' in namespace '{namespace}' deleted successfully.")
64
64
  else:
65
65
  api.delete_namespaced_ingress(name=name, namespace=namespace)
66
- if Config().is_debug():
67
- log2(f"200 Ingress '{name}' in namespace '{namespace}' deleted successfully.")
66
+ debug(f"200 Ingress '{name}' in namespace '{namespace}' deleted successfully.")
68
67
  except client.ApiException as e:
69
68
  log2(f"Error deleting Ingress: {e}")
70
69