kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. adam/__init__.py +1 -0
  2. adam/app_session.py +182 -0
  3. {walker → adam}/apps.py +8 -24
  4. {walker → adam}/batch.py +54 -97
  5. {walker → adam}/checks/check.py +3 -3
  6. {walker → adam}/checks/check_result.py +1 -1
  7. adam/checks/check_utils.py +65 -0
  8. {walker → adam}/checks/compactionstats.py +6 -6
  9. {walker → adam}/checks/cpu.py +14 -8
  10. adam/checks/cpu_metrics.py +52 -0
  11. {walker → adam}/checks/disk.py +6 -6
  12. {walker → adam}/checks/gossip.py +5 -5
  13. {walker → adam}/checks/memory.py +7 -7
  14. {walker → adam}/checks/status.py +5 -5
  15. {walker → adam}/cli.py +3 -3
  16. {walker → adam}/columns/column.py +1 -1
  17. adam/columns/columns.py +45 -0
  18. {walker → adam}/columns/compactions.py +5 -5
  19. {walker → adam}/columns/cpu.py +6 -4
  20. adam/columns/cpu_metrics.py +22 -0
  21. {walker → adam}/columns/dir_data.py +3 -3
  22. {walker → adam}/columns/dir_snapshots.py +3 -3
  23. {walker → adam}/columns/gossip.py +5 -5
  24. {walker → adam}/columns/host_id.py +3 -3
  25. {walker → adam}/columns/memory.py +3 -3
  26. {walker → adam}/columns/node_address.py +3 -3
  27. {walker → adam}/columns/node_load.py +3 -3
  28. {walker → adam}/columns/node_owns.py +3 -3
  29. {walker → adam}/columns/node_status.py +3 -3
  30. {walker → adam}/columns/node_tokens.py +3 -3
  31. {walker → adam}/columns/node_utils.py +2 -2
  32. {walker → adam}/columns/pod_name.py +2 -2
  33. {walker → adam}/columns/volume_cassandra.py +4 -4
  34. {walker → adam}/columns/volume_root.py +3 -3
  35. adam/commands/__init__.py +15 -0
  36. adam/commands/alter_tables.py +81 -0
  37. adam/commands/app_cmd.py +38 -0
  38. {walker → adam}/commands/app_ping.py +10 -16
  39. adam/commands/audit/audit.py +84 -0
  40. adam/commands/audit/audit_repair_tables.py +74 -0
  41. adam/commands/audit/audit_run.py +50 -0
  42. adam/commands/audit/show_last10.py +48 -0
  43. adam/commands/audit/show_slow10.py +47 -0
  44. adam/commands/audit/show_top10.py +45 -0
  45. adam/commands/audit/utils_show_top10.py +59 -0
  46. adam/commands/bash/__init__.py +5 -0
  47. adam/commands/bash/bash.py +36 -0
  48. adam/commands/bash/bash_completer.py +93 -0
  49. adam/commands/bash/utils_bash.py +16 -0
  50. adam/commands/cat.py +50 -0
  51. adam/commands/cd.py +43 -0
  52. adam/commands/check.py +73 -0
  53. {walker → adam}/commands/cli_commands.py +7 -8
  54. adam/commands/code.py +57 -0
  55. adam/commands/command.py +190 -0
  56. {walker → adam}/commands/command_helpers.py +1 -1
  57. {walker → adam}/commands/commands_utils.py +15 -25
  58. adam/commands/cp.py +89 -0
  59. adam/commands/cql/cql_completions.py +33 -0
  60. {walker/commands → adam/commands/cql}/cqlsh.py +20 -35
  61. adam/commands/cql/utils_cql.py +343 -0
  62. {walker/commands/frontend → adam/commands/deploy}/code_start.py +11 -14
  63. adam/commands/deploy/code_stop.py +40 -0
  64. {walker/commands/frontend → adam/commands/deploy}/code_utils.py +7 -9
  65. adam/commands/deploy/deploy.py +25 -0
  66. adam/commands/deploy/deploy_frontend.py +49 -0
  67. adam/commands/deploy/deploy_pg_agent.py +35 -0
  68. adam/commands/deploy/deploy_pod.py +108 -0
  69. adam/commands/deploy/deploy_utils.py +29 -0
  70. adam/commands/deploy/undeploy.py +25 -0
  71. adam/commands/deploy/undeploy_frontend.py +38 -0
  72. adam/commands/deploy/undeploy_pg_agent.py +39 -0
  73. adam/commands/deploy/undeploy_pod.py +48 -0
  74. adam/commands/devices/device.py +118 -0
  75. adam/commands/devices/device_app.py +173 -0
  76. adam/commands/devices/device_auit_log.py +49 -0
  77. adam/commands/devices/device_cass.py +185 -0
  78. adam/commands/devices/device_export.py +86 -0
  79. adam/commands/devices/device_postgres.py +144 -0
  80. adam/commands/devices/devices.py +25 -0
  81. {walker → adam}/commands/exit.py +3 -6
  82. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  83. adam/commands/export/clean_up_export_sessions.py +51 -0
  84. adam/commands/export/drop_export_database.py +55 -0
  85. adam/commands/export/drop_export_databases.py +43 -0
  86. adam/commands/export/export.py +53 -0
  87. adam/commands/export/export_databases.py +170 -0
  88. adam/commands/export/export_handlers.py +71 -0
  89. adam/commands/export/export_select.py +81 -0
  90. adam/commands/export/export_select_x.py +54 -0
  91. adam/commands/export/export_use.py +52 -0
  92. adam/commands/export/exporter.py +352 -0
  93. adam/commands/export/import_session.py +40 -0
  94. adam/commands/export/importer.py +67 -0
  95. adam/commands/export/importer_athena.py +80 -0
  96. adam/commands/export/importer_sqlite.py +47 -0
  97. adam/commands/export/show_column_counts.py +54 -0
  98. adam/commands/export/show_export_databases.py +36 -0
  99. adam/commands/export/show_export_session.py +48 -0
  100. adam/commands/export/show_export_sessions.py +44 -0
  101. adam/commands/export/utils_export.py +314 -0
  102. {walker → adam}/commands/help.py +17 -12
  103. adam/commands/intermediate_command.py +49 -0
  104. adam/commands/issues.py +43 -0
  105. adam/commands/kubectl.py +38 -0
  106. adam/commands/login.py +70 -0
  107. {walker → adam}/commands/logs.py +8 -10
  108. adam/commands/ls.py +41 -0
  109. adam/commands/medusa/medusa.py +27 -0
  110. adam/commands/medusa/medusa_backup.py +57 -0
  111. adam/commands/medusa/medusa_restore.py +83 -0
  112. adam/commands/medusa/medusa_show_backupjobs.py +51 -0
  113. adam/commands/medusa/medusa_show_restorejobs.py +47 -0
  114. {walker → adam}/commands/nodetool.py +17 -21
  115. {walker → adam}/commands/param_get.py +15 -16
  116. adam/commands/param_set.py +43 -0
  117. adam/commands/postgres/postgres.py +104 -0
  118. adam/commands/postgres/postgres_context.py +274 -0
  119. {walker → adam}/commands/postgres/postgres_ls.py +7 -11
  120. {walker → adam}/commands/postgres/postgres_preview.py +8 -13
  121. adam/commands/postgres/psql_completions.py +10 -0
  122. adam/commands/postgres/utils_postgres.py +66 -0
  123. adam/commands/preview_table.py +37 -0
  124. adam/commands/pwd.py +47 -0
  125. adam/commands/reaper/reaper.py +35 -0
  126. adam/commands/reaper/reaper_forward.py +93 -0
  127. adam/commands/reaper/reaper_forward_session.py +6 -0
  128. {walker → adam}/commands/reaper/reaper_forward_stop.py +13 -19
  129. {walker → adam}/commands/reaper/reaper_restart.py +10 -17
  130. adam/commands/reaper/reaper_run_abort.py +46 -0
  131. adam/commands/reaper/reaper_runs.py +82 -0
  132. adam/commands/reaper/reaper_runs_abort.py +63 -0
  133. adam/commands/reaper/reaper_schedule_activate.py +45 -0
  134. adam/commands/reaper/reaper_schedule_start.py +45 -0
  135. adam/commands/reaper/reaper_schedule_stop.py +45 -0
  136. {walker → adam}/commands/reaper/reaper_schedules.py +6 -16
  137. {walker → adam}/commands/reaper/reaper_status.py +11 -19
  138. adam/commands/reaper/utils_reaper.py +196 -0
  139. adam/commands/repair/repair.py +26 -0
  140. {walker → adam}/commands/repair/repair_log.py +7 -10
  141. adam/commands/repair/repair_run.py +70 -0
  142. adam/commands/repair/repair_scan.py +71 -0
  143. {walker → adam}/commands/repair/repair_stop.py +8 -11
  144. adam/commands/report.py +61 -0
  145. adam/commands/restart.py +60 -0
  146. {walker → adam}/commands/rollout.py +25 -30
  147. adam/commands/shell.py +34 -0
  148. adam/commands/show/show.py +39 -0
  149. walker/commands/show/show_version.py → adam/commands/show/show_adam.py +14 -10
  150. adam/commands/show/show_app_actions.py +57 -0
  151. {walker → adam}/commands/show/show_app_id.py +12 -15
  152. {walker → adam}/commands/show/show_app_queues.py +9 -12
  153. adam/commands/show/show_cassandra_repairs.py +38 -0
  154. adam/commands/show/show_cassandra_status.py +124 -0
  155. {walker → adam}/commands/show/show_cassandra_version.py +6 -16
  156. adam/commands/show/show_commands.py +59 -0
  157. walker/commands/show/show_storage.py → adam/commands/show/show_host.py +11 -13
  158. adam/commands/show/show_login.py +62 -0
  159. {walker → adam}/commands/show/show_params.py +4 -4
  160. adam/commands/show/show_processes.py +51 -0
  161. adam/commands/show/show_storage.py +42 -0
  162. adam/commands/watch.py +82 -0
  163. {walker → adam}/config.py +10 -22
  164. {walker → adam}/embedded_apps.py +1 -1
  165. adam/embedded_params.py +2 -0
  166. adam/log.py +47 -0
  167. {walker → adam}/pod_exec_result.py +10 -2
  168. adam/repl.py +182 -0
  169. adam/repl_commands.py +124 -0
  170. adam/repl_state.py +458 -0
  171. adam/sql/__init__.py +0 -0
  172. adam/sql/sql_completer.py +120 -0
  173. adam/sql/sql_state_machine.py +618 -0
  174. adam/sql/term_completer.py +76 -0
  175. adam/sso/__init__.py +0 -0
  176. {walker → adam}/sso/authenticator.py +5 -1
  177. adam/sso/authn_ad.py +170 -0
  178. {walker → adam}/sso/authn_okta.py +39 -22
  179. adam/sso/cred_cache.py +60 -0
  180. adam/sso/id_token.py +23 -0
  181. adam/sso/idp.py +143 -0
  182. adam/sso/idp_login.py +50 -0
  183. adam/sso/idp_session.py +55 -0
  184. adam/sso/sso_config.py +63 -0
  185. adam/utils.py +679 -0
  186. adam/utils_app.py +98 -0
  187. adam/utils_athena.py +145 -0
  188. adam/utils_audits.py +106 -0
  189. adam/utils_issues.py +32 -0
  190. adam/utils_k8s/__init__.py +0 -0
  191. adam/utils_k8s/app_clusters.py +28 -0
  192. adam/utils_k8s/app_pods.py +33 -0
  193. adam/utils_k8s/cassandra_clusters.py +36 -0
  194. adam/utils_k8s/cassandra_nodes.py +33 -0
  195. adam/utils_k8s/config_maps.py +34 -0
  196. {walker/k8s_utils → adam/utils_k8s}/custom_resources.py +7 -2
  197. adam/utils_k8s/deployment.py +56 -0
  198. {walker/k8s_utils → adam/utils_k8s}/ingresses.py +3 -4
  199. {walker/k8s_utils → adam/utils_k8s}/jobs.py +3 -3
  200. adam/utils_k8s/k8s.py +87 -0
  201. {walker/k8s_utils → adam/utils_k8s}/kube_context.py +4 -4
  202. adam/utils_k8s/pods.py +290 -0
  203. {walker/k8s_utils → adam/utils_k8s}/secrets.py +8 -4
  204. adam/utils_k8s/service_accounts.py +170 -0
  205. {walker/k8s_utils → adam/utils_k8s}/services.py +3 -4
  206. {walker/k8s_utils → adam/utils_k8s}/statefulsets.py +6 -16
  207. {walker/k8s_utils → adam/utils_k8s}/volumes.py +10 -1
  208. adam/utils_net.py +24 -0
  209. adam/utils_repl/__init__.py +0 -0
  210. adam/utils_repl/automata_completer.py +48 -0
  211. adam/utils_repl/repl_completer.py +46 -0
  212. adam/utils_repl/state_machine.py +173 -0
  213. adam/utils_sqlite.py +109 -0
  214. adam/version.py +5 -0
  215. {kaqing-1.77.0.dist-info → kaqing-2.0.171.dist-info}/METADATA +1 -1
  216. kaqing-2.0.171.dist-info/RECORD +236 -0
  217. kaqing-2.0.171.dist-info/entry_points.txt +3 -0
  218. kaqing-2.0.171.dist-info/top_level.txt +1 -0
  219. kaqing-1.77.0.dist-info/RECORD +0 -159
  220. kaqing-1.77.0.dist-info/entry_points.txt +0 -3
  221. kaqing-1.77.0.dist-info/top_level.txt +0 -1
  222. walker/__init__.py +0 -3
  223. walker/app_session.py +0 -168
  224. walker/checks/check_utils.py +0 -97
  225. walker/columns/columns.py +0 -43
  226. walker/commands/add_user.py +0 -68
  227. walker/commands/app.py +0 -67
  228. walker/commands/bash.py +0 -87
  229. walker/commands/cd.py +0 -115
  230. walker/commands/check.py +0 -68
  231. walker/commands/command.py +0 -104
  232. walker/commands/cp.py +0 -95
  233. walker/commands/cql_utils.py +0 -53
  234. walker/commands/devices.py +0 -89
  235. walker/commands/frontend/code_stop.py +0 -57
  236. walker/commands/frontend/setup.py +0 -60
  237. walker/commands/frontend/setup_frontend.py +0 -58
  238. walker/commands/frontend/teardown.py +0 -61
  239. walker/commands/frontend/teardown_frontend.py +0 -42
  240. walker/commands/issues.py +0 -69
  241. walker/commands/login.py +0 -72
  242. walker/commands/ls.py +0 -145
  243. walker/commands/medusa/medusa.py +0 -69
  244. walker/commands/medusa/medusa_backup.py +0 -61
  245. walker/commands/medusa/medusa_restore.py +0 -86
  246. walker/commands/medusa/medusa_show_backupjobs.py +0 -52
  247. walker/commands/medusa/medusa_show_restorejobs.py +0 -52
  248. walker/commands/param_set.py +0 -44
  249. walker/commands/postgres/postgres.py +0 -113
  250. walker/commands/postgres/postgres_session.py +0 -225
  251. walker/commands/preview_table.py +0 -98
  252. walker/commands/processes.py +0 -53
  253. walker/commands/pwd.py +0 -64
  254. walker/commands/reaper/reaper.py +0 -78
  255. walker/commands/reaper/reaper_forward.py +0 -100
  256. walker/commands/reaper/reaper_run_abort.py +0 -65
  257. walker/commands/reaper/reaper_runs.py +0 -97
  258. walker/commands/reaper/reaper_runs_abort.py +0 -83
  259. walker/commands/reaper/reaper_schedule_activate.py +0 -64
  260. walker/commands/reaper/reaper_schedule_start.py +0 -64
  261. walker/commands/reaper/reaper_schedule_stop.py +0 -64
  262. walker/commands/reaper/reaper_session.py +0 -159
  263. walker/commands/repair/repair.py +0 -68
  264. walker/commands/repair/repair_run.py +0 -72
  265. walker/commands/repair/repair_scan.py +0 -79
  266. walker/commands/report.py +0 -57
  267. walker/commands/restart.py +0 -61
  268. walker/commands/show/show.py +0 -72
  269. walker/commands/show/show_app_actions.py +0 -53
  270. walker/commands/show/show_cassandra_status.py +0 -35
  271. walker/commands/show/show_commands.py +0 -58
  272. walker/commands/show/show_processes.py +0 -35
  273. walker/commands/show/show_repairs.py +0 -47
  274. walker/commands/status.py +0 -128
  275. walker/commands/storage.py +0 -52
  276. walker/commands/user_entry.py +0 -69
  277. walker/commands/watch.py +0 -85
  278. walker/embedded_params.py +0 -2
  279. walker/k8s_utils/cassandra_clusters.py +0 -48
  280. walker/k8s_utils/cassandra_nodes.py +0 -26
  281. walker/k8s_utils/pods.py +0 -211
  282. walker/repl.py +0 -165
  283. walker/repl_commands.py +0 -58
  284. walker/repl_state.py +0 -211
  285. walker/sso/authn_ad.py +0 -94
  286. walker/sso/idp.py +0 -150
  287. walker/sso/idp_login.py +0 -29
  288. walker/sso/sso_config.py +0 -45
  289. walker/utils.py +0 -194
  290. walker/version.py +0 -5
  291. {walker → adam}/checks/__init__.py +0 -0
  292. {walker → adam}/checks/check_context.py +0 -0
  293. {walker → adam}/checks/issue.py +0 -0
  294. {walker → adam}/cli_group.py +0 -0
  295. {walker → adam}/columns/__init__.py +0 -0
  296. {walker/commands → adam/commands/audit}/__init__.py +0 -0
  297. {walker/commands/frontend → adam/commands/cql}/__init__.py +0 -0
  298. {walker/commands/medusa → adam/commands/deploy}/__init__.py +0 -0
  299. {walker/commands/postgres → adam/commands/devices}/__init__.py +0 -0
  300. {walker/commands/reaper → adam/commands/export}/__init__.py +0 -0
  301. {walker/commands/repair → adam/commands/medusa}/__init__.py +0 -0
  302. {walker → adam}/commands/nodetool_commands.py +0 -0
  303. {walker/commands/show → adam/commands/postgres}/__init__.py +0 -0
  304. {walker/k8s_utils → adam/commands/reaper}/__init__.py +0 -0
  305. {walker/sso → adam/commands/repair}/__init__.py +0 -0
  306. /walker/medusa_show_restorejobs.py → /adam/commands/show/__init__.py +0 -0
  307. {walker → adam}/repl_session.py +0 -0
  308. {kaqing-1.77.0.dist-info → kaqing-2.0.171.dist-info}/WHEEL +0 -0
@@ -0,0 +1,343 @@
1
+ import functools
2
+ import re
3
+ from typing import Union
4
+
5
+ from adam.commands.command import Command
6
+ from adam.commands.commands_utils import show_table
7
+ from adam.utils_k8s.cassandra_clusters import CassandraClusters
8
+ from adam.utils_k8s.cassandra_nodes import CassandraNodes
9
+ from adam.utils_k8s.secrets import Secrets
10
+ from adam.pod_exec_result import PodExecResult
11
+ from adam.repl_state import ReplState
12
+ from adam.utils import log2, log_timing, wait_log
13
+ from adam.utils_k8s.statefulsets import StatefulSets
14
+
15
+ @functools.lru_cache()
16
+ def cassandra_keyspaces(state: ReplState, on_any=True):
17
+ if state.pod:
18
+ wait_log(f'Inspecting Cassandra Keyspaces on {state.pod}...')
19
+ else:
20
+ wait_log(f'Inspecting Cassandra Keyspaces...')
21
+
22
+ r: list[PodExecResult] = run_cql(state, 'describe keyspaces', show_out=False, on_any=on_any)
23
+ if not r:
24
+ log2('No pod is available')
25
+ return []
26
+
27
+ return parse_cql_desc_keyspaces(r.stdout if state.pod else r[0].stdout)
28
+
29
+ def cassandra_table_names(state: ReplState, keyspace = None):
30
+ return [f'{k}.{t}' for k, ts in cassandra_tables(state, on_any=True).items() for t in ts if not keyspace or keyspace == '*' or k == keyspace]
31
+
32
+ @functools.lru_cache()
33
+ def cassandra_tables(state: ReplState, on_any=False) -> dict[str, list[str]]:
34
+ r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=on_any)
35
+ if not r:
36
+ log2('No pod is available')
37
+ return {}
38
+
39
+ return parse_cql_desc_tables(r.stdout if state.pod else r[0].stdout)
40
+
41
+ @functools.lru_cache()
42
+ def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
43
+ r: list[PodExecResult] = run_cql(state, f'describe table {table}', show_out=False, on_any=on_any)
44
+ if not r:
45
+ log2('No pod is available')
46
+ return None
47
+
48
+ return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
49
+
50
+ def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, background=False, log_file=None) -> list[PodExecResult]:
51
+ if show_query:
52
+ log2(cql)
53
+
54
+ command = None
55
+ with log_timing('Secrets.get_user_pass'):
56
+ user, pw = Secrets.get_user_pass(state.sts if state.sts else state.pod, state.namespace, secret_path='cql.secret')
57
+ if use_single_quotes:
58
+ command = f"cqlsh -u {user} -p {pw} {' '.join(opts)} -e '{cql}'"
59
+ else:
60
+ command = f'cqlsh -u {user} -p {pw} {" ".join(opts)} -e "{cql}"'
61
+
62
+ with log_timing(cql):
63
+ with cassandra(state) as pods:
64
+ return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, background=background, log_file=log_file)
65
+
66
+ def parse_cql_desc_tables(out: str):
67
+ # Keyspace data_endpoint_auth
68
+ # ---------------------------
69
+ # "token"
70
+
71
+ # Keyspace reaper_db
72
+ # ------------------
73
+ # repair_run schema_migration
74
+ # repair_run_by_cluster schema_migration_leader
75
+
76
+ # Keyspace system
77
+ tables_by_keyspace: dict[str, list[str]] = {}
78
+ keyspace = None
79
+ state = 's0'
80
+ for line in out.split('\n'):
81
+ if state == 's0':
82
+ groups = re.match(r'^Keyspace (.*)$', line)
83
+ if groups:
84
+ keyspace = groups[1].strip(' \r')
85
+ state = 's1'
86
+ elif state == 's1':
87
+ if line.startswith('---'):
88
+ state = 's2'
89
+ elif state == 's2':
90
+ if not line.strip(' \r'):
91
+ state = 's0'
92
+ else:
93
+ for table in line.split(' '):
94
+ if t := table.strip(' \r'):
95
+ if not keyspace in tables_by_keyspace:
96
+ tables_by_keyspace[keyspace] = []
97
+ tables_by_keyspace[keyspace].append(t)
98
+
99
+ return tables_by_keyspace
100
+
101
+ def parse_cql_desc_keyspaces(out: str) -> list[str]:
102
+ #
103
+ # Warning: Cannot create directory at `/home/cassandra/.cassandra`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.
104
+ #
105
+ #
106
+ # Warning: Using a password on the command line interface can be insecure.
107
+ # Recommendation: use the credentials file to securely provide the password.
108
+ #
109
+ #
110
+ # azops88_db system_auth system_traces
111
+ # reaper_db system_distributed system_views
112
+ # system system_schema system_virtual_schema
113
+ #
114
+ kses = []
115
+ for line in out.split('\n'):
116
+ line = line.strip(' \r')
117
+ if not line:
118
+ continue
119
+ if line.startswith('Warning:'):
120
+ continue
121
+ if line.startswith('Recommendation:'):
122
+ continue
123
+
124
+ for ks in line.split(' '):
125
+ if s := ks.strip(' \r\t'):
126
+ kses.append(s)
127
+
128
+ return kses
129
+
130
+ class ColumnSpec:
131
+ def __init__(self, name: str, type: str, key_index = -1):
132
+ self.name = name
133
+ self.type = type
134
+ self.key_index = key_index
135
+
136
+ def __eq__(self, other):
137
+ if not isinstance(other, ColumnSpec):
138
+ return NotImplemented
139
+
140
+ return self.name == other.name and self.type == other.type and self.key_index == other.key_index
141
+
142
+ class TableSpec:
143
+ def __init__(self, columns: list[ColumnSpec]):
144
+ self.columns = columns
145
+
146
+ def row_key(self):
147
+ for c in self.columns:
148
+ if c.key_index == 0:
149
+ return c.name
150
+
151
+ def keys(self):
152
+ return [c.name for c in self.columns if c.key_index > -1]
153
+
154
+ def parse_cql_desc_table(out: str) -> TableSpec:
155
+ # CREATE TABLE azops88_db.analyticscontainer_dfeevalhistory (
156
+ # id text,
157
+ # columnname text,
158
+ # version bigint static,
159
+ # contentb blob,
160
+ # contentbool boolean,
161
+ # contentn double,
162
+ # contents text,
163
+ # PRIMARY KEY (id, columnname)
164
+ # ) WITH CLUSTERING ORDER BY (columnname ASC)
165
+ # AND additional_write_policy = '99p'
166
+ # AND bloom_filter_fp_chance = 0.1
167
+ # AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
168
+ # AND cdc = false
169
+ # AND comment = ''
170
+ # AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
171
+ # AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.SnappyCompressor'}
172
+ # AND memtable = 'default'
173
+ # AND crc_check_chance = 1.0
174
+ # AND default_time_to_live = 0
175
+ # AND extensions = {}
176
+ # AND gc_grace_seconds = 3600
177
+ # AND max_index_interval = 2048
178
+ # AND memtable_flush_period_in_ms = 0
179
+ # AND min_index_interval = 128
180
+ # AND read_repair = 'BLOCKING'
181
+ # AND speculative_retry = '99p';
182
+ pkeys = {}
183
+ columns: list[ColumnSpec] = []
184
+
185
+ state = 's0'
186
+ for line in out.split('\n'):
187
+ if state == 's0':
188
+ if line.startswith('CREATE TABLE'):
189
+ state = 's1'
190
+ elif state == 's1':
191
+ if line.startswith(')'):
192
+ state = 's2'
193
+ continue
194
+
195
+ groups = re.match(r'^\s*PRIMARY KEY\s*\((.*)\).*$', line)
196
+ if groups:
197
+ pkeys = {n.strip(' '): i for i, n in enumerate(groups[1].strip(' \r').split(','))}
198
+ continue
199
+
200
+ # single key column - name text PRIMARY KEY,
201
+ groups = re.match(r'^\s*(\S*?)\s*(\S*?)\s*PRIMARY KEY,.*$', line)
202
+ if groups:
203
+ columns.append(ColumnSpec(groups[1], groups[2]))
204
+ pkeys[groups[1]] = 0
205
+ else:
206
+ groups = re.match(r'^\s*(\S*?)\s*(\S*?),.*$', line)
207
+ if groups:
208
+ columns.append(ColumnSpec(groups[1], groups[2]))
209
+ elif state == 's2':
210
+ pass
211
+
212
+ for column in columns:
213
+ if column.name in pkeys.keys():
214
+ column.key_index = pkeys[column.name]
215
+
216
+ return TableSpec(columns)
217
+
218
+ class CqlShHandler:
219
+ def __init__(self, state: ReplState, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, background=False):
220
+ self.state = state
221
+ self.opts = opts
222
+ self.show_out = show_out
223
+ self.show_query = show_query
224
+ self.use_single_quotes = use_single_quotes
225
+ self.on_any = on_any
226
+ self.backgroud = background
227
+
228
+ def __enter__(self):
229
+ return self.query
230
+
231
+ def __exit__(self, exc_type, exc_val, exc_tb):
232
+ return False
233
+
234
+ def query(self, args: list[str], log_file: str = None):
235
+ query: str = args
236
+ background = self.backgroud
237
+ show_out = self.show_out
238
+
239
+ if isinstance(query, list):
240
+ opts = []
241
+ cqls = []
242
+ for index, arg in enumerate(args):
243
+ if arg.startswith('--'):
244
+ opts.append(arg)
245
+ # elif index == len(args) -1 and arg == '&':
246
+ # background = True
247
+ elif arg != '-e':
248
+ cqls.append(arg)
249
+ if not cqls:
250
+ if self.state.in_repl:
251
+ log2('Please enter cql statement. e.g. select host_id from system.local')
252
+ else:
253
+ log2('* CQL statement is missing.')
254
+ log2()
255
+ Command.display_help()
256
+
257
+ return 'no-cql'
258
+
259
+ query = ' '.join(cqls)
260
+ show_out = True
261
+
262
+ return run_cql(self.state, query, opts=self.opts, show_out=show_out, show_query=self.show_query, use_single_quotes=self.use_single_quotes, on_any=self.on_any, background=background, log_file=log_file)
263
+
264
+ class CassandraPodService:
265
+ def __init__(self, handler: 'CassandraExecHandler'):
266
+ self.handler = handler
267
+
268
+ def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', background = False, log_file = None) -> Union[PodExecResult, list[PodExecResult]]:
269
+ state = self.handler.state
270
+ pod = self.handler.pod
271
+
272
+ if pod:
273
+ return CassandraNodes.exec(pod, state.namespace, command,
274
+ show_out=show_out, throw_err=throw_err, shell=shell, background=background, log_file=log_file)
275
+ elif state.sts:
276
+ return CassandraClusters.exec(state.sts, state.namespace, command, action=action,
277
+ show_out=show_out, on_any=on_any, shell=shell, background=background, log_file=log_file)
278
+
279
+ return []
280
+
281
+ def cql(self, args: list[str], opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, background=False, log_file: str = None):
282
+ state = self.handler.state
283
+ query: str = args
284
+
285
+ if isinstance(query, list):
286
+ opts = []
287
+ cqls = []
288
+ for arg in args:
289
+ if arg.startswith('--'):
290
+ opts.append(arg)
291
+ elif arg != '-e':
292
+ cqls.append(arg)
293
+ if not cqls:
294
+ if self.state.in_repl:
295
+ log2('Please enter cql statement. e.g. select host_id from system.local')
296
+ else:
297
+ log2('* CQL statement is missing.')
298
+ log2()
299
+ Command.display_help()
300
+
301
+ return 'no-cql'
302
+
303
+ query = ' '.join(cqls)
304
+ show_out = True
305
+
306
+ return run_cql(state, query, opts=opts, show_out=show_out, show_query=show_query, use_single_quotes=use_single_quotes, on_any=on_any, background=background, log_file=log_file)
307
+
308
+ def show_table(self, cols: str, header: str, show_out = True):
309
+ state = self.handler.state
310
+
311
+ if state.pod:
312
+ show_table(state, [state.pod], cols, header, show_out=show_out)
313
+ elif state.sts:
314
+ pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
315
+ show_table(state, pod_names, cols, header, show_out=show_out)
316
+
317
+ def nodetool(self, args: str, show_out = True) -> Union[PodExecResult, list[PodExecResult]]:
318
+ state = self.handler.state
319
+ pod = self.handler.pod
320
+
321
+ user, pw = state.user_pass()
322
+ command = f"nodetool -u {user} -pw {pw} {args}"
323
+
324
+ if pod:
325
+ return CassandraNodes.exec(pod, state.namespace, command, show_out=show_out)
326
+ else:
327
+ return CassandraClusters.exec(state.sts, state.namespace, command, action='nodetool', show_out=show_out)
328
+
329
+ class CassandraExecHandler:
330
+ def __init__(self, state: ReplState, pod: str = None):
331
+ self.state = state
332
+ self.pod = pod
333
+ if not pod and state.pod:
334
+ self.pod = state.pod
335
+
336
+ def __enter__(self):
337
+ return CassandraPodService(self)
338
+
339
+ def __exit__(self, exc_type, exc_val, exc_tb):
340
+ return False
341
+
342
+ def cassandra(state: ReplState, pod: str=None):
343
+ return CassandraExecHandler(state, pod=pod)
@@ -1,7 +1,7 @@
1
- from walker.commands.command import Command
2
- from walker.commands.frontend.code_utils import start_user_code, stop_user_codes
3
- from walker.repl_state import ReplState, RequiredState
4
- from walker.utils import log2
1
+ from adam.commands.command import Command
2
+ from adam.commands.deploy.code_utils import start_user_code, stop_user_codes
3
+ from adam.repl_state import ReplState, RequiredState
4
+ from adam.utils import log2
5
5
 
6
6
  class CodeStart(Command):
7
7
  COMMAND = 'code start'
@@ -25,19 +25,16 @@ class CodeStart(Command):
25
25
  if not(args := self.args(cmd)):
26
26
  return super().run(cmd, state)
27
27
 
28
- state, args = self.apply_state(args, state)
29
- if not self.validate_state(state):
30
- return state
31
-
32
- log2('This will support c3/c3 only for demo.')
28
+ with self.validate(args, state) as (args, state):
29
+ log2('This will support c3/c3 only for demo.')
33
30
 
34
- stop_user_codes(state.namespace)
35
- try:
36
- start_user_code(state.namespace)
37
- finally:
38
31
  stop_user_codes(state.namespace)
32
+ try:
33
+ start_user_code(state.namespace)
34
+ finally:
35
+ stop_user_codes(state.namespace)
39
36
 
40
- return state
37
+ return state
41
38
 
42
39
  def completion(self, state: ReplState):
43
40
  if state.namespace:
@@ -0,0 +1,40 @@
1
+ from adam.commands.command import Command
2
+ from adam.commands.deploy.code_utils import stop_user_codes
3
+ from adam.repl_state import ReplState, RequiredState
4
+
5
+ class CodeStop(Command):
6
+ COMMAND = 'code stop'
7
+
8
+ # the singleton pattern
9
+ def __new__(cls, *args, **kwargs):
10
+ if not hasattr(cls, 'instance'): cls.instance = super(CodeStop, cls).__new__(cls)
11
+
12
+ return cls.instance
13
+
14
+ def __init__(self, successor: Command=None):
15
+ super().__init__(successor)
16
+
17
+ def command(self):
18
+ return CodeStop.COMMAND
19
+
20
+ def required(self):
21
+ return RequiredState.NAMESPACE
22
+
23
+ def run(self, cmd: str, state: ReplState):
24
+ if not(args := self.args(cmd)):
25
+ return super().run(cmd, state)
26
+
27
+ with self.validate(args, state) as (args, state):
28
+ _, dry = Command.extract_options(args, '--dry')
29
+ stop_user_codes(state.namespace, dry)
30
+
31
+ return state
32
+
33
+ def completion(self, state: ReplState):
34
+ if state.namespace:
35
+ return super().completion(state)
36
+
37
+ return {}
38
+
39
+ def help(self, _: ReplState):
40
+ return f'{CodeStop.COMMAND}\t stop code server'
@@ -3,11 +3,11 @@ from pathlib import Path
3
3
  import socket
4
4
  import subprocess
5
5
 
6
- from walker.apps import Apps
7
- from walker.config import Config
8
- from walker.k8s_utils.ingresses import Ingresses
9
- from walker.k8s_utils.services import Services
10
- from walker.utils import log2, random_alphanumeric
6
+ from adam.apps import Apps
7
+ from adam.config import Config
8
+ from adam.utils_k8s.ingresses import Ingresses
9
+ from adam.utils_k8s.services import Services
10
+ from adam.utils import debug, log2, random_alphanumeric
11
11
 
12
12
  def start_user_code(namespace: str):
13
13
  try:
@@ -57,8 +57,7 @@ def kill_process_by_pattern(pattern, dry=False):
57
57
  pids = process.stdout.strip().split('\n')
58
58
 
59
59
  if not pids or pids == ['']:
60
- if Config().is_debug():
61
- log2(f"No processes found matching pattern: '{pattern}'")
60
+ debug(f"No processes found matching pattern: '{pattern}'")
62
61
  return
63
62
 
64
63
  for pid in pids:
@@ -67,8 +66,7 @@ def kill_process_by_pattern(pattern, dry=False):
67
66
  if not dry:
68
67
  subprocess.run(f"kill -9 {pid}", shell=True, check=True)
69
68
 
70
- if Config().is_debug():
71
- log2(f"Killed process with PID: {pid} (matching pattern: '{pattern}')")
69
+ debug(f"Killed process with PID: {pid} (matching pattern: '{pattern}')")
72
70
  except subprocess.CalledProcessError as e:
73
71
  log2(f"Error killing process {pid}: {e}")
74
72
 
@@ -0,0 +1,25 @@
1
+ import click
2
+
3
+ from adam.commands.deploy.deploy_pg_agent import DeployPgAgent
4
+ from adam.commands.deploy.deploy_pod import DeployPod
5
+ from adam.commands.intermediate_command import IntermediateCommand
6
+ from .deploy_frontend import DeployFrontend
7
+
8
+ class Deploy(IntermediateCommand):
9
+ COMMAND = 'deploy'
10
+
11
+ # the singleton pattern
12
+ def __new__(cls, *args, **kwargs):
13
+ if not hasattr(cls, 'instance'): cls.instance = super(Deploy, cls).__new__(cls)
14
+
15
+ return cls.instance
16
+
17
+ def command(self):
18
+ return Deploy.COMMAND
19
+
20
+ def cmd_list(self):
21
+ return [DeployFrontend(), DeployPod(), DeployPgAgent()]
22
+
23
+ class DeployCommandHelper(click.Command):
24
+ def get_help(self, ctx: click.Context):
25
+ IntermediateCommand.intermediate_help(super().get_help(ctx), Deploy.COMMAND, Deploy().cmd_list())
@@ -0,0 +1,49 @@
1
+ from adam.commands.command import Command
2
+ from adam.commands.deploy.deploy_utils import deploy_frontend
3
+ from adam.config import Config
4
+ from adam.repl_state import ReplState, RequiredState
5
+ from adam.utils import log2
6
+
7
+ class DeployFrontend(Command):
8
+ COMMAND = 'deploy frontend'
9
+
10
+ # the singleton pattern
11
+ def __new__(cls, *args, **kwargs):
12
+ if not hasattr(cls, 'instance'): cls.instance = super(DeployFrontend, cls).__new__(cls)
13
+
14
+ return cls.instance
15
+
16
+ def __init__(self, successor: Command=None):
17
+ super().__init__(successor)
18
+
19
+ def command(self):
20
+ return DeployFrontend.COMMAND
21
+
22
+ def required(self):
23
+ return RequiredState.NAMESPACE
24
+
25
+ def run(self, cmd: str, state: ReplState):
26
+ if not(args := self.args(cmd)):
27
+ return super().run(cmd, state)
28
+
29
+ with self.validate(args, state) as (args, state):
30
+ log2('This will support c3/c3 only for demo.')
31
+
32
+ pod_name = Config().get('pod.name', 'ops')
33
+ label_selector = Config().get('pod.label-selector', 'run=ops')
34
+ try:
35
+ uri = deploy_frontend(pod_name, state.namespace, label_selector)
36
+ log2(f'Ops pod is available at {uri}.')
37
+ except Exception as e:
38
+ if e.status == 409:
39
+ log2(f"Error: '{pod_name}' already exists in namespace '{state.namespace}'.")
40
+ else:
41
+ log2(f"Error creating ingress or service: {e}")
42
+
43
+ return state
44
+
45
+ def completion(self, state: ReplState):
46
+ return super().completion(state)
47
+
48
+ def help(self, _: ReplState):
49
+ return f'{DeployFrontend.COMMAND}\t deploy Web frontend'
@@ -0,0 +1,35 @@
1
+ from adam.commands.command import Command
2
+ from adam.commands.postgres.postgres_context import PostgresContext
3
+ from adam.config import Config
4
+ from adam.repl_state import ReplState, RequiredState
5
+
6
+ class DeployPgAgent(Command):
7
+ COMMAND = 'deploy pg-agent'
8
+
9
+ # the singleton pattern
10
+ def __new__(cls, *args, **kwargs):
11
+ if not hasattr(cls, 'instance'): cls.instance = super(DeployPgAgent, cls).__new__(cls)
12
+
13
+ return cls.instance
14
+
15
+ def __init__(self, successor: Command=None):
16
+ super().__init__(successor)
17
+
18
+ def command(self):
19
+ return DeployPgAgent.COMMAND
20
+
21
+ def required(self):
22
+ return RequiredState.NAMESPACE
23
+
24
+ def run(self, cmd: str, state: ReplState):
25
+ if not(args := self.args(cmd)):
26
+ return super().run(cmd, state)
27
+
28
+ with self.validate(args, state) as (args, state):
29
+ PostgresContext.deploy_pg_agent(Config().get('pg.agent.name', 'ops-pg-agent'), state.namespace)
30
+
31
+ def completion(self, state: ReplState):
32
+ return super().completion(state)
33
+
34
+ def help(self, _: ReplState):
35
+ return f'{DeployPgAgent.COMMAND}\t deploy Postgres agent'
@@ -0,0 +1,108 @@
1
+ from kubernetes import client
2
+
3
+ from adam.commands import extract_options
4
+ from adam.commands.command import Command
5
+ from adam.commands.deploy.deploy_utils import deploy_frontend, gen_labels
6
+ from adam.commands.deploy.undeploy_pod import UndeployPod
7
+ from adam.config import Config
8
+ from adam.utils_k8s.config_maps import ConfigMaps
9
+ from adam.utils_k8s.deployment import Deployments
10
+ from adam.utils_k8s.kube_context import KubeContext
11
+ from adam.utils_k8s.pods import Pods
12
+ from adam.utils_k8s.service_accounts import ServiceAccounts
13
+ from adam.utils_k8s.volumes import ConfigMapMount
14
+ from adam.repl_state import ReplState, RequiredState
15
+ from adam.utils import ing, log2
16
+
17
+ class DeployPod(Command):
18
+ COMMAND = 'deploy pod'
19
+
20
+ # the singleton pattern
21
+ def __new__(cls, *args, **kwargs):
22
+ if not hasattr(cls, 'instance'): cls.instance = super(DeployPod, cls).__new__(cls)
23
+
24
+ return cls.instance
25
+
26
+ def __init__(self, successor: Command=None):
27
+ super().__init__(successor)
28
+
29
+ def command(self):
30
+ return DeployPod.COMMAND
31
+
32
+ def required(self):
33
+ return RequiredState.NAMESPACE
34
+
35
+ def run(self, cmd: str, state: ReplState):
36
+ if not(args := self.args(cmd)):
37
+ return super().run(cmd, state)
38
+
39
+ with self.validate(args, state) as (args, state):
40
+ with extract_options(args, '--force') as (args, forced):
41
+ if forced:
42
+ UndeployPod().run(UndeployPod.COMMAND, state)
43
+
44
+ if KubeContext.in_cluster():
45
+ log2('This is doable only from outside of the Kubernetes cluster.')
46
+ return state
47
+
48
+ sa_name = Config().get('pod.sa.name', 'ops')
49
+ sa_proto = Config().get('pod.sa.proto', 'c3')
50
+ additional_cluster_roles = Config().get('pod.sa.additional-cluster-roles', 'c3aiops-k8ssandra-operator').split(',')
51
+ label_selector = Config().get('pod.label-selector', 'run=ops')
52
+ labels = gen_labels(label_selector)
53
+
54
+ with ing('Creating service account'):
55
+ ServiceAccounts.replicate(sa_name, state.namespace, sa_proto, labels=labels, add_cluster_roles=additional_cluster_roles)
56
+
57
+ settings_filename = 'settings.yaml'
58
+ settings_path = f'/kaqing/{settings_filename}'
59
+ settings_data = None
60
+ try:
61
+ with open(settings_filename, 'r') as file:
62
+ settings_data = file.read()
63
+ except:
64
+ try:
65
+ with open(settings_path, 'r') as file:
66
+ settings_data = file.read()
67
+ except:
68
+ pass
69
+
70
+ if not settings_data:
71
+ log2(f'{settings_filename} not found.')
72
+ return state
73
+
74
+ cm_name = Config().get('pod.cm.name', 'ops')
75
+ map_data = {
76
+ settings_filename : settings_data
77
+ }
78
+ with ing('Creating config map'):
79
+ ConfigMaps.create(cm_name, state.namespace, map_data, labels=labels)
80
+
81
+ pod_name = Config().get('pod.name', 'ops')
82
+ image = Config().get('pod.image', 'seanahnsf/kaqing')
83
+ security_context = client.V1SecurityContext(
84
+ capabilities=client.V1Capabilities(
85
+ add=["SYS_PTRACE"]
86
+ )
87
+ )
88
+ with ing('Creating deployment'):
89
+ Deployments.create(state.namespace,
90
+ pod_name,
91
+ image,
92
+ env={'NAMESPACE': state.namespace},
93
+ container_security_context=security_context,
94
+ labels=labels,
95
+ sa_name=sa_name,
96
+ config_map_mount=ConfigMapMount(cm_name, settings_filename, settings_path))
97
+
98
+ uri = deploy_frontend(pod_name, state.namespace, label_selector)
99
+
100
+ Pods.wait_for_running(state.namespace, pod_name, msg=f'In moments, ops pod will be available at {uri}.', label_selector=label_selector)
101
+
102
+ return state
103
+
104
+ def completion(self, state: ReplState):
105
+ return super().completion(state, {'--force': None})
106
+
107
+ def help(self, _: ReplState):
108
+ return f'{DeployPod.COMMAND} [--force]\t deploy Ops pod, --force to undeploy first'