kaqing 2.0.145__py3-none-any.whl → 2.0.189__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kaqing might be problematic. Click here for more details.

Files changed (209) hide show
  1. adam/__init__.py +0 -2
  2. adam/app_session.py +9 -12
  3. adam/apps.py +18 -4
  4. adam/batch.py +4 -4
  5. adam/checks/check_utils.py +16 -46
  6. adam/checks/cpu.py +7 -1
  7. adam/checks/cpu_metrics.py +52 -0
  8. adam/checks/disk.py +2 -3
  9. adam/columns/columns.py +3 -1
  10. adam/columns/cpu.py +3 -1
  11. adam/columns/cpu_metrics.py +22 -0
  12. adam/columns/memory.py +3 -4
  13. adam/commands/__init__.py +24 -0
  14. adam/commands/alter_tables.py +33 -48
  15. adam/commands/app/__init__.py +0 -0
  16. adam/commands/app/app.py +38 -0
  17. adam/commands/{app_ping.py → app/app_ping.py} +7 -13
  18. adam/commands/app/show_app_actions.py +49 -0
  19. adam/commands/{show → app}/show_app_id.py +8 -11
  20. adam/commands/{show → app}/show_app_queues.py +7 -14
  21. adam/commands/app/utils_app.py +106 -0
  22. adam/commands/audit/audit.py +21 -40
  23. adam/commands/audit/audit_repair_tables.py +14 -19
  24. adam/commands/audit/audit_run.py +14 -22
  25. adam/commands/audit/completions_l.py +15 -0
  26. adam/commands/audit/show_last10.py +4 -19
  27. adam/commands/audit/show_slow10.py +4 -18
  28. adam/commands/audit/show_top10.py +4 -16
  29. adam/commands/audit/utils_show_top10.py +15 -3
  30. adam/commands/bash/__init__.py +5 -0
  31. adam/commands/bash/bash.py +7 -104
  32. adam/commands/bash/utils_bash.py +16 -0
  33. adam/commands/cat.py +7 -27
  34. adam/commands/cd.py +7 -11
  35. adam/commands/check.py +15 -24
  36. adam/commands/cli_commands.py +8 -4
  37. adam/commands/clipboard_copy.py +87 -0
  38. adam/commands/code.py +21 -24
  39. adam/commands/command.py +207 -42
  40. adam/commands/commands_utils.py +25 -27
  41. adam/commands/cql/completions_c.py +28 -0
  42. adam/commands/cql/cqlsh.py +9 -33
  43. adam/commands/cql/{cql_utils.py → utils_cql.py} +111 -15
  44. adam/commands/deploy/code_start.py +7 -10
  45. adam/commands/deploy/code_stop.py +4 -21
  46. adam/commands/deploy/code_utils.py +3 -3
  47. adam/commands/deploy/deploy.py +4 -27
  48. adam/commands/deploy/deploy_frontend.py +14 -17
  49. adam/commands/deploy/deploy_pg_agent.py +3 -6
  50. adam/commands/deploy/deploy_pod.py +64 -68
  51. adam/commands/deploy/undeploy.py +4 -27
  52. adam/commands/deploy/undeploy_frontend.py +4 -7
  53. adam/commands/deploy/undeploy_pg_agent.py +5 -8
  54. adam/commands/deploy/undeploy_pod.py +9 -12
  55. adam/commands/devices/device.py +124 -2
  56. adam/commands/devices/device_app.py +41 -24
  57. adam/commands/devices/device_auit_log.py +10 -4
  58. adam/commands/devices/device_cass.py +48 -14
  59. adam/commands/devices/device_export.py +13 -12
  60. adam/commands/devices/device_postgres.py +105 -54
  61. adam/commands/download_file.py +47 -0
  62. adam/commands/exit.py +1 -4
  63. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  64. adam/commands/export/clean_up_export_sessions.py +9 -10
  65. adam/commands/export/completions_x.py +11 -0
  66. adam/commands/export/download_export_session.py +40 -0
  67. adam/commands/export/drop_export_database.py +7 -26
  68. adam/commands/export/drop_export_databases.py +5 -14
  69. adam/commands/export/export.py +6 -52
  70. adam/commands/export/export_databases.py +108 -32
  71. adam/commands/export/export_select.py +8 -59
  72. adam/commands/export/export_sessions.py +209 -0
  73. adam/commands/export/export_use.py +14 -20
  74. adam/commands/export/export_x_select.py +48 -0
  75. adam/commands/export/exporter.py +135 -167
  76. adam/commands/export/import_files.py +44 -0
  77. adam/commands/export/import_session.py +11 -35
  78. adam/commands/export/importer.py +19 -5
  79. adam/commands/export/importer_athena.py +112 -44
  80. adam/commands/export/importer_sqlite.py +42 -22
  81. adam/commands/export/show_column_counts.py +13 -31
  82. adam/commands/export/show_export_databases.py +7 -7
  83. adam/commands/export/show_export_session.py +8 -20
  84. adam/commands/export/show_export_sessions.py +6 -16
  85. adam/commands/export/utils_export.py +64 -11
  86. adam/commands/find_files.py +51 -0
  87. adam/commands/find_processes.py +76 -0
  88. adam/commands/head.py +36 -0
  89. adam/commands/help.py +2 -2
  90. adam/commands/intermediate_command.py +52 -0
  91. adam/commands/issues.py +11 -43
  92. adam/commands/kubectl.py +3 -6
  93. adam/commands/login.py +22 -24
  94. adam/commands/logs.py +3 -6
  95. adam/commands/ls.py +9 -10
  96. adam/commands/medusa/medusa.py +4 -22
  97. adam/commands/medusa/medusa_backup.py +20 -27
  98. adam/commands/medusa/medusa_restore.py +49 -46
  99. adam/commands/medusa/medusa_show_backupjobs.py +16 -18
  100. adam/commands/medusa/medusa_show_restorejobs.py +13 -18
  101. adam/commands/medusa/utils_medusa.py +15 -0
  102. adam/commands/nodetool.py +7 -21
  103. adam/commands/param_get.py +11 -14
  104. adam/commands/param_set.py +8 -12
  105. adam/commands/postgres/completions_p.py +22 -0
  106. adam/commands/postgres/postgres.py +34 -57
  107. adam/commands/postgres/postgres_databases.py +270 -0
  108. adam/commands/postgres/postgres_ls.py +4 -8
  109. adam/commands/postgres/postgres_preview.py +5 -9
  110. adam/commands/postgres/utils_postgres.py +79 -0
  111. adam/commands/preview_table.py +8 -45
  112. adam/commands/pwd.py +13 -16
  113. adam/commands/reaper/reaper.py +4 -27
  114. adam/commands/reaper/reaper_forward.py +49 -56
  115. adam/commands/reaper/reaper_forward_session.py +6 -0
  116. adam/commands/reaper/reaper_forward_stop.py +10 -16
  117. adam/commands/reaper/reaper_restart.py +7 -14
  118. adam/commands/reaper/reaper_run_abort.py +8 -33
  119. adam/commands/reaper/reaper_runs.py +43 -58
  120. adam/commands/reaper/reaper_runs_abort.py +29 -49
  121. adam/commands/reaper/reaper_schedule_activate.py +14 -33
  122. adam/commands/reaper/reaper_schedule_start.py +9 -33
  123. adam/commands/reaper/reaper_schedule_stop.py +9 -33
  124. adam/commands/reaper/reaper_schedules.py +4 -14
  125. adam/commands/reaper/reaper_status.py +8 -16
  126. adam/commands/reaper/utils_reaper.py +203 -0
  127. adam/commands/repair/repair.py +4 -22
  128. adam/commands/repair/repair_log.py +5 -11
  129. adam/commands/repair/repair_run.py +27 -34
  130. adam/commands/repair/repair_scan.py +32 -40
  131. adam/commands/repair/repair_stop.py +5 -12
  132. adam/commands/report.py +27 -29
  133. adam/commands/restart.py +25 -26
  134. adam/commands/rollout.py +19 -24
  135. adam/commands/shell.py +12 -4
  136. adam/commands/show/show.py +11 -27
  137. adam/commands/show/show_adam.py +3 -3
  138. adam/commands/show/show_cassandra_repairs.py +37 -0
  139. adam/commands/show/show_cassandra_status.py +47 -51
  140. adam/commands/show/show_cassandra_version.py +5 -18
  141. adam/commands/show/show_cli_commands.py +56 -0
  142. adam/commands/show/show_host.py +1 -1
  143. adam/commands/show/show_login.py +20 -27
  144. adam/commands/show/show_params.py +2 -5
  145. adam/commands/show/show_processes.py +18 -21
  146. adam/commands/show/show_storage.py +11 -20
  147. adam/commands/watch.py +26 -29
  148. adam/config.py +5 -16
  149. adam/embedded_params.py +1 -1
  150. adam/log.py +4 -4
  151. adam/pod_exec_result.py +3 -3
  152. adam/repl.py +45 -39
  153. adam/repl_commands.py +26 -19
  154. adam/repl_session.py +8 -1
  155. adam/repl_state.py +85 -36
  156. adam/sql/lark_completer.py +284 -0
  157. adam/sql/lark_parser.py +604 -0
  158. adam/sql/sql_completer.py +4 -6
  159. adam/sql/sql_state_machine.py +29 -16
  160. adam/sso/authn_ad.py +6 -8
  161. adam/sso/authn_okta.py +4 -6
  162. adam/sso/cred_cache.py +3 -5
  163. adam/sso/idp.py +9 -12
  164. adam/utils.py +484 -37
  165. adam/utils_athena.py +19 -19
  166. adam/utils_audits.py +12 -12
  167. adam/utils_issues.py +32 -0
  168. adam/utils_k8s/app_clusters.py +14 -19
  169. adam/utils_k8s/app_pods.py +7 -2
  170. adam/utils_k8s/cassandra_clusters.py +30 -19
  171. adam/utils_k8s/cassandra_nodes.py +2 -2
  172. adam/utils_k8s/custom_resources.py +16 -17
  173. adam/utils_k8s/ingresses.py +2 -2
  174. adam/utils_k8s/jobs.py +7 -11
  175. adam/utils_k8s/k8s.py +96 -0
  176. adam/utils_k8s/kube_context.py +2 -2
  177. adam/utils_k8s/pods.py +37 -81
  178. adam/utils_k8s/secrets.py +4 -4
  179. adam/utils_k8s/service_accounts.py +5 -4
  180. adam/utils_k8s/services.py +2 -2
  181. adam/utils_k8s/statefulsets.py +6 -14
  182. adam/utils_local.py +4 -0
  183. adam/utils_repl/appendable_completer.py +6 -0
  184. adam/utils_repl/repl_completer.py +128 -2
  185. adam/utils_repl/state_machine.py +3 -3
  186. adam/utils_sqlite.py +78 -42
  187. adam/version.py +1 -1
  188. {kaqing-2.0.145.dist-info → kaqing-2.0.189.dist-info}/METADATA +1 -1
  189. kaqing-2.0.189.dist-info/RECORD +253 -0
  190. kaqing-2.0.189.dist-info/top_level.txt +2 -0
  191. teddy/__init__.py +0 -0
  192. teddy/lark_parser.py +436 -0
  193. teddy/lark_parser2.py +618 -0
  194. adam/commands/app.py +0 -67
  195. adam/commands/cp.py +0 -95
  196. adam/commands/cql/cql_completions.py +0 -28
  197. adam/commands/export/clean_up_export_session.py +0 -53
  198. adam/commands/export/export_select_x.py +0 -54
  199. adam/commands/postgres/postgres_context.py +0 -248
  200. adam/commands/postgres/postgres_utils.py +0 -31
  201. adam/commands/postgres/psql_completions.py +0 -10
  202. adam/commands/reaper/reaper_session.py +0 -159
  203. adam/commands/show/show_app_actions.py +0 -56
  204. adam/commands/show/show_commands.py +0 -61
  205. adam/commands/show/show_repairs.py +0 -47
  206. kaqing-2.0.145.dist-info/RECORD +0 -227
  207. kaqing-2.0.145.dist-info/top_level.txt +0 -1
  208. {kaqing-2.0.145.dist-info → kaqing-2.0.189.dist-info}/WHEEL +0 -0
  209. {kaqing-2.0.145.dist-info → kaqing-2.0.189.dist-info}/entry_points.txt +0 -0
adam/commands/cp.py DELETED
@@ -1,95 +0,0 @@
1
- import click
2
- import pyperclip
3
-
4
- from adam.commands.command import Command
5
- from adam.commands.command_helpers import ClusterOrPodCommandHelper
6
- from adam.commands.cli_commands import CliCommands
7
- from adam.repl_state import ReplState, RequiredState
8
- from adam.utils import lines_to_tabular, log, log2
9
-
10
- class ClipboardCopy(Command):
11
- COMMAND = 'cp'
12
-
13
- # the singleton pattern
14
- def __new__(cls, *args, **kwargs):
15
- if not hasattr(cls, 'instance'): cls.instance = super(ClipboardCopy, cls).__new__(cls)
16
-
17
- return cls.instance
18
-
19
- def __init__(self, successor: Command=None):
20
- super().__init__(successor)
21
-
22
- def command(self):
23
- return ClipboardCopy.COMMAND
24
-
25
- def required(self):
26
- return RequiredState.CLUSTER_OR_POD
27
-
28
- def run(self, cmd: str, state: ReplState):
29
- if not(args := self.args(cmd)):
30
- return super().run(cmd, state)
31
-
32
- state, args = self.apply_state(args, state)
33
- if not self.validate_state(state):
34
- return state
35
-
36
- if len(args) < 1:
37
- if state.in_repl:
38
- log2('Key is required.')
39
- log2()
40
- log2('Keys:')
41
- log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
42
- else:
43
- log2('* Key is missing.')
44
- Command.display_help()
45
-
46
- return 'command-missing'
47
-
48
- key = args[0]
49
- if not key in CliCommands.values(state):
50
- if state.in_repl:
51
- log2('Key is required.')
52
- log2()
53
- log2('Keys:')
54
- log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
55
- else:
56
- log2('* Invalid key')
57
- Command.display_help()
58
-
59
- return 'command-invalid'
60
-
61
- value = CliCommands.values(state)[key]
62
- pyperclip.copy(value)
63
- log2('The following line has been copied to clipboard. Use <Ctrl-V> to use it.')
64
- log2(f' {value}')
65
-
66
- return 'value-copied'
67
-
68
- def completion(self, state: ReplState):
69
- if state.sts:
70
- return {ClipboardCopy.COMMAND: {key: None for key in CliCommands.values(state).keys()}}
71
-
72
- return {}
73
-
74
- def help(self, _: ReplState):
75
- return f"{ClipboardCopy.COMMAND} <key>\t copy a value to clipboard for conveninence"
76
-
77
- class CopyCommandHelper(click.Command):
78
- def lines(self):
79
- return [
80
- 'node-exec-?: kubectl exec command to the Cassandra pod',
81
- 'reaper-exec: kubectl exec command to the Reaper pod',
82
- 'reaper-forward: kubectl port-forward command to the Reaper pod',
83
- 'reaper-ui: uri to Reaper ui',
84
- 'reaper-username: Reaper user name',
85
- 'reaper-password: Reaper password',
86
- ]
87
-
88
- def get_help(self, ctx: click.Context):
89
- log(super().get_help(ctx))
90
- log()
91
- log('Keys:')
92
-
93
- log(lines_to_tabular(self.lines(), separator=':'))
94
- log()
95
- ClusterOrPodCommandHelper.cluter_or_pod_help()
@@ -1,28 +0,0 @@
1
- from adam.commands.cql.cql_utils import cassandra_keyspaces, cassandra_table_names
2
- from adam.commands.export.exporter import Exporter
3
- from adam.commands.export.export_databases import ExportDatabases
4
- from adam.config import Config
5
- from adam.repl_state import ReplState
6
- from adam.sql.sql_completer import SqlCompleter, SqlVariant
7
-
8
- def cql_completions(state: ReplState) -> dict[str, any]:
9
- ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
10
- # warm up caches
11
- cassandra_keyspaces(state)
12
- cassandra_table_names(state)
13
- ExportDatabases.database_names()
14
-
15
- expandables = {
16
- 'keyspaces': lambda: cassandra_keyspaces(state),
17
- 'table-props': lambda: {
18
- 'GC_GRACE_SECONDS': ps
19
- },
20
- 'export-dbs': lambda: ExportDatabases.database_names(),
21
- 'export-sessions': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace),
22
- 'export-sessions-incomplete': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
23
- }
24
-
25
- return SqlCompleter(
26
- lambda: cassandra_table_names(state),
27
- expandables=expandables,
28
- variant=SqlVariant.CQL).completions_for_nesting()
@@ -1,53 +0,0 @@
1
- from adam.commands.command import Command
2
- from adam.commands.export.exporter import Exporter
3
- from adam.repl_state import ReplState, RequiredState
4
- from adam.utils import log, log2
5
-
6
- class CleanUpExportSession(Command):
7
- COMMAND = 'clean up export session'
8
-
9
- # the singleton pattern
10
- def __new__(cls, *args, **kwargs):
11
- if not hasattr(cls, 'instance'): cls.instance = super(CleanUpExportSession, cls).__new__(cls)
12
-
13
- return cls.instance
14
-
15
- def __init__(self, successor: Command=None):
16
- super().__init__(successor)
17
-
18
- def command(self):
19
- return CleanUpExportSession.COMMAND
20
-
21
- def required(self):
22
- return RequiredState.CLUSTER_OR_POD
23
-
24
- def run(self, cmd: str, state: ReplState):
25
- if not(args := self.args(cmd)):
26
- return super().run(cmd, state)
27
-
28
- state, args = self.apply_state(args, state)
29
- if not self.validate_state(state):
30
- return state
31
-
32
- if not args:
33
- if state.in_repl:
34
- log2('Specify export session name.')
35
- else:
36
- log2('* Session name is missing.')
37
-
38
- Command.display_help()
39
-
40
- return 'command-missing'
41
-
42
- csv_cnt, log_cnt = Exporter.clean_up_session(state.sts, state.pod, state.namespace, args[0])
43
- log(f'Removed {csv_cnt} csv and {log_cnt} log files.')
44
-
45
- Exporter.clear_export_session_cache()
46
-
47
- return state
48
-
49
- def completion(self, _: ReplState):
50
- return {}
51
-
52
- def help(self, _: ReplState):
53
- return f'{CleanUpExportSession.COMMAND} <export-session-name>\t clean up export session'
@@ -1,54 +0,0 @@
1
- from adam.commands.command import Command
2
- from adam.commands.export.export_databases import ExportDatabases
3
- from adam.repl_state import ReplState, RequiredState
4
- from adam.sql.sql_completer import SqlCompleter, SqlVariant
5
- from adam.utils_athena import Athena
6
-
7
- # No action body, only for a help entry and auto-completion
8
- class ExportSelectX(Command):
9
- COMMAND = 'select_on_x'
10
-
11
- # the singleton pattern
12
- def __new__(cls, *args, **kwargs):
13
- if not hasattr(cls, 'instance'): cls.instance = super(ExportSelectX, cls).__new__(cls)
14
-
15
- return cls.instance
16
-
17
- def __init__(self, successor: Command=None):
18
- super().__init__(successor)
19
-
20
- def command(self):
21
- return ExportSelectX.COMMAND
22
-
23
- def required(self):
24
- return RequiredState.EXPORT_DB
25
-
26
- def completion(self, state: ReplState):
27
- completions = {}
28
-
29
- if state.device == ReplState.X:
30
- completions = {'drop': SqlCompleter(
31
- lambda: ExportDatabases.table_names(state.export_session),
32
- dml='drop',
33
- expandables={
34
- 'export-dbs': lambda: ExportDatabases.database_names(),
35
- 'columns':lambda _: Athena.column_names(database=state.export_session, function='export'),
36
- },
37
- variant=SqlVariant.ATHENA
38
- )}
39
-
40
- if state.export_session:
41
- completions |= {'select': SqlCompleter(
42
- lambda: ExportDatabases.table_names(state.export_session),
43
- dml='select',
44
- expandables={
45
- 'export-dbs': lambda: ExportDatabases.database_names(),
46
- 'columns':lambda _: Athena.column_names(database=state.export_session, function='export'),
47
- },
48
- variant=SqlVariant.ATHENA
49
- )}
50
-
51
- return completions
52
-
53
- def help(self, _: ReplState):
54
- return f'<sql-select-statements>\t run queries on export database'
@@ -1,248 +0,0 @@
1
- from datetime import datetime
2
- import functools
3
- import re
4
- import subprocess
5
-
6
- from adam.config import Config
7
- from adam.repl_session import ReplSession
8
- from adam.utils_k8s.kube_context import KubeContext
9
- from adam.utils_k8s.pods import Pods
10
- from adam.utils_k8s.secrets import Secrets
11
- from adam.utils import log2
12
-
13
- class PostgresContext:
14
- def apply(namespace: str, path: str, arg: str = None) -> 'PostgresContext':
15
- context = PostgresContext(namespace, path)
16
-
17
- if arg:
18
- if arg == '..':
19
- if context.db:
20
- context.db = None
21
- else:
22
- context.host = None
23
- else:
24
- tks = arg.split('@')
25
- if not context.host:
26
- context.host = tks[0]
27
- else:
28
- context.db = tks[0]
29
-
30
- if not namespace and tks[1]:
31
- context.namespace = tks[1]
32
-
33
- return context
34
-
35
- def __init__(self, ns: str, path: str):
36
- self.namespace = ns
37
- self.conn_details = None
38
- self.host = None
39
- self.db = None
40
-
41
- if path:
42
- tks = path.split('/')
43
- hn = tks[0].split('@')
44
- self.host = hn[0]
45
- if len(hn) > 1 and not ns:
46
- self.namespace = hn[1]
47
-
48
- if len(tks) > 1:
49
- self.db = tks[1]
50
-
51
- def path(self):
52
- if not self.host:
53
- return None
54
-
55
- d = self.host
56
- if not self.db:
57
- return d
58
-
59
- return f'{self.host}/{self.db}'
60
-
61
- def hosts(ns: str):
62
- return PostgresContext.hosts_for_namespace(ns)
63
-
64
- @functools.lru_cache()
65
- def hosts_for_namespace(ns: str):
66
- ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
67
-
68
- def excludes(name: str):
69
- exs = Config().get('pg.excludes', '.helm., -admin-secret')
70
- if exs:
71
- for ex in exs.split(','):
72
- if ex.strip(' ') in name:
73
- return True
74
-
75
- return False
76
-
77
- return [s for s in ss if not excludes(s)]
78
-
79
- def databases(self):
80
- dbs = []
81
- # List of databases
82
- # Name | Owner | Encoding | Collate | Ctype | ICU Locale | Locale Provider | Access privileges
83
- # ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
84
- # postgres | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc |
85
- # stgawsscpsr_c3_c3 | postgres | UTF8 | C | C | | libc |
86
- # template1 | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc | =c/postgres +
87
- # | | | | | | | postgres=CTc/postgres
88
- # (48 rows)
89
- if r := self.run_sql('\l', show_out=False):
90
- s = 0
91
- for line in r.stdout.split('\n'):
92
- line: str = line.strip(' \r')
93
- if s == 0:
94
- if 'List of databases' in line:
95
- s = 1
96
- elif s == 1:
97
- if 'Name' in line and 'Owner' in line and 'Encoding' in line:
98
- s = 2
99
- elif s == 2:
100
- if line.startswith('---------'):
101
- s = 3
102
- elif s == 3:
103
- groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
104
- if groups and groups[1] != '|':
105
- dbs.append({'name': groups[1], 'owner': groups[2]})
106
-
107
- return dbs
108
-
109
- def tables(self):
110
- dbs = []
111
- # List of relations
112
- # Schema | Name | Type | Owner
113
- # ----------+------------------------------------------------------------+-------+---------------
114
- # postgres | c3_2_admin_aclpriv | table | postgres
115
- # postgres | c3_2_admin_aclpriv_a | table | postgres
116
- if r := self.run_sql('\dt', show_out=False):
117
- s = 0
118
- for line in r.stdout.split('\n'):
119
- line: str = line.strip(' \r')
120
- if s == 0:
121
- if 'List of relations' in line:
122
- s = 1
123
- elif s == 1:
124
- if 'Schema' in line and 'Name' in line and 'Type' in line:
125
- s = 2
126
- elif s == 2:
127
- if line.startswith('---------'):
128
- s = 3
129
- elif s == 3:
130
- groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
131
- if groups and groups[1] != '|':
132
- dbs.append({'schema': groups[1], 'name': groups[2]})
133
-
134
- return dbs
135
-
136
- def run_sql(self, sql: str, show_out = True, background = False):
137
- db = self.db if self.db else PostgresContext.default_db()
138
-
139
- if KubeContext.in_cluster():
140
- cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
141
- log2(f'{cmd1} "{sql}"')
142
- # remove double quotes from the sql argument
143
- cmd = cmd1.split(' ') + [sql]
144
-
145
- r = subprocess.run(cmd, capture_output=not background, text=True)
146
- if show_out:
147
- log2(r.stdout)
148
- log2(r.stderr)
149
-
150
- return r
151
- else:
152
- ns = self.namespace
153
- pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
154
-
155
- if Config().get('pg.agent.just-in-time', False):
156
- if not PostgresContext.deploy_pg_agent(pod_name, ns):
157
- return
158
-
159
- real_pod_name = pod_name
160
- try:
161
- # try with dedicated pg agent pod name configured
162
- Pods.get(ns, pod_name)
163
- except:
164
- try:
165
- # try with the ops pod
166
- pod_name = Config().get('pod.name', 'ops')
167
- real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
168
- except:
169
- log2(f"Could not locate {pod_name} pod.")
170
- return None
171
-
172
- cmd = f'psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
173
- env_prefix = f'PGPASSWORD="{self.password()}"'
174
-
175
- r = Pods.exec(real_pod_name, pod_name, ns, cmd, show_out=show_out, background=background, env_prefix=env_prefix)
176
- if r and Config().get('repl.history.push-cat-remote-log-file', True):
177
- if r.log_file and ReplSession().prompt_session:
178
- ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
179
-
180
- return r
181
-
182
- def deploy_pg_agent(pod_name: str, ns: str) -> str:
183
- image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
184
- timeout = Config().get('pg.agent.timeout', 3600)
185
- try:
186
- Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
187
- except Exception as e:
188
- if e.status == 409:
189
- if Pods.completed(ns, pod_name):
190
- try:
191
- Pods.delete(pod_name, ns)
192
- Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
193
- except Exception as e2:
194
- log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
195
-
196
- return
197
- else:
198
- log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
199
-
200
- return
201
-
202
- Pods.wait_for_running(ns, pod_name)
203
-
204
- return pod_name
205
-
206
- def undeploy_pg_agent(pod_name: str, ns: str):
207
- Pods.delete(pod_name, ns, grace_period_seconds=0)
208
-
209
- def endpoint(self):
210
- if not self.conn_details:
211
- self.conn_details = Secrets.get_data(self.namespace, self.host)
212
-
213
- endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
214
-
215
- return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
216
-
217
- def port(self):
218
- if not self.conn_details:
219
- self.conn_details = Secrets.get_data(self.namespace, self.host)
220
-
221
- port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
222
-
223
- return self.conn_details[port_key] if port_key in self.conn_details else ''
224
-
225
- def username(self):
226
- if not self.conn_details:
227
- self.conn_details = Secrets.get_data(self.namespace, self.host)
228
-
229
- username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
230
-
231
- return self.conn_details[username_key] if username_key in self.conn_details else ''
232
-
233
- def password(self):
234
- if not self.conn_details:
235
- self.conn_details = Secrets.get_data(self.namespace, self.host)
236
-
237
- password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
238
-
239
- return self.conn_details[password_key] if password_key in self.conn_details else ''
240
-
241
- def default_db():
242
- return Config().get('pg.default-db', 'postgres')
243
-
244
- def default_owner():
245
- return Config().get('pg.default-owner', 'postgres')
246
-
247
- def default_schema():
248
- return Config().get('pg.default-schema', 'postgres')
@@ -1,31 +0,0 @@
1
- import functools
2
-
3
- from adam.commands.postgres.postgres_context import PostgresContext
4
- from adam.config import Config
5
-
6
- TestPG = [False]
7
-
8
- @functools.lru_cache()
9
- def pg_database_names(ns: str, pg_path: str):
10
- if TestPG[0]:
11
- return ['azops88_c3ai_c3']
12
-
13
- Config().wait_log('Inspecting Postgres Databases...')
14
-
15
- pg = PostgresContext.apply(ns, pg_path)
16
- return [db['name'] for db in pg.databases() if db['owner'] == PostgresContext.default_owner()]
17
-
18
- @functools.lru_cache()
19
- def pg_table_names(ns: str, pg_path: str):
20
- if TestPG[0]:
21
- return ['C3_2_XYZ1']
22
-
23
- Config().wait_log('Inspecting Postgres Database...')
24
- return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresContext.default_schema()]
25
-
26
- def pg_tables(ns: str, pg_path: str):
27
- pg = PostgresContext.apply(ns, pg_path)
28
- if pg.db:
29
- return pg.tables()
30
-
31
- return []
@@ -1,10 +0,0 @@
1
- from adam.commands.postgres.postgres_utils import pg_table_names
2
- from adam.sql.sql_completer import SqlCompleter
3
-
4
- def psql_completions(ns: str, pg_path: str):
5
- return {
6
- '\h': None,
7
- '\d': None,
8
- '\dt': None,
9
- '\du': None
10
- } | SqlCompleter(lambda: pg_table_names(ns, pg_path)).completions_for_nesting()
@@ -1,159 +0,0 @@
1
- from collections.abc import Callable
2
- import threading
3
- from kubernetes import client
4
- import portforward
5
- import re
6
- import requests
7
- from typing import List, cast
8
-
9
- from adam.config import Config
10
- from adam.utils_k8s.kube_context import KubeContext
11
- from adam.repl_state import ReplState
12
- from adam.utils import lines_to_tabular, log2
13
-
14
- class ReaperSession:
15
- is_forwarding = False
16
- stopping = threading.Event()
17
- schedules_ids_by_cluster: dict[str, list[str]] = {}
18
-
19
- def __init__(self, pod: str, headers: dict[str, str] = None):
20
- self.pod = pod
21
- self.headers = headers
22
-
23
- def login(self, state: ReplState, local_addr: str, remote_addr: str, show_output = True) -> str :
24
- user, pw = state.user_pass(secret_path='reaper.secret')
25
-
26
- response = requests.post(f'http://{local_addr}/login', headers={
27
- 'Accept': '*'
28
- },data={
29
- 'username':user,
30
- 'password':pw})
31
- if show_output:
32
- log2(f'POST {remote_addr}/login')
33
- log2(f' username={user}&password={pw}')
34
-
35
- if int(response.status_code / 100) != 2:
36
- if show_output:
37
- log2("login failed")
38
- return None
39
-
40
- return response.headers['Set-Cookie']
41
-
42
- def port_forwarded(self, state: ReplState, path: str, body: Callable[[str, dict[str, str]], requests.Response], method: str = None, show_output = True):
43
- local_port = Config().get('reaper.port-forward.local-port', 9001)
44
- target_port = 8080
45
-
46
- def f(local_addr: str, remote_addr: str):
47
- if not self.headers:
48
- self.headers = self.cookie_header(state, local_addr, remote_addr, show_output=show_output)
49
-
50
- if show_output and method:
51
- log2(f'{method} {remote_addr}/{path}')
52
- response = body(f'http://{local_addr}/{path}', self.headers)
53
-
54
- if response:
55
- if int(response.status_code / 100) != 2:
56
- if show_output:
57
- log2(response.status_code)
58
- return response
59
-
60
- if show_output:
61
- log2()
62
-
63
- return response if response else 'no-response'
64
-
65
- if KubeContext.in_cluster():
66
- # cs-a526330d23-cs-a526330d23-default-sts-0 ->
67
- # curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
68
- groups = re.match(r'^(.*?-.*?-.*?-.*?-).*', state.sts)
69
- if groups:
70
- svc_name = Config().get('reaper.service-name', 'reaper-service')
71
- svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{target_port}'
72
- return f(local_addr=svc, remote_addr=svc)
73
- else:
74
- return None
75
- else:
76
- with portforward.forward(state.namespace, self.pod, local_port, target_port):
77
- return f(local_addr=f'localhost:{local_port}', remote_addr=f'{self.pod}:{target_port}')
78
-
79
- def cookie_header(self, state: ReplState, local_addr, remote_addr, show_output = True):
80
- return {'Cookie': self.login(state, local_addr, remote_addr, show_output=show_output)}
81
-
82
- def create(state: ReplState) -> 'ReaperSession':
83
- pods = ReaperSession.list_reaper_pods(state.sts if state.sts else state.pod, state.namespace)
84
- if pods:
85
- return ReaperSession(pods[0].metadata.name)
86
- else:
87
- log2('No reaper found.')
88
-
89
- return None
90
-
91
- def list_reaper_pods(sts_name: str, namespace: str) -> List[client.V1Pod]:
92
- v1 = client.CoreV1Api()
93
-
94
- # k8ssandra.io/reaper: cs-d0767a536f-cs-d0767a536f-reaper
95
- groups = re.match(Config().get('reaper.pod.cluster-regex', r'(.*?-.*?-.*?-.*?)-.*'), sts_name)
96
- label_selector = Config().get('reaper.pod.label-selector', 'k8ssandra.io/reaper={cluster}-reaper').replace('{cluster}', groups[1])
97
-
98
- return cast(List[client.V1Pod], v1.list_namespaced_pod(namespace, label_selector=label_selector).items)
99
-
100
- def show_schedules(self, state: ReplState, filter: Callable[[list[dict]], dict] = None):
101
- schedules = self.list_schedules(state, filter=filter)
102
- # forced refresh of schedule list
103
- if not filter:
104
- self.schedules_ids_by_cluster[state.sts] = [schedule['id'] for schedule in schedules]
105
- self.show_schedules_tabular(schedules)
106
-
107
- def schedule_ids(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None):
108
- schedules = self.list_schedules(state, show_output=show_output, filter=filter)
109
- return [schedule['id'] for schedule in schedules]
110
-
111
- def list_schedules(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None) -> list[dict]:
112
- def body(uri: str, headers: dict[str, str]):
113
- return requests.get(uri, headers=headers)
114
-
115
- response = self.port_forwarded(state, 'repair_schedule', body, method='GET', show_output=show_output)
116
- if not response:
117
- return
118
-
119
- res = response.json()
120
- if filter:
121
- res = filter(res)
122
-
123
- return res
124
-
125
- def show_schedules_tabular(self, schedules: list[dict]):
126
- log2(lines_to_tabular([f"{schedule['id']} {schedule['state']} {schedule['cluster_name']} {schedule['keyspace_name']}" for schedule in schedules], 'ID STATE CLUSTER KEYSPACE'))
127
-
128
- def show_schedule(self, state: ReplState, schedule_id: str):
129
- def filter(schedules: list[dict]):
130
- return [schedule for schedule in schedules if schedule['id'] == schedule_id]
131
-
132
- self.show_schedules(state, filter)
133
-
134
- def reaper_spec(self, state: ReplState) -> dict[str, any]:
135
- user, pw = state.user_pass(secret_path='reaper.secret')
136
- local_port = Config().get('reaper.port-forward.local-port', 9001)
137
-
138
- return {
139
- 'pod': self.pod,
140
- 'exec': f'kubectl exec -it {self.pod} -n {state.namespace} -- bash',
141
- 'forward': f'kubectl port-forward pods/{self.pod} -n {state.namespace} {local_port}:8080',
142
- 'web-uri': f'http://localhost:{local_port}/webui',
143
- 'username': user,
144
- 'password': pw
145
- }
146
-
147
- def cached_schedule_ids(state: ReplState) -> list[str]:
148
- if state.sts in ReaperSession.schedules_ids_by_cluster:
149
- return ReaperSession.schedules_ids_by_cluster[state.sts]
150
-
151
- if reaper := ReaperSession.create(state):
152
- Config().wait_log('Inspecting Cassandra Reaper...')
153
-
154
- schedules = reaper.schedule_ids(state, show_output = False)
155
- ReaperSession.schedules_ids_by_cluster[state.sts] = schedules
156
-
157
- return schedules
158
-
159
- return []