kaqing 2.0.86__py3-none-any.whl → 2.0.88__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/commands/audit/audit.py +13 -2
- adam/embedded_params.py +1 -1
- adam/repl.py +1 -1
- adam/sql/sql_completer.py +12 -6
- adam/sql/state_machine.py +25 -11
- adam/utils_athena.py +42 -12
- adam/version.py +1 -1
- {kaqing-2.0.86.dist-info → kaqing-2.0.88.dist-info}/METADATA +1 -1
- {kaqing-2.0.86.dist-info → kaqing-2.0.88.dist-info}/RECORD +12 -12
- {kaqing-2.0.86.dist-info → kaqing-2.0.88.dist-info}/WHEEL +0 -0
- {kaqing-2.0.86.dist-info → kaqing-2.0.88.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.86.dist-info → kaqing-2.0.88.dist-info}/top_level.txt +0 -0
adam/commands/audit/audit.py
CHANGED
|
@@ -2,10 +2,11 @@ import click
|
|
|
2
2
|
|
|
3
3
|
from adam.commands.audit.audit_repair_tables import AuditRepairTables
|
|
4
4
|
from adam.commands.command import Command
|
|
5
|
+
from adam.config import Config
|
|
5
6
|
from adam.repl_state import ReplState
|
|
6
7
|
from adam.sql.sql_completer import SqlCompleter
|
|
7
8
|
from adam.utils import log2
|
|
8
|
-
from adam.utils_athena import audit_table_names, run_audit_query
|
|
9
|
+
from adam.utils_athena import audit_column_names, audit_table_names, run_audit_query
|
|
9
10
|
|
|
10
11
|
class Audit(Command):
|
|
11
12
|
COMMAND = 'audit'
|
|
@@ -18,6 +19,7 @@ class Audit(Command):
|
|
|
18
19
|
|
|
19
20
|
def __init__(self, successor: Command=None):
|
|
20
21
|
super().__init__(successor)
|
|
22
|
+
self.schema_read = False
|
|
21
23
|
|
|
22
24
|
def command(self):
|
|
23
25
|
return Audit.COMMAND
|
|
@@ -45,7 +47,16 @@ class Audit(Command):
|
|
|
45
47
|
|
|
46
48
|
def completion(self, state: ReplState):
|
|
47
49
|
if state.device == ReplState.L:
|
|
48
|
-
|
|
50
|
+
if not self.schema_read:
|
|
51
|
+
Config().wait_log(f'Inspecting audit database schema...')
|
|
52
|
+
self.schema_read = True
|
|
53
|
+
audit_column_names()
|
|
54
|
+
|
|
55
|
+
def columns(_):
|
|
56
|
+
return audit_column_names()
|
|
57
|
+
|
|
58
|
+
return super().completion(state) | SqlCompleter.completions(lambda: audit_table_names(), columns=columns) | {
|
|
59
|
+
'desc': {table: None for table in audit_table_names()}}
|
|
49
60
|
|
|
50
61
|
return {}
|
|
51
62
|
|
adam/embedded_params.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
def config():
|
|
2
|
-
return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'athena': {'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'output': 's3://s3.ops--audit/ddl/results'}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'a', 'auto-enter-app': 'c3/c3', 'auto-enter-only-cluster': True}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}
|
|
2
|
+
return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results'}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'a', 'auto-enter-app': 'c3/c3', 'auto-enter-only-cluster': True}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}
|
adam/repl.py
CHANGED
|
@@ -175,7 +175,7 @@ def enter_repl(state: ReplState):
|
|
|
175
175
|
log2(f'Timing command {cmd}: {time.time() - s0:.2f}')
|
|
176
176
|
|
|
177
177
|
# offload audit logging
|
|
178
|
-
if cmd:
|
|
178
|
+
if cmd and (state.device != ReplState.L or Config().get('audit.log-audit-queries', False)):
|
|
179
179
|
executor.submit(audit_log, cmd, state)
|
|
180
180
|
|
|
181
181
|
def audit_log(cmd: str, state: ReplState):
|
adam/sql/sql_completer.py
CHANGED
|
@@ -13,11 +13,15 @@ __all__ = [
|
|
|
13
13
|
|
|
14
14
|
DML_COMPLETER = TermCompleter(['select', 'insert', 'delete', 'update'])
|
|
15
15
|
|
|
16
|
+
def default_columns(tables: list[str]):
|
|
17
|
+
return 'id,x.,y.,z.'.split(',')
|
|
18
|
+
|
|
16
19
|
class SqlCompleter(Completer):
|
|
17
|
-
def __init__(self, tables: Callable[[], list[str]], dml: str = None, debug = False):
|
|
20
|
+
def __init__(self, tables: Callable[[], list[str]], dml: str = None, columns: Callable[[list[str]], list[str]] = default_columns, debug = False):
|
|
18
21
|
super().__init__()
|
|
19
22
|
self.dml = dml
|
|
20
23
|
self.tables = tables
|
|
24
|
+
self.columns = columns
|
|
21
25
|
self.debug = debug
|
|
22
26
|
self.machine = StateMachine(debug=self.debug)
|
|
23
27
|
|
|
@@ -49,6 +53,8 @@ class SqlCompleter(Completer):
|
|
|
49
53
|
for word in self.machine.suggestions[state.to_s].strip(' ').split(','):
|
|
50
54
|
if word == 'tables':
|
|
51
55
|
terms.extend(self.tables())
|
|
56
|
+
elif word == 'columns':
|
|
57
|
+
terms.extend(self.columns([]))
|
|
52
58
|
elif word == 'single':
|
|
53
59
|
terms.append("'")
|
|
54
60
|
elif word == 'comma':
|
|
@@ -63,10 +69,10 @@ class SqlCompleter(Completer):
|
|
|
63
69
|
for c in completer.get_completions(document, complete_event):
|
|
64
70
|
yield c
|
|
65
71
|
|
|
66
|
-
def completions(table_names: Callable[[], list[str]]):
|
|
72
|
+
def completions(table_names: Callable[[], list[str]], columns: Callable[[list[str]], list[str]] = default_columns):
|
|
67
73
|
return {
|
|
68
|
-
'delete': SqlCompleter(table_names, 'delete'),
|
|
69
|
-
'insert': SqlCompleter(table_names, 'insert'),
|
|
70
|
-
'select': SqlCompleter(table_names, 'select'),
|
|
71
|
-
'update': SqlCompleter(table_names, 'update'),
|
|
74
|
+
'delete': SqlCompleter(table_names, 'delete', columns=columns),
|
|
75
|
+
'insert': SqlCompleter(table_names, 'insert', columns=columns),
|
|
76
|
+
'select': SqlCompleter(table_names, 'select', columns=columns),
|
|
77
|
+
'update': SqlCompleter(table_names, 'update', columns=columns),
|
|
72
78
|
}
|
adam/sql/state_machine.py
CHANGED
|
@@ -141,10 +141,12 @@ SPEC = [
|
|
|
141
141
|
'select_from_x > , > select_from_x_comma_',
|
|
142
142
|
'select_from_sq_ > as > select_from_x_as ^ as',
|
|
143
143
|
'select_from_x_comma_ > name > select_from_x ^ tables',
|
|
144
|
-
'select_from_x_ ^ as,where,inner join,left outer join,right outer join,full outer join,group by,limit',
|
|
145
|
-
'select_from_x_as_x_ > , > select_from_x_comma_ ^ where,inner join,left outer join,right outer join,full outer join,group by,limit',
|
|
144
|
+
'select_from_x_ ^ as,where,inner join,left outer join,right outer join,full outer join,group by,order by,limit',
|
|
145
|
+
'select_from_x_as_x_ > , > select_from_x_comma_ ^ where,inner join,left outer join,right outer join,full outer join,group by,order by,limit',
|
|
146
146
|
'- > as > select_from_x_as',
|
|
147
147
|
'- > where > select_where',
|
|
148
|
+
'- > order > select_order',
|
|
149
|
+
'- > order by > select_order_by',
|
|
148
150
|
'- > limit > select_where_sc_limit',
|
|
149
151
|
'- > group > select_group',
|
|
150
152
|
'- > group by > select_group_by',
|
|
@@ -161,8 +163,8 @@ SPEC = [
|
|
|
161
163
|
'select_from_x_as_ > name > select_from_x_as_x ^ x,y,z',
|
|
162
164
|
'select_from_x_as_x > , > select_from_x_as_x_comma_',
|
|
163
165
|
'select_from_x_as_x_comma_ > name > select_from_x ^ tables',
|
|
164
|
-
'select_where_ > name > select_where_a ^
|
|
165
|
-
'select_where_a > comparison > select_where_a_op ^
|
|
166
|
+
'select_where_ > name > select_where_a ^ columns',
|
|
167
|
+
'select_where_a > comparison > select_where_a_op ^ =,<,<=,>,>=,<>',
|
|
166
168
|
'select_where_a_ > comparison > select_where_a_op ^ =,<,<=,>,>=,<>,like,not,in',
|
|
167
169
|
'- > not > select_where_a_not',
|
|
168
170
|
'- > in > select_where_a_in',
|
|
@@ -177,15 +179,27 @@ SPEC = [
|
|
|
177
179
|
'select_where_a_in_lp_a_comma_ > name|single|num > select_where_a_in_lp_a ^ single',
|
|
178
180
|
'select_where_a_not_op > name|single|num > select_where_sc ^ single',
|
|
179
181
|
'select_where_a_op > name|single|num > select_where_sc ^ single',
|
|
180
|
-
'select_where_sc_ > and|or > select_where ^ and,or,group by,limit',
|
|
182
|
+
'select_where_sc_ > and|or > select_where ^ and,or,order by,group by,limit',
|
|
181
183
|
'- > group > select_group',
|
|
182
184
|
'- > group by > select_group_by',
|
|
185
|
+
'- > order > select_order',
|
|
186
|
+
'- > order by > select_order_by',
|
|
183
187
|
'- > limit > select_where_sc_limit',
|
|
184
188
|
'select_group_ > by > select_group_by ^ by',
|
|
185
|
-
'select_group_by_ > name > select_group_by_a ^
|
|
189
|
+
'select_group_by_ > name > select_group_by_a ^ columns',
|
|
186
190
|
'select_group_by_a > , > select_group_by_a_comma_',
|
|
187
|
-
'select_group_by_a_comma_ > name > select_group_by_a ^
|
|
188
|
-
'select_group_by_a_ > limit > select_where_sc_limit ^ limit',
|
|
191
|
+
'select_group_by_a_comma_ > name > select_group_by_a ^ columns',
|
|
192
|
+
'select_group_by_a_ > limit > select_where_sc_limit ^ limit,order by',
|
|
193
|
+
'- > order > select_order',
|
|
194
|
+
'- > order by > select_order_by',
|
|
195
|
+
'select_order_ > by > select_order_by ^ by',
|
|
196
|
+
'select_order_by_ > name > select_order_by_a ^ columns',
|
|
197
|
+
'select_order_by_a > , > select_order_by_a_comma_',
|
|
198
|
+
'select_order_by_a_comma_ > name > select_order_by_a ^ columns',
|
|
199
|
+
'select_order_by_a_ > desc|asc > select_order_by_a_desc ^ desc,asc,limit',
|
|
200
|
+
'- > limit > select_where_sc_limit',
|
|
201
|
+
'select_order_by_a_desc > , > select_order_by_a_comma_',
|
|
202
|
+
'select_order_by_a_desc_ > limit > select_where_sc_limit ^ limit',
|
|
189
203
|
'select_where_sc_limit_ > num > select_where_sc_limit_num ^ 1',
|
|
190
204
|
'select_where_sc_limit_num_rp__ > as > select_from_x_as ^ as',
|
|
191
205
|
'select_where_x_inner_ > join > select_join',
|
|
@@ -203,9 +217,9 @@ SPEC = [
|
|
|
203
217
|
'- > on > select_x_join_y_on ^ as,on',
|
|
204
218
|
'select_x_join_y_as_ > name > select_x_join_y_as_y ^ x,y,z',
|
|
205
219
|
'select_x_join_y_as_y_ > on > select_x_join_y_on ^ on',
|
|
206
|
-
'select_x_join_y_on_ > name > select_x_join_y_on_a ^
|
|
220
|
+
'select_x_join_y_on_ > name > select_x_join_y_on_a ^ columns',
|
|
207
221
|
'select_x_join_y_on_a > comparison > select_x_join_y_on_a_op ^ =',
|
|
208
|
-
'select_x_join_y_on_a_op > name > select_x_join_y_on_a_op_b ^
|
|
222
|
+
'select_x_join_y_on_a_op > name > select_x_join_y_on_a_op_b ^ columns',
|
|
209
223
|
'select_x_join_y_on_a_op_b > _ > select_from_x_as_x_',
|
|
210
224
|
|
|
211
225
|
|
|
@@ -264,7 +278,7 @@ SPEC = [
|
|
|
264
278
|
|
|
265
279
|
KEYWORDS = [
|
|
266
280
|
'select', 'from', 'as', 'not', 'in', 'where',
|
|
267
|
-
'and', 'or', 'group', 'by', 'group by', 'limit',
|
|
281
|
+
'and', 'or', 'group', 'by', 'group by', 'order', 'order by', 'limit', 'asc', 'desc',
|
|
268
282
|
'inner join', 'on', 'left', 'right', 'full', 'outer', 'left outer join',
|
|
269
283
|
'left join', 'right outer join', 'right join', 'full join', 'full outer join',
|
|
270
284
|
'insert', 'into', 'values',
|
adam/utils_athena.py
CHANGED
|
@@ -21,16 +21,52 @@ def audit_table_names():
|
|
|
21
21
|
|
|
22
22
|
return table_names
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
@functools.lru_cache()
|
|
25
|
+
def audit_column_names(tables: list[str] = [], database: str = None):
|
|
26
|
+
if not database:
|
|
27
|
+
database = Config().get('audit.athena.database', 'audit')
|
|
28
|
+
|
|
29
|
+
if not tables:
|
|
30
|
+
tables = Config().get('audit.athena.tables', 'audit').split(',')
|
|
31
|
+
|
|
32
|
+
table_names = "'" + "','".join([table.strip() for table in tables]) + "'"
|
|
33
|
+
|
|
34
|
+
query = f"select column_name from information_schema.columns where table_name in ({table_names}) and table_schema = '{database}'"
|
|
35
|
+
_, _, rs = audit_query(query)
|
|
36
|
+
if rs:
|
|
37
|
+
return [row['Data'][0].get('VarCharValue') for row in rs[1:]]
|
|
38
|
+
|
|
39
|
+
return []
|
|
40
|
+
|
|
41
|
+
def run_audit_query(sql: str, database: str = None):
|
|
42
|
+
state, reason, rs = audit_query(sql, database)
|
|
43
|
+
|
|
44
|
+
if state == 'SUCCEEDED':
|
|
45
|
+
if rs:
|
|
46
|
+
column_info = rs[0]['Data']
|
|
47
|
+
columns = [col.get('VarCharValue') for col in column_info]
|
|
48
|
+
lines = []
|
|
49
|
+
for row in rs[1:]:
|
|
50
|
+
row_data = [col.get('VarCharValue') for col in row['Data']]
|
|
51
|
+
lines.append('\t'.join(row_data))
|
|
52
|
+
|
|
53
|
+
log(lines_to_tabular(lines, header='\t'.join(columns), separator='\t'))
|
|
54
|
+
else:
|
|
55
|
+
log2(f"Query failed or was cancelled. State: {state}")
|
|
56
|
+
log2(f"Reason: {reason}")
|
|
57
|
+
|
|
58
|
+
def audit_query(sql: str, database: str = None) -> tuple[str, str, list]:
|
|
25
59
|
athena_client = boto3.client('athena')
|
|
26
60
|
|
|
27
|
-
|
|
61
|
+
if not database:
|
|
62
|
+
database = Config().get('audit.athena.database', 'audit')
|
|
63
|
+
|
|
28
64
|
s3_output_location = Config().get('audit.athena.output', 's3://s3.ops--audit/ddl/results')
|
|
29
65
|
|
|
30
66
|
response = athena_client.start_query_execution(
|
|
31
67
|
QueryString=sql,
|
|
32
68
|
QueryExecutionContext={
|
|
33
|
-
'Database':
|
|
69
|
+
'Database': database
|
|
34
70
|
},
|
|
35
71
|
ResultConfiguration={
|
|
36
72
|
'OutputLocation': s3_output_location
|
|
@@ -49,14 +85,8 @@ def run_audit_query(sql: str):
|
|
|
49
85
|
if state == 'SUCCEEDED':
|
|
50
86
|
results_response = athena_client.get_query_results(QueryExecutionId=query_execution_id)
|
|
51
87
|
if results_response['ResultSet']['Rows']:
|
|
52
|
-
|
|
53
|
-
columns = [col.get('VarCharValue') for col in column_info]
|
|
54
|
-
lines = []
|
|
55
|
-
for row in results_response['ResultSet']['Rows'][1:]:
|
|
56
|
-
row_data = [col.get('VarCharValue') for col in row['Data']]
|
|
57
|
-
lines.append('\t'.join(row_data))
|
|
88
|
+
return (state, None, results_response['ResultSet']['Rows'])
|
|
58
89
|
|
|
59
|
-
|
|
90
|
+
return (state, None, [])
|
|
60
91
|
else:
|
|
61
|
-
|
|
62
|
-
log2(f"Reason: {query_status['QueryExecution']['Status'].get('StateChangeReason')}")
|
|
92
|
+
return (state, query_status['QueryExecution']['Status'].get('StateChangeReason'), [])
|
adam/version.py
CHANGED
|
@@ -6,17 +6,17 @@ adam/cli.py,sha256=03pIZdomAu7IL-GSP6Eun_PKwwISShRAmfx6eVRPGC0,458
|
|
|
6
6
|
adam/cli_group.py,sha256=W3zy1BghCtVcEXizq8fBH-93ZRVVwgAyGPzy0sHno1Y,593
|
|
7
7
|
adam/config.py,sha256=hDxd86MbKVVKoHhTT077L3YRKYcr0i6xVADLlnoE684,2818
|
|
8
8
|
adam/embedded_apps.py,sha256=lKPx63mKzJbNmwz0rgL4gF76M9fDGxraYTtNAIGnZ_s,419
|
|
9
|
-
adam/embedded_params.py,sha256=
|
|
9
|
+
adam/embedded_params.py,sha256=FRG1eCyTHX5ynyt3Q8gc3EF2FeelBhW_mh94SSc6PXE,4878
|
|
10
10
|
adam/log.py,sha256=gg5DK52wLPc9cjykeh0WFHyAk1qI3HEpGaAK8W2dzXY,1146
|
|
11
11
|
adam/pod_exec_result.py,sha256=nq0xnCNOpUGBSijGF0H-YNrwBc9vUQs4DkvLMIFS5LQ,951
|
|
12
|
-
adam/repl.py,sha256=
|
|
12
|
+
adam/repl.py,sha256=xDG3l76O4LXyWhobmyvOd-Txs9ND-pQrTB0I9uhMU1I,9407
|
|
13
13
|
adam/repl_commands.py,sha256=1AFE8AJoyQdJd0tlcwtMnwMjV5G7jL9MDkxJZxdo150,4963
|
|
14
14
|
adam/repl_session.py,sha256=uIogcvWBh7wd8QQ-p_JgLsyJ8YJgINw5vOd6JIsd7Vo,472
|
|
15
15
|
adam/repl_state.py,sha256=dXyGlWXcSsfCjrYwMhU44PVn_oThSZ8dhJ5HCVE8-qU,8743
|
|
16
16
|
adam/utils.py,sha256=sbsNZP3qGJtb6fXCa4dDXHry5ay9ev583cCZIQzy07s,7382
|
|
17
|
-
adam/utils_athena.py,sha256=
|
|
17
|
+
adam/utils_athena.py,sha256=tU6Arg4g7eKV6ei9SLgakOJJqxKgSCsII-7a68OI7_g,3199
|
|
18
18
|
adam/utils_net.py,sha256=65fhBnWMCkhGtyHqz95qcHaCo35q-WX1RBkkXG8dKpI,416
|
|
19
|
-
adam/version.py,sha256=
|
|
19
|
+
adam/version.py,sha256=S-XCwdC7VoXNjayM8Pyp0O0jiDzTyl9G0bRjQb73Is4,139
|
|
20
20
|
adam/checks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
21
|
adam/checks/check.py,sha256=Qopr3huYcMu2bzQgb99dEUYjFzkjKHRI76S6KA9b9Rk,702
|
|
22
22
|
adam/checks/check_context.py,sha256=FEHkQ32jY1EDopQ2uYWqy9v7aEEX1orLpJWhopwAlh4,402
|
|
@@ -79,7 +79,7 @@ adam/commands/rollout.py,sha256=52_4ijna3v-8Oug12et43DRHFDNhiN34p6xLTQmhdbQ,2959
|
|
|
79
79
|
adam/commands/shell.py,sha256=wY_PIx7Lt6vuxhFArlfxdEnBbrouCJ3yNHhFn17DEqw,848
|
|
80
80
|
adam/commands/watch.py,sha256=mmBFpB8T1V7zrNs5b2YNyDDztMym_ILPDdkrbdAXTas,2438
|
|
81
81
|
adam/commands/audit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
82
|
-
adam/commands/audit/audit.py,sha256
|
|
82
|
+
adam/commands/audit/audit.py,sha256=uXyzGRRW4mfg4AAjxMe6yrTrC3UNQnZh9_8n7-Y3EVE,2324
|
|
83
83
|
adam/commands/audit/audit_repair_tables.py,sha256=5Glsy29md3KH_4_smuVBeu6c4zygICPK8Jiv-fNRC38,1186
|
|
84
84
|
adam/commands/cql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
85
85
|
adam/commands/cql/cql_completions.py,sha256=dXe51NTWEJis76587IWSn9Av-cjC0J6KMaxlBKfF4wM,411
|
|
@@ -170,8 +170,8 @@ adam/k8s_utils/services.py,sha256=EOJJGACVbbRvu5T3rMKqIJqgYic1_MSJ17EA0TJ6UOk,31
|
|
|
170
170
|
adam/k8s_utils/statefulsets.py,sha256=5g7KxGRHgEewT8rnZneDTaJDylUf-dHH2edWJEoorr8,4667
|
|
171
171
|
adam/k8s_utils/volumes.py,sha256=RIBmlOSWM3V3QVXLCFT0owVOyh4rGG1ETp521a-6ndo,1137
|
|
172
172
|
adam/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
173
|
-
adam/sql/sql_completer.py,sha256=
|
|
174
|
-
adam/sql/state_machine.py,sha256=
|
|
173
|
+
adam/sql/sql_completer.py,sha256=zH2I-WLc7aZmdIg0PlHPVLdxemI8oN8YTCnG7qxcYqA,2907
|
|
174
|
+
adam/sql/state_machine.py,sha256=CwpK6ierTESCsRjCDOs4YS9TGFPsey5dhn1O-8VY5i4,24489
|
|
175
175
|
adam/sql/term_completer.py,sha256=bNnHAVf9NZl52xS_BQpikbOK39gDBJADnT9gSvG0iqI,2539
|
|
176
176
|
adam/sso/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
177
177
|
adam/sso/authenticator.py,sha256=BCm16L9zf5aLU47-sTCnudn2zLPwd8M2wwRminJfsqw,615
|
|
@@ -183,8 +183,8 @@ adam/sso/idp.py,sha256=fvcwUw_URTgsO6ySaqTIw0zQT2qRO1IPSGhf6rPtybo,5804
|
|
|
183
183
|
adam/sso/idp_login.py,sha256=QAtCUeDTVWliJy40RK_oac8Vgybr13xH8wzeBoxPaa8,1754
|
|
184
184
|
adam/sso/idp_session.py,sha256=9BUHNRf70u4rVKrVY1HKPOEmOviXvkjam8WJxmXSKIM,1735
|
|
185
185
|
adam/sso/sso_config.py,sha256=5N8WZgIJQBtHUy585XLRWKjpU87_v6QluyNK9E27D5s,2459
|
|
186
|
-
kaqing-2.0.
|
|
187
|
-
kaqing-2.0.
|
|
188
|
-
kaqing-2.0.
|
|
189
|
-
kaqing-2.0.
|
|
190
|
-
kaqing-2.0.
|
|
186
|
+
kaqing-2.0.88.dist-info/METADATA,sha256=0eswLcaqx6RfV4TO6uBdGAVTBMT7yrsYCwsk7tjYs18,132
|
|
187
|
+
kaqing-2.0.88.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
188
|
+
kaqing-2.0.88.dist-info/entry_points.txt,sha256=SkzhuQJUWsXOzHeZ5TgQ2c3_g53UGK23zzJU_JTZOZI,39
|
|
189
|
+
kaqing-2.0.88.dist-info/top_level.txt,sha256=8_2PZkwBb-xDcnc8a2rAbQeJhXKXskc7zTP7pSPa1fw,5
|
|
190
|
+
kaqing-2.0.88.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|