amulog 0.3.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amulog/__init__.py +10 -0
- amulog/__main__.py +677 -0
- amulog/alg/__init__.py +1 -0
- amulog/alg/crf/__init__.py +2 -0
- amulog/alg/crf/__main__.py +176 -0
- amulog/alg/crf/_convert.py +130 -0
- amulog/alg/crf/_items.py +56 -0
- amulog/alg/crf/lt_crf.py +588 -0
- amulog/alg/crf/train.py +123 -0
- amulog/alg/dlog/__init__.py +2 -0
- amulog/alg/dlog/dlog.py +147 -0
- amulog/alg/drain/__init__.py +4 -0
- amulog/alg/drain/drain.py +136 -0
- amulog/alg/fttree/__init__.py +4 -0
- amulog/alg/fttree/fttree.py +159 -0
- amulog/alg/lenma/__init__.py +4 -0
- amulog/alg/lenma/lenma.py +123 -0
- amulog/alg/meta.py +22 -0
- amulog/alg/shiso/__init__.py +3 -0
- amulog/alg/shiso/shiso.py +247 -0
- amulog/anonymize.py +183 -0
- amulog/cli.py +63 -0
- amulog/common.py +453 -0
- amulog/config.py +623 -0
- amulog/data/config.conf.default +345 -0
- amulog/data/crf_template.default +12 -0
- amulog/data/crf_template.sample +10 -0
- amulog/data/drain_regex.conf +9 -0
- amulog/data/host_alias.txt.sample +26 -0
- amulog/data/lt_label.conf.sample +55 -0
- amulog/data/testlog.conf +84 -0
- amulog/db_common.py +232 -0
- amulog/db_mysql.py +126 -0
- amulog/db_sqlite.py +110 -0
- amulog/edit/__init__.py +0 -0
- amulog/edit/__main__.py +331 -0
- amulog/edit/lt_tool.py +288 -0
- amulog/edit/search.py +82 -0
- amulog/eval/__init__.py +0 -0
- amulog/eval/__main__.py +421 -0
- amulog/eval/cluster_metrics.py +202 -0
- amulog/eval/maketpl.py +964 -0
- amulog/eval/param_searcher.py +102 -0
- amulog/eval/structure_metrics.py +86 -0
- amulog/external/__init__.py +2 -0
- amulog/external/mod_tplseq.py +57 -0
- amulog/external/regexhash.py +77 -0
- amulog/external/tpl_match.py +63 -0
- amulog/host_alias.py +165 -0
- amulog/log_db.py +1203 -0
- amulog/logparser.py +307 -0
- amulog/lt_common.py +722 -0
- amulog/lt_import.py +87 -0
- amulog/lt_import_ext.py +102 -0
- amulog/lt_label.py +159 -0
- amulog/lt_misc.py +127 -0
- amulog/lt_regex.py +134 -0
- amulog/lt_search.py +420 -0
- amulog/lt_va.py +121 -0
- amulog/ltg_semantics.py +253 -0
- amulog/manager.py +670 -0
- amulog/mproc_queue.py +112 -0
- amulog/strutil.py +46 -0
- amulog/testutil.py +219 -0
- amulog-0.3.14.dist-info/METADATA +261 -0
- amulog-0.3.14.dist-info/RECORD +92 -0
- amulog-0.3.14.dist-info/WHEEL +5 -0
- amulog-0.3.14.dist-info/entry_points.txt +4 -0
- amulog-0.3.14.dist-info/licenses/LICENSE +11 -0
- amulog-0.3.14.dist-info/top_level.txt +3 -0
- tests/__init__.py +0 -0
- tests/test_cli.py +44 -0
- tests/test_cluster_metrics.py +69 -0
- tests/test_common.py +59 -0
- tests/test_config.py +97 -0
- tests/test_crf.py +89 -0
- tests/test_db.py +241 -0
- tests/test_edit.py +71 -0
- tests/test_host_alias.py +153 -0
- tests/test_lenma_cluster.py +60 -0
- tests/test_lt_common.py +65 -0
- tests/test_lt_misc.py +164 -0
- tests/test_lt_regex.py +35 -0
- tests/test_lt_search.py +104 -0
- tests/test_lt_va.py +61 -0
- tests/test_ltg_semantics.py +93 -0
- tests/test_ltgen.py +125 -0
- tests/test_maketpl.py +78 -0
- tests/test_postprocess.py +164 -0
- tests/test_shiso.py +62 -0
- tests-ext/__init__.py +0 -0
- tests-ext/test_mysql.py +78 -0
amulog/__init__.py
ADDED
amulog/__main__.py
ADDED
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# coding: utf-8
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
Interface to use some functions about Log DB from CLI.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import datetime
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
from . import cli
|
|
12
|
+
from . import common
|
|
13
|
+
from . import config
|
|
14
|
+
|
|
15
|
+
_logger = logging.getLogger(__package__)
|
|
16
|
+
SUBLIB = ["edit", "eval"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_targets_arg(ns):
|
|
20
|
+
if ns.recur:
|
|
21
|
+
targets = common.recur_dir(ns.files)
|
|
22
|
+
else:
|
|
23
|
+
targets = common.rep_dir(ns.files)
|
|
24
|
+
return targets
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_targets_conf(conf):
|
|
28
|
+
l_path = config.getlist(conf, "general", "src_path")
|
|
29
|
+
if conf.getboolean("general", "src_recur"):
|
|
30
|
+
return common.recur_dir(l_path)
|
|
31
|
+
else:
|
|
32
|
+
return common.rep_dir(l_path)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_targets(ns, conf):
|
|
36
|
+
if ns is None or len(ns.files) == 0:
|
|
37
|
+
return get_targets_conf(conf)
|
|
38
|
+
else:
|
|
39
|
+
return get_targets_arg(ns)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def is_online(conf, parallel):
|
|
43
|
+
from .alg import is_online as alg_is_online
|
|
44
|
+
mode = conf["log_template"]["processing_mode"]
|
|
45
|
+
lt_methods = config.getlist(conf, "log_template", "lt_methods")
|
|
46
|
+
return alg_is_online(mode, lt_methods, parallel)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def data_from_db(ns):
|
|
50
|
+
conf = config.open_config(ns.conf_path)
|
|
51
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
52
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
53
|
+
dirname = ns.dirname
|
|
54
|
+
if ns.incr:
|
|
55
|
+
method = "incremental"
|
|
56
|
+
else:
|
|
57
|
+
method = "commit"
|
|
58
|
+
reset = ns.reset
|
|
59
|
+
|
|
60
|
+
from . import log_db
|
|
61
|
+
log_db.data_from_db(conf, dirname, method, reset)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def data_from_data(ns):
|
|
65
|
+
conf = config.open_config(ns.conf_path)
|
|
66
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
67
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
68
|
+
dirname = ns.dirname
|
|
69
|
+
targets = get_targets(ns, conf)
|
|
70
|
+
if ns.incr:
|
|
71
|
+
method = "incremental"
|
|
72
|
+
else:
|
|
73
|
+
method = "commit"
|
|
74
|
+
reset = ns.reset
|
|
75
|
+
|
|
76
|
+
from . import manager
|
|
77
|
+
manager.data_from_data(conf, targets, dirname, method, reset)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def data_parse(ns):
|
|
81
|
+
conf = config.open_config(ns.conf_path)
|
|
82
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
83
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
84
|
+
targets = get_targets(ns, conf)
|
|
85
|
+
|
|
86
|
+
from . import strutil
|
|
87
|
+
from . import manager
|
|
88
|
+
lp = manager.load_log2seq(conf)
|
|
89
|
+
for line in manager.iter_lines(targets):
|
|
90
|
+
pline = manager.parse_line(strutil.add_esc(line), lp)
|
|
91
|
+
if pline is None:
|
|
92
|
+
pass
|
|
93
|
+
elif ns.words:
|
|
94
|
+
print(" ".join(pline["words"]))
|
|
95
|
+
else:
|
|
96
|
+
print(pline)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def db_make(ns):
|
|
100
|
+
conf = config.open_config(ns.conf_path)
|
|
101
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
102
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
103
|
+
targets = get_targets(ns, conf)
|
|
104
|
+
|
|
105
|
+
from . import manager
|
|
106
|
+
timer = common.Timer("db-make", output=_logger)
|
|
107
|
+
timer.start()
|
|
108
|
+
if is_online(conf, ns.parallel):
|
|
109
|
+
manager.process_files_online(conf, targets, True)
|
|
110
|
+
else:
|
|
111
|
+
manager.process_files_offline(conf, targets, True, ns.parallel)
|
|
112
|
+
timer.stop()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def db_add(ns):
|
|
116
|
+
conf = config.open_config(ns.conf_path)
|
|
117
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
118
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
119
|
+
targets = get_targets_arg(ns)
|
|
120
|
+
|
|
121
|
+
from . import manager
|
|
122
|
+
|
|
123
|
+
timer = common.Timer("db-add", output=_logger)
|
|
124
|
+
timer.start()
|
|
125
|
+
if is_online(conf, ns.parallel):
|
|
126
|
+
manager.process_files_online(conf, targets, False)
|
|
127
|
+
else:
|
|
128
|
+
manager.process_files_offline(conf, targets, False, ns.parallel)
|
|
129
|
+
timer.stop()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def db_remake_group(ns):
|
|
133
|
+
conf = config.open_config(ns.conf_path)
|
|
134
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
135
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
136
|
+
|
|
137
|
+
from . import manager
|
|
138
|
+
timer = common.Timer("db-remake-group", output=_logger)
|
|
139
|
+
timer.start()
|
|
140
|
+
manager.remake_ltgroup(conf)
|
|
141
|
+
timer.stop()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def db_tag(ns):
|
|
145
|
+
conf = config.open_config(ns.conf_path)
|
|
146
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
147
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
148
|
+
|
|
149
|
+
from . import lt_label
|
|
150
|
+
timer = common.Timer("db-tag", output=_logger)
|
|
151
|
+
timer.start()
|
|
152
|
+
lt_label.generate_all_tags(conf)
|
|
153
|
+
timer.stop()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def db_repair(ns):
|
|
157
|
+
conf = config.open_config(ns.conf_path)
|
|
158
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
159
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
160
|
+
|
|
161
|
+
from . import log_db
|
|
162
|
+
db = log_db.LogDB(conf, edit=True, reset_db=False)
|
|
163
|
+
db.repair_tables()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def db_anonymize(ns):
|
|
167
|
+
conf = config.open_config(ns.conf_path)
|
|
168
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
169
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
170
|
+
|
|
171
|
+
timer = common.Timer("db-anonymize", output=_logger)
|
|
172
|
+
timer.start()
|
|
173
|
+
if ns.conf_export:
|
|
174
|
+
conf2 = config.open_config(ns.conf_export)
|
|
175
|
+
else:
|
|
176
|
+
conf2 = None
|
|
177
|
+
from . import anonymize
|
|
178
|
+
am = anonymize.AnonymizeMapper(conf)
|
|
179
|
+
am.anonymize(conf2)
|
|
180
|
+
fp = am.dump()
|
|
181
|
+
timer.stop()
|
|
182
|
+
|
|
183
|
+
print("> " + fp)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def db_anonymize_mapping(ns):
|
|
187
|
+
conf = config.open_config(ns.conf_path)
|
|
188
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
189
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
190
|
+
|
|
191
|
+
from . import anonymize
|
|
192
|
+
am = anonymize.AnonymizeMapper(conf)
|
|
193
|
+
am.mapping()
|
|
194
|
+
fp = am.dump()
|
|
195
|
+
|
|
196
|
+
print("> " + fp)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def clean(ns):
|
|
200
|
+
conf = config.open_config(ns.conf_path)
|
|
201
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
202
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
203
|
+
|
|
204
|
+
from . import manager
|
|
205
|
+
manager.clean(conf)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def show_db_info(ns):
|
|
209
|
+
conf = config.open_config(ns.conf_path)
|
|
210
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
211
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
212
|
+
|
|
213
|
+
from . import log_db
|
|
214
|
+
if ns.dt_range is None:
|
|
215
|
+
log_db.info(conf)
|
|
216
|
+
else:
|
|
217
|
+
from dateutil import parser
|
|
218
|
+
dts = parser.parse(ns.dt_range[0])
|
|
219
|
+
dte = parser.parse(ns.dt_range[1])
|
|
220
|
+
log_db.info_term(conf, dts, dte)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def show_lt(ns):
|
|
224
|
+
conf = config.open_config(ns.conf_path)
|
|
225
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
226
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
227
|
+
from . import log_db
|
|
228
|
+
|
|
229
|
+
print(log_db.show_all_lt(conf, simple=ns.simple))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def show_ltg(ns):
|
|
233
|
+
conf = config.open_config(ns.conf_path)
|
|
234
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
235
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
236
|
+
from . import log_db
|
|
237
|
+
|
|
238
|
+
print(log_db.show_all_ltg(conf))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def show_tag(ns):
|
|
242
|
+
conf = config.open_config(ns.conf_path)
|
|
243
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
244
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
245
|
+
from . import log_db
|
|
246
|
+
|
|
247
|
+
print(log_db.show_tag(conf, tag=ns.tag))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def show_tag_stats(ns):
|
|
251
|
+
conf = config.open_config(ns.conf_path)
|
|
252
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
253
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
254
|
+
from . import log_db
|
|
255
|
+
|
|
256
|
+
print(log_db.show_tag_stats(conf))
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def show_lt_import(ns):
|
|
260
|
+
conf = config.open_config(ns.conf_path)
|
|
261
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
262
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
263
|
+
from . import log_db
|
|
264
|
+
|
|
265
|
+
external = ns.external
|
|
266
|
+
ld = log_db.LogData(conf)
|
|
267
|
+
for ltobj in ld.iter_lt():
|
|
268
|
+
if external:
|
|
269
|
+
# unsegmented, with escape
|
|
270
|
+
print(ltobj.restore_message(None, esc=True))
|
|
271
|
+
else:
|
|
272
|
+
# segmented, with escape
|
|
273
|
+
print(" ".join(ltobj.ltw))
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def show_host(ns):
|
|
277
|
+
conf = config.open_config(ns.conf_path)
|
|
278
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
279
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
280
|
+
from . import log_db
|
|
281
|
+
|
|
282
|
+
log_db.show_all_host(conf)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def show_log(ns):
|
|
286
|
+
conf = config.open_config(ns.conf_path)
|
|
287
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
288
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
289
|
+
from . import log_db
|
|
290
|
+
lidflag = ns.lid
|
|
291
|
+
|
|
292
|
+
d = parse_condition(ns.conditions)
|
|
293
|
+
ld = log_db.LogData(conf)
|
|
294
|
+
for e in ld.iter_lines(**d):
|
|
295
|
+
if lidflag:
|
|
296
|
+
print("{0} {1}".format(e.lid, e.restore_line()))
|
|
297
|
+
else:
|
|
298
|
+
print(e.restore_line())
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def parse_condition(conditions):
|
|
302
|
+
"""
|
|
303
|
+
Args:
|
|
304
|
+
conditions (list)
|
|
305
|
+
"""
|
|
306
|
+
from dateutil import parser
|
|
307
|
+
|
|
308
|
+
d = {}
|
|
309
|
+
for arg in conditions:
|
|
310
|
+
if "=" not in arg:
|
|
311
|
+
raise SyntaxError
|
|
312
|
+
key = arg.partition("=")[0]
|
|
313
|
+
if key == "ltid":
|
|
314
|
+
d["ltid"] = int(arg.partition("=")[-1])
|
|
315
|
+
elif key in ("gid", "ltgid"):
|
|
316
|
+
d["ltgid"] = int(arg.partition("=")[-1])
|
|
317
|
+
elif key in ("time_from", "dts"):
|
|
318
|
+
time_string = arg.partition("=")[-1]
|
|
319
|
+
time_string = time_string.strip('"' + "'")
|
|
320
|
+
d["dts"] = parser.parse(time_string)
|
|
321
|
+
elif key in ("time_to", "dte"):
|
|
322
|
+
time_string = arg.partition("=")[-1]
|
|
323
|
+
time_string = time_string.strip('"' + "'")
|
|
324
|
+
d["dte"] = parser.parse(time_string)
|
|
325
|
+
# elif key in ("date_from", "top_date"):
|
|
326
|
+
# date_string = arg.partition("=")[-1]
|
|
327
|
+
# d["dts"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
|
|
328
|
+
# elif key in ("date_to", "end_date"):
|
|
329
|
+
# date_string = arg.partition("=")[-1]
|
|
330
|
+
# d["dte"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
|
|
331
|
+
elif key == "date":
|
|
332
|
+
date_string = arg.partition("=")[-1]
|
|
333
|
+
d["dts"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
|
|
334
|
+
d["dte"] = d["dts"] + datetime.timedelta(days=1)
|
|
335
|
+
elif key == "host":
|
|
336
|
+
d["host"] = arg.partition("=")[-1]
|
|
337
|
+
elif key == "host_like":
|
|
338
|
+
d["host_like"] = arg.partition("=")[-1]
|
|
339
|
+
elif key == "host_regexp":
|
|
340
|
+
d["host_regexp"] = arg.partition("=")[-1]
|
|
341
|
+
else:
|
|
342
|
+
raise ValueError
|
|
343
|
+
return d
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def show_line(ns):
|
|
347
|
+
conf = config.open_config(ns.conf_path)
|
|
348
|
+
lv = logging.DEBUG if ns.debug else logging.INFO
|
|
349
|
+
config.set_common_logging(conf, logger=_logger, lv=lv)
|
|
350
|
+
from . import log_db
|
|
351
|
+
lid = ns.lid
|
|
352
|
+
|
|
353
|
+
ld = log_db.LogData(conf)
|
|
354
|
+
lm = ld.get_line(lid)
|
|
355
|
+
print(lm.restore_line())
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def conf_defaults(_):
|
|
359
|
+
config.show_default_config()
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def conf_diff(ns):
|
|
363
|
+
files = ns.files[:]
|
|
364
|
+
if ns.configset:
|
|
365
|
+
files += config.read_config_group(ns.configset)
|
|
366
|
+
config.show_config_diff(files)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def conf_minimum(ns):
|
|
370
|
+
conf = config.open_config(ns.conf_path, base_default=False, ignore_import=True)
|
|
371
|
+
conf = config.minimize(conf)
|
|
372
|
+
config.write(ns.conf_path, conf)
|
|
373
|
+
print("rewrite {0}".format(ns.conf_path))
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def conf_set_edit(ns):
|
|
377
|
+
l_conf_name = config.read_config_group(ns.configset)
|
|
378
|
+
key = ns.key
|
|
379
|
+
rulestr = ns.rule
|
|
380
|
+
|
|
381
|
+
if "(" in rulestr:
|
|
382
|
+
temp = rulestr.rstrip(")").split("(")
|
|
383
|
+
if not len(temp) == 2:
|
|
384
|
+
raise ValueError("bad format for value specification")
|
|
385
|
+
rulename, argstr = temp
|
|
386
|
+
args = argstr.split(",")
|
|
387
|
+
|
|
388
|
+
if rulename == "list":
|
|
389
|
+
assert len(args) == len(l_conf_name)
|
|
390
|
+
d = {key: args}
|
|
391
|
+
elif rulename == "range":
|
|
392
|
+
assert len(args) == 2
|
|
393
|
+
start, step = [int(v) for v in args]
|
|
394
|
+
l_val = [start + i * step for i in range(len(l_conf_name))]
|
|
395
|
+
d = {key: l_val}
|
|
396
|
+
elif rulename == "power":
|
|
397
|
+
assert len(args) == 2
|
|
398
|
+
start, step = [int(v) for v in args]
|
|
399
|
+
l_val = [start * (i ** step) for i in range(len(l_conf_name))]
|
|
400
|
+
d = {key: l_val}
|
|
401
|
+
elif rulename == "withconf":
|
|
402
|
+
assert len(args) == 1
|
|
403
|
+
l_val = [args[0] + name for i, name in enumerate(l_conf_name)]
|
|
404
|
+
d = {key: l_val}
|
|
405
|
+
elif rulename == "namerange":
|
|
406
|
+
assert len(args) == 1
|
|
407
|
+
l_val = [args[0] + str(i) for i in range(len(l_conf_name))]
|
|
408
|
+
d = {key: l_val}
|
|
409
|
+
else:
|
|
410
|
+
raise NotImplementedError("invalid rule name")
|
|
411
|
+
else:
|
|
412
|
+
l_val = [rulestr] * len(l_conf_name)
|
|
413
|
+
d = {key: l_val}
|
|
414
|
+
|
|
415
|
+
config.config_group_edit(l_conf_name, d)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def conf_shadow(ns):
|
|
419
|
+
cond = {}
|
|
420
|
+
incr = []
|
|
421
|
+
for rule in ns.rules:
|
|
422
|
+
if "=" in rule:
|
|
423
|
+
key, val = rule.split("=")
|
|
424
|
+
cond[key] = val
|
|
425
|
+
else:
|
|
426
|
+
incr.append(rule)
|
|
427
|
+
l_conf_name = config.config_shadow(n=ns.number, cond=cond, incr=incr,
|
|
428
|
+
fn=ns.conf_path, output=ns.output,
|
|
429
|
+
ignore_overwrite=ns.force)
|
|
430
|
+
|
|
431
|
+
if ns.configset is not None:
|
|
432
|
+
config.dump_config_group(ns.configset, l_conf_name)
|
|
433
|
+
print(ns.configset)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# common argument settings
|
|
437
|
+
OPT_DEBUG = [["--debug"],
|
|
438
|
+
{"dest": "debug", "action": "store_true",
|
|
439
|
+
"help": "set logging level to debug (default: info)"}]
|
|
440
|
+
OPT_CONFIG = [["-c", "--config"],
|
|
441
|
+
{"dest": "conf_path", "metavar": "CONFIG", "action": "store",
|
|
442
|
+
# "default": config.DEFAULT_CONFIG,
|
|
443
|
+
"default": None,
|
|
444
|
+
"help": "configuration file path for amulog"}]
|
|
445
|
+
OPT_CONFIG_SET = [["-s", "--configset"],
|
|
446
|
+
{"dest": "configset", "metavar": "CONFIG_SET",
|
|
447
|
+
"default": None,
|
|
448
|
+
"help": "use config group definition file"}]
|
|
449
|
+
OPT_PARALLEL = [["-p", "--parallel"],
|
|
450
|
+
{"dest": "parallel", "action": "store_true",
|
|
451
|
+
"help": "parallel processing in offline mode"}]
|
|
452
|
+
OPT_RECUR = [["-r", "--recur"],
|
|
453
|
+
{"dest": "recur", "action": "store_true",
|
|
454
|
+
"help": "recursively search files to process"}]
|
|
455
|
+
OPT_RESET = [["--reset"],
|
|
456
|
+
{"dest": "reset", "action": "store_true",
|
|
457
|
+
"help": "reset log template generation model"}]
|
|
458
|
+
OPT_DRY = [["-d", "--dry"],
|
|
459
|
+
{"dest": "dry", "action": "store_true",
|
|
460
|
+
"help": "do not store data into db"}]
|
|
461
|
+
OPT_TERM = [["-t", "--term"],
|
|
462
|
+
{"dest": "dt_range", "default": None,
|
|
463
|
+
"metavar": "DATE", "nargs": 2,
|
|
464
|
+
"help": ("datetime range (start end), e.g. -t 2012-10-10 "
|
|
465
|
+
"2012-10-20. Optional; without it, use all data.")}]
|
|
466
|
+
ARG_FILE = [["file"],
|
|
467
|
+
{"metavar": "PATH", "action": "store",
|
|
468
|
+
"help": "filepath to output"}]
|
|
469
|
+
ARG_FILES = [["files"],
|
|
470
|
+
{"metavar": "PATH", "nargs": "+",
|
|
471
|
+
"help": "files or directories as input"}]
|
|
472
|
+
ARG_FILES_OPT = [["files"],
|
|
473
|
+
{"metavar": "PATH", "nargs": "*",
|
|
474
|
+
"help": ("files or directories as input "
|
|
475
|
+
"(optional; defaultly read from config")}]
|
|
476
|
+
ARG_DBSEARCH = [["conditions"],
|
|
477
|
+
{"metavar": "CONDITION", "nargs": "+",
|
|
478
|
+
"help": ("Conditions to search log messages. "
|
|
479
|
+
"Example: MODE gid=24 date=2012-10-10 ..., "
|
|
480
|
+
"Keys: ltid, gid, date, time_from, time_to, host, "
|
|
481
|
+
"host_like, host_regexp")}]
|
|
482
|
+
ARG_LID = [["lid"],
|
|
483
|
+
{"metavar": "LID", "action": "store",
|
|
484
|
+
"help": "log message identifier"}]
|
|
485
|
+
|
|
486
|
+
# argument settings for each modes
|
|
487
|
+
# description, List[args, kwargs], func
|
|
488
|
+
# defined after functions because these settings use functions
|
|
489
|
+
DICT_ARGSET = {
|
|
490
|
+
"data-from-db": ["Generate log data from DB.",
|
|
491
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
492
|
+
[["-d", "--dirname"],
|
|
493
|
+
{"dest": "dirname", "metavar": "DIRNAME",
|
|
494
|
+
"action": "store",
|
|
495
|
+
"help": "directory name to output"}],
|
|
496
|
+
[["-i", "--incr"],
|
|
497
|
+
{"dest": "incr", "action": "store_true",
|
|
498
|
+
"help": "output incrementally, use with small memory"}],
|
|
499
|
+
[["--reset"],
|
|
500
|
+
{"dest": "reset", "action": "store_true",
|
|
501
|
+
"help": "reset log file directory before processing"}],
|
|
502
|
+
],
|
|
503
|
+
data_from_db],
|
|
504
|
+
"data-from-data": ["Re-arrange log file, splitting messages by date.",
|
|
505
|
+
[OPT_CONFIG, OPT_DEBUG, OPT_RECUR, ARG_FILES_OPT,
|
|
506
|
+
[["-d", "--dirname"],
|
|
507
|
+
{"dest": "dirname", "metavar": "DIRNAME",
|
|
508
|
+
"action": "store",
|
|
509
|
+
"help": "directory name to output"}],
|
|
510
|
+
[["-i", "--incr"],
|
|
511
|
+
{"dest": "incr", "action": "store_true",
|
|
512
|
+
"help": "output incrementally, use with small memory"}],
|
|
513
|
+
[["--reset"],
|
|
514
|
+
{"dest": "reset", "action": "store_true",
|
|
515
|
+
"help": "reset log file directory before processing"}],
|
|
516
|
+
],
|
|
517
|
+
data_from_data],
|
|
518
|
+
"data-parse": ["Check log data parsing with log2seq.",
|
|
519
|
+
[OPT_CONFIG, OPT_DEBUG, OPT_RECUR,
|
|
520
|
+
[["-w", "--words"],
|
|
521
|
+
{"dest": "words", "action": "store_true",
|
|
522
|
+
"help": "only show parsed words"}],
|
|
523
|
+
ARG_FILES_OPT],
|
|
524
|
+
data_parse],
|
|
525
|
+
"db-make": ["Initialize database and add log data. ",
|
|
526
|
+
[OPT_CONFIG, OPT_DEBUG, OPT_RECUR,
|
|
527
|
+
OPT_RESET, OPT_PARALLEL, ARG_FILES_OPT],
|
|
528
|
+
db_make],
|
|
529
|
+
"db-add": ["Add log data to existing database.",
|
|
530
|
+
[OPT_CONFIG, OPT_DEBUG, OPT_RECUR, OPT_PARALLEL, ARG_FILES],
|
|
531
|
+
db_add],
|
|
532
|
+
"db-remake-group": ["Remake log template groups",
|
|
533
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
534
|
+
db_remake_group],
|
|
535
|
+
"db-tag": ["Make log template tags.",
|
|
536
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
537
|
+
db_tag],
|
|
538
|
+
"db-repair": ["Repair db schema after version updates.",
|
|
539
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
540
|
+
db_repair],
|
|
541
|
+
"db-anonymize": ["Anonymize templates and hostnames.",
|
|
542
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
543
|
+
[["--config-export"],
|
|
544
|
+
{"dest": "conf_export", "metavar": "CONFIG_EXPORT",
|
|
545
|
+
"action": "store", "default": None,
|
|
546
|
+
"help": "generate another DB with given config"}],
|
|
547
|
+
],
|
|
548
|
+
db_anonymize],
|
|
549
|
+
"db-anonymize-mapping": ["Generate anonymization mapping json.",
|
|
550
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
551
|
+
db_anonymize_mapping],
|
|
552
|
+
"clean": ["Remove database and related files to initialize.",
|
|
553
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
554
|
+
clean],
|
|
555
|
+
"show-db-info": ["Show abstruction of database status.",
|
|
556
|
+
[OPT_CONFIG, OPT_DEBUG, OPT_TERM],
|
|
557
|
+
show_db_info],
|
|
558
|
+
"show-lt": ["Show all log templates in database.",
|
|
559
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
560
|
+
[["-s", "--simple"],
|
|
561
|
+
{"dest": "simple", "action": "store_true",
|
|
562
|
+
"help": "only show log templates"}]],
|
|
563
|
+
show_lt],
|
|
564
|
+
"show-ltg": ["Show all log template groups and their members.",
|
|
565
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
566
|
+
[["-g", "--group"],
|
|
567
|
+
{"dest": "group", "action": "store", "default": None,
|
|
568
|
+
"help": "show members of given labeling group"}]],
|
|
569
|
+
show_ltg],
|
|
570
|
+
"show-tag": ["Show all tags and their templates.",
|
|
571
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
572
|
+
[["tag"],
|
|
573
|
+
{"metavar": "TAG", "action": "store",
|
|
574
|
+
"nargs": "?", "default": None,
|
|
575
|
+
"help": "tag to search (if not given, show all tags)"}]],
|
|
576
|
+
show_tag],
|
|
577
|
+
"show-tag-stats": ["Show all tags and their templates.",
|
|
578
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
579
|
+
show_tag_stats],
|
|
580
|
+
"show-lt-import": ["Output log template definitions in importable format.",
|
|
581
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
582
|
+
[["-e", "--external"],
|
|
583
|
+
{"dest": "external", "action": "store_true",
|
|
584
|
+
"help": "output in non-amulog RE parser format"}]],
|
|
585
|
+
show_lt_import],
|
|
586
|
+
"show-host": ["Show all hostnames in database.",
|
|
587
|
+
[OPT_CONFIG, OPT_DEBUG],
|
|
588
|
+
show_host],
|
|
589
|
+
"show-log": ["Search and show log messages.",
|
|
590
|
+
[OPT_CONFIG, OPT_DEBUG,
|
|
591
|
+
[["--lid"],
|
|
592
|
+
{"dest": "lid", "action": "store_true",
|
|
593
|
+
"help": "show lid"}],
|
|
594
|
+
ARG_DBSEARCH],
|
|
595
|
+
show_log],
|
|
596
|
+
"show-line": ["Show log line with given log message identifier",
|
|
597
|
+
[OPT_CONFIG, OPT_DEBUG, ARG_LID],
|
|
598
|
+
show_line],
|
|
599
|
+
"conf-defaults": ["Show default configurations.",
|
|
600
|
+
[],
|
|
601
|
+
conf_defaults],
|
|
602
|
+
"conf-diff": ["Show differences of 2 configuration files.",
|
|
603
|
+
[OPT_CONFIG_SET,
|
|
604
|
+
[["files"],
|
|
605
|
+
{"metavar": "FILENAME", "nargs": "*",
|
|
606
|
+
"help": "configuration file"}]],
|
|
607
|
+
conf_diff],
|
|
608
|
+
"conf-minimum": ["Remove default options and comments.",
|
|
609
|
+
[[["-o", "--overwrite"],
|
|
610
|
+
{"dest": "overwrite", "action": "store_true",
|
|
611
|
+
"help": "overwrite file instead of stdout dumping"}],
|
|
612
|
+
[["conf_path"],
|
|
613
|
+
{"metavar": "PATH",
|
|
614
|
+
"help": "config filepath to load"}]],
|
|
615
|
+
conf_minimum],
|
|
616
|
+
"conf-group-edit": ["Edit configuration files in a config group.",
|
|
617
|
+
[[["configset"],
|
|
618
|
+
{"metavar": "CONFIG_SET",
|
|
619
|
+
"help": "config group definition file to use"}],
|
|
620
|
+
[["key"],
|
|
621
|
+
{"metavar": "KEY",
|
|
622
|
+
"help": "\"SECTION.OPTION\" to edit"}],
|
|
623
|
+
[["rule"],
|
|
624
|
+
{"metavar": "RULE",
|
|
625
|
+
"help": ("Value specification rule "
|
|
626
|
+
"defined in function-like format. "
|
|
627
|
+
"Example: \"List(1,10,100,1000)\" "
|
|
628
|
+
"Available format: "
|
|
629
|
+
"list(values for each config), "
|
|
630
|
+
"range(START,STEP), "
|
|
631
|
+
"power(START,STEP), "
|
|
632
|
+
"withconf(NAME),"
|
|
633
|
+
"namerange(NAME)."
|
|
634
|
+
"Note that 1 rule for 1 execution.")}]],
|
|
635
|
+
conf_set_edit],
|
|
636
|
+
"conf-shadow": ["Copy configuration files.",
|
|
637
|
+
[OPT_CONFIG,
|
|
638
|
+
[["-f", "--force"],
|
|
639
|
+
{"dest": "force", "action": "store_true",
|
|
640
|
+
"help": "Ignore overwrite of output file"}],
|
|
641
|
+
[["-n", "--number"],
|
|
642
|
+
{"dest": "number", "metavar": "INT",
|
|
643
|
+
"action": "store", "type": int, "default": 1,
|
|
644
|
+
"help": "number of files to generate"}],
|
|
645
|
+
[["-o", "--output"],
|
|
646
|
+
{"dest": "output", "metavar": "FILENAME",
|
|
647
|
+
"action": "store", "type": str, "default": None,
|
|
648
|
+
"help": "basic output filename"}],
|
|
649
|
+
[["-s", "--configset"],
|
|
650
|
+
{"dest": "configset", "metavar": "CONFIG_SET",
|
|
651
|
+
"default": None,
|
|
652
|
+
"help": ("define config group "
|
|
653
|
+
"and dump it in given filename")}],
|
|
654
|
+
[["rules"],
|
|
655
|
+
{"metavar": "RULES", "nargs": "*",
|
|
656
|
+
"help": ("Rules to replace options. You can indicate "
|
|
657
|
+
"option, or option and its value with =. "
|
|
658
|
+
"You can use both of them together. "
|
|
659
|
+
"For example: \"general.import=hoge.conf "
|
|
660
|
+
"general.logging\"")}]],
|
|
661
|
+
conf_shadow],
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
ALIASES = {
|
|
665
|
+
"dump-lt": "show-lt-import",
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def main():
|
|
670
|
+
cli.main(DICT_ARGSET, ALIASES, SUBLIB)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
if __name__ == "__main__":
|
|
674
|
+
main()
|
|
675
|
+
|
|
676
|
+
# import cProfile
|
|
677
|
+
# cProfile.run('_main()', filename='main.prof')
|
amulog/alg/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .meta import is_online
|