amulog 0.3.14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. amulog/__init__.py +10 -0
  2. amulog/__main__.py +677 -0
  3. amulog/alg/__init__.py +1 -0
  4. amulog/alg/crf/__init__.py +2 -0
  5. amulog/alg/crf/__main__.py +176 -0
  6. amulog/alg/crf/_convert.py +130 -0
  7. amulog/alg/crf/_items.py +56 -0
  8. amulog/alg/crf/lt_crf.py +588 -0
  9. amulog/alg/crf/train.py +123 -0
  10. amulog/alg/dlog/__init__.py +2 -0
  11. amulog/alg/dlog/dlog.py +147 -0
  12. amulog/alg/drain/__init__.py +4 -0
  13. amulog/alg/drain/drain.py +136 -0
  14. amulog/alg/fttree/__init__.py +4 -0
  15. amulog/alg/fttree/fttree.py +159 -0
  16. amulog/alg/lenma/__init__.py +4 -0
  17. amulog/alg/lenma/lenma.py +123 -0
  18. amulog/alg/meta.py +22 -0
  19. amulog/alg/shiso/__init__.py +3 -0
  20. amulog/alg/shiso/shiso.py +247 -0
  21. amulog/anonymize.py +183 -0
  22. amulog/cli.py +63 -0
  23. amulog/common.py +453 -0
  24. amulog/config.py +623 -0
  25. amulog/data/config.conf.default +345 -0
  26. amulog/data/crf_template.default +12 -0
  27. amulog/data/crf_template.sample +10 -0
  28. amulog/data/drain_regex.conf +9 -0
  29. amulog/data/host_alias.txt.sample +26 -0
  30. amulog/data/lt_label.conf.sample +55 -0
  31. amulog/data/testlog.conf +84 -0
  32. amulog/db_common.py +232 -0
  33. amulog/db_mysql.py +126 -0
  34. amulog/db_sqlite.py +110 -0
  35. amulog/edit/__init__.py +0 -0
  36. amulog/edit/__main__.py +331 -0
  37. amulog/edit/lt_tool.py +288 -0
  38. amulog/edit/search.py +82 -0
  39. amulog/eval/__init__.py +0 -0
  40. amulog/eval/__main__.py +421 -0
  41. amulog/eval/cluster_metrics.py +202 -0
  42. amulog/eval/maketpl.py +964 -0
  43. amulog/eval/param_searcher.py +102 -0
  44. amulog/eval/structure_metrics.py +86 -0
  45. amulog/external/__init__.py +2 -0
  46. amulog/external/mod_tplseq.py +57 -0
  47. amulog/external/regexhash.py +77 -0
  48. amulog/external/tpl_match.py +63 -0
  49. amulog/host_alias.py +165 -0
  50. amulog/log_db.py +1203 -0
  51. amulog/logparser.py +307 -0
  52. amulog/lt_common.py +722 -0
  53. amulog/lt_import.py +87 -0
  54. amulog/lt_import_ext.py +102 -0
  55. amulog/lt_label.py +159 -0
  56. amulog/lt_misc.py +127 -0
  57. amulog/lt_regex.py +134 -0
  58. amulog/lt_search.py +420 -0
  59. amulog/lt_va.py +121 -0
  60. amulog/ltg_semantics.py +253 -0
  61. amulog/manager.py +670 -0
  62. amulog/mproc_queue.py +112 -0
  63. amulog/strutil.py +46 -0
  64. amulog/testutil.py +219 -0
  65. amulog-0.3.14.dist-info/METADATA +261 -0
  66. amulog-0.3.14.dist-info/RECORD +92 -0
  67. amulog-0.3.14.dist-info/WHEEL +5 -0
  68. amulog-0.3.14.dist-info/entry_points.txt +4 -0
  69. amulog-0.3.14.dist-info/licenses/LICENSE +11 -0
  70. amulog-0.3.14.dist-info/top_level.txt +3 -0
  71. tests/__init__.py +0 -0
  72. tests/test_cli.py +44 -0
  73. tests/test_cluster_metrics.py +69 -0
  74. tests/test_common.py +59 -0
  75. tests/test_config.py +97 -0
  76. tests/test_crf.py +89 -0
  77. tests/test_db.py +241 -0
  78. tests/test_edit.py +71 -0
  79. tests/test_host_alias.py +153 -0
  80. tests/test_lenma_cluster.py +60 -0
  81. tests/test_lt_common.py +65 -0
  82. tests/test_lt_misc.py +164 -0
  83. tests/test_lt_regex.py +35 -0
  84. tests/test_lt_search.py +104 -0
  85. tests/test_lt_va.py +61 -0
  86. tests/test_ltg_semantics.py +93 -0
  87. tests/test_ltgen.py +125 -0
  88. tests/test_maketpl.py +78 -0
  89. tests/test_postprocess.py +164 -0
  90. tests/test_shiso.py +62 -0
  91. tests-ext/__init__.py +0 -0
  92. tests-ext/test_mysql.py +78 -0
amulog/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+
2
+ """
3
+ A system log management tool.
4
+ The main function is to classify log messages based
5
+ on their log templates (formats and variable locations),
6
+ and to store the data in a database.
7
+ """
8
+
9
+ __version__ = '0.3.14'
10
+
amulog/__main__.py ADDED
@@ -0,0 +1,677 @@
1
+ #!/usr/bin/env python
2
+ # coding: utf-8
3
+
4
+ """
5
+ Interface to use some functions about Log DB from CLI.
6
+ """
7
+
8
+ import datetime
9
+ import logging
10
+
11
+ from . import cli
12
+ from . import common
13
+ from . import config
14
+
15
+ _logger = logging.getLogger(__package__)
16
+ SUBLIB = ["edit", "eval"]
17
+
18
+
19
+ def get_targets_arg(ns):
20
+ if ns.recur:
21
+ targets = common.recur_dir(ns.files)
22
+ else:
23
+ targets = common.rep_dir(ns.files)
24
+ return targets
25
+
26
+
27
+ def get_targets_conf(conf):
28
+ l_path = config.getlist(conf, "general", "src_path")
29
+ if conf.getboolean("general", "src_recur"):
30
+ return common.recur_dir(l_path)
31
+ else:
32
+ return common.rep_dir(l_path)
33
+
34
+
35
+ def get_targets(ns, conf):
36
+ if ns is None or len(ns.files) == 0:
37
+ return get_targets_conf(conf)
38
+ else:
39
+ return get_targets_arg(ns)
40
+
41
+
42
+ def is_online(conf, parallel):
43
+ from .alg import is_online as alg_is_online
44
+ mode = conf["log_template"]["processing_mode"]
45
+ lt_methods = config.getlist(conf, "log_template", "lt_methods")
46
+ return alg_is_online(mode, lt_methods, parallel)
47
+
48
+
49
+ def data_from_db(ns):
50
+ conf = config.open_config(ns.conf_path)
51
+ lv = logging.DEBUG if ns.debug else logging.INFO
52
+ config.set_common_logging(conf, logger=_logger, lv=lv)
53
+ dirname = ns.dirname
54
+ if ns.incr:
55
+ method = "incremental"
56
+ else:
57
+ method = "commit"
58
+ reset = ns.reset
59
+
60
+ from . import log_db
61
+ log_db.data_from_db(conf, dirname, method, reset)
62
+
63
+
64
+ def data_from_data(ns):
65
+ conf = config.open_config(ns.conf_path)
66
+ lv = logging.DEBUG if ns.debug else logging.INFO
67
+ config.set_common_logging(conf, logger=_logger, lv=lv)
68
+ dirname = ns.dirname
69
+ targets = get_targets(ns, conf)
70
+ if ns.incr:
71
+ method = "incremental"
72
+ else:
73
+ method = "commit"
74
+ reset = ns.reset
75
+
76
+ from . import manager
77
+ manager.data_from_data(conf, targets, dirname, method, reset)
78
+
79
+
80
+ def data_parse(ns):
81
+ conf = config.open_config(ns.conf_path)
82
+ lv = logging.DEBUG if ns.debug else logging.INFO
83
+ config.set_common_logging(conf, logger=_logger, lv=lv)
84
+ targets = get_targets(ns, conf)
85
+
86
+ from . import strutil
87
+ from . import manager
88
+ lp = manager.load_log2seq(conf)
89
+ for line in manager.iter_lines(targets):
90
+ pline = manager.parse_line(strutil.add_esc(line), lp)
91
+ if pline is None:
92
+ pass
93
+ elif ns.words:
94
+ print(" ".join(pline["words"]))
95
+ else:
96
+ print(pline)
97
+
98
+
99
+ def db_make(ns):
100
+ conf = config.open_config(ns.conf_path)
101
+ lv = logging.DEBUG if ns.debug else logging.INFO
102
+ config.set_common_logging(conf, logger=_logger, lv=lv)
103
+ targets = get_targets(ns, conf)
104
+
105
+ from . import manager
106
+ timer = common.Timer("db-make", output=_logger)
107
+ timer.start()
108
+ if is_online(conf, ns.parallel):
109
+ manager.process_files_online(conf, targets, True)
110
+ else:
111
+ manager.process_files_offline(conf, targets, True, ns.parallel)
112
+ timer.stop()
113
+
114
+
115
+ def db_add(ns):
116
+ conf = config.open_config(ns.conf_path)
117
+ lv = logging.DEBUG if ns.debug else logging.INFO
118
+ config.set_common_logging(conf, logger=_logger, lv=lv)
119
+ targets = get_targets_arg(ns)
120
+
121
+ from . import manager
122
+
123
+ timer = common.Timer("db-add", output=_logger)
124
+ timer.start()
125
+ if is_online(conf, ns.parallel):
126
+ manager.process_files_online(conf, targets, False)
127
+ else:
128
+ manager.process_files_offline(conf, targets, False, ns.parallel)
129
+ timer.stop()
130
+
131
+
132
+ def db_remake_group(ns):
133
+ conf = config.open_config(ns.conf_path)
134
+ lv = logging.DEBUG if ns.debug else logging.INFO
135
+ config.set_common_logging(conf, logger=_logger, lv=lv)
136
+
137
+ from . import manager
138
+ timer = common.Timer("db-remake-group", output=_logger)
139
+ timer.start()
140
+ manager.remake_ltgroup(conf)
141
+ timer.stop()
142
+
143
+
144
+ def db_tag(ns):
145
+ conf = config.open_config(ns.conf_path)
146
+ lv = logging.DEBUG if ns.debug else logging.INFO
147
+ config.set_common_logging(conf, logger=_logger, lv=lv)
148
+
149
+ from . import lt_label
150
+ timer = common.Timer("db-tag", output=_logger)
151
+ timer.start()
152
+ lt_label.generate_all_tags(conf)
153
+ timer.stop()
154
+
155
+
156
+ def db_repair(ns):
157
+ conf = config.open_config(ns.conf_path)
158
+ lv = logging.DEBUG if ns.debug else logging.INFO
159
+ config.set_common_logging(conf, logger=_logger, lv=lv)
160
+
161
+ from . import log_db
162
+ db = log_db.LogDB(conf, edit=True, reset_db=False)
163
+ db.repair_tables()
164
+
165
+
166
+ def db_anonymize(ns):
167
+ conf = config.open_config(ns.conf_path)
168
+ lv = logging.DEBUG if ns.debug else logging.INFO
169
+ config.set_common_logging(conf, logger=_logger, lv=lv)
170
+
171
+ timer = common.Timer("db-anonymize", output=_logger)
172
+ timer.start()
173
+ if ns.conf_export:
174
+ conf2 = config.open_config(ns.conf_export)
175
+ else:
176
+ conf2 = None
177
+ from . import anonymize
178
+ am = anonymize.AnonymizeMapper(conf)
179
+ am.anonymize(conf2)
180
+ fp = am.dump()
181
+ timer.stop()
182
+
183
+ print("> " + fp)
184
+
185
+
186
+ def db_anonymize_mapping(ns):
187
+ conf = config.open_config(ns.conf_path)
188
+ lv = logging.DEBUG if ns.debug else logging.INFO
189
+ config.set_common_logging(conf, logger=_logger, lv=lv)
190
+
191
+ from . import anonymize
192
+ am = anonymize.AnonymizeMapper(conf)
193
+ am.mapping()
194
+ fp = am.dump()
195
+
196
+ print("> " + fp)
197
+
198
+
199
+ def clean(ns):
200
+ conf = config.open_config(ns.conf_path)
201
+ lv = logging.DEBUG if ns.debug else logging.INFO
202
+ config.set_common_logging(conf, logger=_logger, lv=lv)
203
+
204
+ from . import manager
205
+ manager.clean(conf)
206
+
207
+
208
+ def show_db_info(ns):
209
+ conf = config.open_config(ns.conf_path)
210
+ lv = logging.DEBUG if ns.debug else logging.INFO
211
+ config.set_common_logging(conf, logger=_logger, lv=lv)
212
+
213
+ from . import log_db
214
+ if ns.dt_range is None:
215
+ log_db.info(conf)
216
+ else:
217
+ from dateutil import parser
218
+ dts = parser.parse(ns.dt_range[0])
219
+ dte = parser.parse(ns.dt_range[1])
220
+ log_db.info_term(conf, dts, dte)
221
+
222
+
223
+ def show_lt(ns):
224
+ conf = config.open_config(ns.conf_path)
225
+ lv = logging.DEBUG if ns.debug else logging.INFO
226
+ config.set_common_logging(conf, logger=_logger, lv=lv)
227
+ from . import log_db
228
+
229
+ print(log_db.show_all_lt(conf, simple=ns.simple))
230
+
231
+
232
+ def show_ltg(ns):
233
+ conf = config.open_config(ns.conf_path)
234
+ lv = logging.DEBUG if ns.debug else logging.INFO
235
+ config.set_common_logging(conf, logger=_logger, lv=lv)
236
+ from . import log_db
237
+
238
+ print(log_db.show_all_ltg(conf))
239
+
240
+
241
+ def show_tag(ns):
242
+ conf = config.open_config(ns.conf_path)
243
+ lv = logging.DEBUG if ns.debug else logging.INFO
244
+ config.set_common_logging(conf, logger=_logger, lv=lv)
245
+ from . import log_db
246
+
247
+ print(log_db.show_tag(conf, tag=ns.tag))
248
+
249
+
250
+ def show_tag_stats(ns):
251
+ conf = config.open_config(ns.conf_path)
252
+ lv = logging.DEBUG if ns.debug else logging.INFO
253
+ config.set_common_logging(conf, logger=_logger, lv=lv)
254
+ from . import log_db
255
+
256
+ print(log_db.show_tag_stats(conf))
257
+
258
+
259
+ def show_lt_import(ns):
260
+ conf = config.open_config(ns.conf_path)
261
+ lv = logging.DEBUG if ns.debug else logging.INFO
262
+ config.set_common_logging(conf, logger=_logger, lv=lv)
263
+ from . import log_db
264
+
265
+ external = ns.external
266
+ ld = log_db.LogData(conf)
267
+ for ltobj in ld.iter_lt():
268
+ if external:
269
+ # unsegmented, with escape
270
+ print(ltobj.restore_message(None, esc=True))
271
+ else:
272
+ # segmented, with escape
273
+ print(" ".join(ltobj.ltw))
274
+
275
+
276
+ def show_host(ns):
277
+ conf = config.open_config(ns.conf_path)
278
+ lv = logging.DEBUG if ns.debug else logging.INFO
279
+ config.set_common_logging(conf, logger=_logger, lv=lv)
280
+ from . import log_db
281
+
282
+ log_db.show_all_host(conf)
283
+
284
+
285
+ def show_log(ns):
286
+ conf = config.open_config(ns.conf_path)
287
+ lv = logging.DEBUG if ns.debug else logging.INFO
288
+ config.set_common_logging(conf, logger=_logger, lv=lv)
289
+ from . import log_db
290
+ lidflag = ns.lid
291
+
292
+ d = parse_condition(ns.conditions)
293
+ ld = log_db.LogData(conf)
294
+ for e in ld.iter_lines(**d):
295
+ if lidflag:
296
+ print("{0} {1}".format(e.lid, e.restore_line()))
297
+ else:
298
+ print(e.restore_line())
299
+
300
+
301
+ def parse_condition(conditions):
302
+ """
303
+ Args:
304
+ conditions (list)
305
+ """
306
+ from dateutil import parser
307
+
308
+ d = {}
309
+ for arg in conditions:
310
+ if "=" not in arg:
311
+ raise SyntaxError
312
+ key = arg.partition("=")[0]
313
+ if key == "ltid":
314
+ d["ltid"] = int(arg.partition("=")[-1])
315
+ elif key in ("gid", "ltgid"):
316
+ d["ltgid"] = int(arg.partition("=")[-1])
317
+ elif key in ("time_from", "dts"):
318
+ time_string = arg.partition("=")[-1]
319
+ time_string = time_string.strip('"' + "'")
320
+ d["dts"] = parser.parse(time_string)
321
+ elif key in ("time_to", "dte"):
322
+ time_string = arg.partition("=")[-1]
323
+ time_string = time_string.strip('"' + "'")
324
+ d["dte"] = parser.parse(time_string)
325
+ # elif key in ("date_from", "top_date"):
326
+ # date_string = arg.partition("=")[-1]
327
+ # d["dts"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
328
+ # elif key in ("date_to", "end_date"):
329
+ # date_string = arg.partition("=")[-1]
330
+ # d["dte"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
331
+ elif key == "date":
332
+ date_string = arg.partition("=")[-1]
333
+ d["dts"] = datetime.datetime.strptime(date_string, "%Y-%m-%d")
334
+ d["dte"] = d["dts"] + datetime.timedelta(days=1)
335
+ elif key == "host":
336
+ d["host"] = arg.partition("=")[-1]
337
+ elif key == "host_like":
338
+ d["host_like"] = arg.partition("=")[-1]
339
+ elif key == "host_regexp":
340
+ d["host_regexp"] = arg.partition("=")[-1]
341
+ else:
342
+ raise ValueError
343
+ return d
344
+
345
+
346
+ def show_line(ns):
347
+ conf = config.open_config(ns.conf_path)
348
+ lv = logging.DEBUG if ns.debug else logging.INFO
349
+ config.set_common_logging(conf, logger=_logger, lv=lv)
350
+ from . import log_db
351
+ lid = ns.lid
352
+
353
+ ld = log_db.LogData(conf)
354
+ lm = ld.get_line(lid)
355
+ print(lm.restore_line())
356
+
357
+
358
+ def conf_defaults(_):
359
+ config.show_default_config()
360
+
361
+
362
+ def conf_diff(ns):
363
+ files = ns.files[:]
364
+ if ns.configset:
365
+ files += config.read_config_group(ns.configset)
366
+ config.show_config_diff(files)
367
+
368
+
369
+ def conf_minimum(ns):
370
+ conf = config.open_config(ns.conf_path, base_default=False, ignore_import=True)
371
+ conf = config.minimize(conf)
372
+ config.write(ns.conf_path, conf)
373
+ print("rewrite {0}".format(ns.conf_path))
374
+
375
+
376
+ def conf_set_edit(ns):
377
+ l_conf_name = config.read_config_group(ns.configset)
378
+ key = ns.key
379
+ rulestr = ns.rule
380
+
381
+ if "(" in rulestr:
382
+ temp = rulestr.rstrip(")").split("(")
383
+ if not len(temp) == 2:
384
+ raise ValueError("bad format for value specification")
385
+ rulename, argstr = temp
386
+ args = argstr.split(",")
387
+
388
+ if rulename == "list":
389
+ assert len(args) == len(l_conf_name)
390
+ d = {key: args}
391
+ elif rulename == "range":
392
+ assert len(args) == 2
393
+ start, step = [int(v) for v in args]
394
+ l_val = [start + i * step for i in range(len(l_conf_name))]
395
+ d = {key: l_val}
396
+ elif rulename == "power":
397
+ assert len(args) == 2
398
+ start, step = [int(v) for v in args]
399
+ l_val = [start * (i ** step) for i in range(len(l_conf_name))]
400
+ d = {key: l_val}
401
+ elif rulename == "withconf":
402
+ assert len(args) == 1
403
+ l_val = [args[0] + name for i, name in enumerate(l_conf_name)]
404
+ d = {key: l_val}
405
+ elif rulename == "namerange":
406
+ assert len(args) == 1
407
+ l_val = [args[0] + str(i) for i in range(len(l_conf_name))]
408
+ d = {key: l_val}
409
+ else:
410
+ raise NotImplementedError("invalid rule name")
411
+ else:
412
+ l_val = [rulestr] * len(l_conf_name)
413
+ d = {key: l_val}
414
+
415
+ config.config_group_edit(l_conf_name, d)
416
+
417
+
418
+ def conf_shadow(ns):
419
+ cond = {}
420
+ incr = []
421
+ for rule in ns.rules:
422
+ if "=" in rule:
423
+ key, val = rule.split("=")
424
+ cond[key] = val
425
+ else:
426
+ incr.append(rule)
427
+ l_conf_name = config.config_shadow(n=ns.number, cond=cond, incr=incr,
428
+ fn=ns.conf_path, output=ns.output,
429
+ ignore_overwrite=ns.force)
430
+
431
+ if ns.configset is not None:
432
+ config.dump_config_group(ns.configset, l_conf_name)
433
+ print(ns.configset)
434
+
435
+
436
+ # common argument settings
437
+ OPT_DEBUG = [["--debug"],
438
+ {"dest": "debug", "action": "store_true",
439
+ "help": "set logging level to debug (default: info)"}]
440
+ OPT_CONFIG = [["-c", "--config"],
441
+ {"dest": "conf_path", "metavar": "CONFIG", "action": "store",
442
+ # "default": config.DEFAULT_CONFIG,
443
+ "default": None,
444
+ "help": "configuration file path for amulog"}]
445
+ OPT_CONFIG_SET = [["-s", "--configset"],
446
+ {"dest": "configset", "metavar": "CONFIG_SET",
447
+ "default": None,
448
+ "help": "use config group definition file"}]
449
+ OPT_PARALLEL = [["-p", "--parallel"],
450
+ {"dest": "parallel", "action": "store_true",
451
+ "help": "parallel processing in offline mode"}]
452
+ OPT_RECUR = [["-r", "--recur"],
453
+ {"dest": "recur", "action": "store_true",
454
+ "help": "recursively search files to process"}]
455
+ OPT_RESET = [["--reset"],
456
+ {"dest": "reset", "action": "store_true",
457
+ "help": "reset log template generation model"}]
458
+ OPT_DRY = [["-d", "--dry"],
459
+ {"dest": "dry", "action": "store_true",
460
+ "help": "do not store data into db"}]
461
+ OPT_TERM = [["-t", "--term"],
462
+ {"dest": "dt_range", "default": None,
463
+ "metavar": "DATE", "nargs": 2,
464
+ "help": ("datetime range (start end), e.g. -t 2012-10-10 "
465
+ "2012-10-20. Optional; without it, use all data.")}]
466
+ ARG_FILE = [["file"],
467
+ {"metavar": "PATH", "action": "store",
468
+ "help": "filepath to output"}]
469
+ ARG_FILES = [["files"],
470
+ {"metavar": "PATH", "nargs": "+",
471
+ "help": "files or directories as input"}]
472
+ ARG_FILES_OPT = [["files"],
473
+ {"metavar": "PATH", "nargs": "*",
474
+ "help": ("files or directories as input "
475
+ "(optional; defaultly read from config")}]
476
+ ARG_DBSEARCH = [["conditions"],
477
+ {"metavar": "CONDITION", "nargs": "+",
478
+ "help": ("Conditions to search log messages. "
479
+ "Example: MODE gid=24 date=2012-10-10 ..., "
480
+ "Keys: ltid, gid, date, time_from, time_to, host, "
481
+ "host_like, host_regexp")}]
482
+ ARG_LID = [["lid"],
483
+ {"metavar": "LID", "action": "store",
484
+ "help": "log message identifier"}]
485
+
486
+ # argument settings for each modes
487
+ # description, List[args, kwargs], func
488
+ # defined after functions because these settings use functions
489
+ DICT_ARGSET = {
490
+ "data-from-db": ["Generate log data from DB.",
491
+ [OPT_CONFIG, OPT_DEBUG,
492
+ [["-d", "--dirname"],
493
+ {"dest": "dirname", "metavar": "DIRNAME",
494
+ "action": "store",
495
+ "help": "directory name to output"}],
496
+ [["-i", "--incr"],
497
+ {"dest": "incr", "action": "store_true",
498
+ "help": "output incrementally, use with small memory"}],
499
+ [["--reset"],
500
+ {"dest": "reset", "action": "store_true",
501
+ "help": "reset log file directory before processing"}],
502
+ ],
503
+ data_from_db],
504
+ "data-from-data": ["Re-arrange log file, splitting messages by date.",
505
+ [OPT_CONFIG, OPT_DEBUG, OPT_RECUR, ARG_FILES_OPT,
506
+ [["-d", "--dirname"],
507
+ {"dest": "dirname", "metavar": "DIRNAME",
508
+ "action": "store",
509
+ "help": "directory name to output"}],
510
+ [["-i", "--incr"],
511
+ {"dest": "incr", "action": "store_true",
512
+ "help": "output incrementally, use with small memory"}],
513
+ [["--reset"],
514
+ {"dest": "reset", "action": "store_true",
515
+ "help": "reset log file directory before processing"}],
516
+ ],
517
+ data_from_data],
518
+ "data-parse": ["Check log data parsing with log2seq.",
519
+ [OPT_CONFIG, OPT_DEBUG, OPT_RECUR,
520
+ [["-w", "--words"],
521
+ {"dest": "words", "action": "store_true",
522
+ "help": "only show parsed words"}],
523
+ ARG_FILES_OPT],
524
+ data_parse],
525
+ "db-make": ["Initialize database and add log data. ",
526
+ [OPT_CONFIG, OPT_DEBUG, OPT_RECUR,
527
+ OPT_RESET, OPT_PARALLEL, ARG_FILES_OPT],
528
+ db_make],
529
+ "db-add": ["Add log data to existing database.",
530
+ [OPT_CONFIG, OPT_DEBUG, OPT_RECUR, OPT_PARALLEL, ARG_FILES],
531
+ db_add],
532
+ "db-remake-group": ["Remake log template groups",
533
+ [OPT_CONFIG, OPT_DEBUG],
534
+ db_remake_group],
535
+ "db-tag": ["Make log template tags.",
536
+ [OPT_CONFIG, OPT_DEBUG],
537
+ db_tag],
538
+ "db-repair": ["Repair db schema after version updates.",
539
+ [OPT_CONFIG, OPT_DEBUG],
540
+ db_repair],
541
+ "db-anonymize": ["Anonymize templates and hostnames.",
542
+ [OPT_CONFIG, OPT_DEBUG,
543
+ [["--config-export"],
544
+ {"dest": "conf_export", "metavar": "CONFIG_EXPORT",
545
+ "action": "store", "default": None,
546
+ "help": "generate another DB with given config"}],
547
+ ],
548
+ db_anonymize],
549
+ "db-anonymize-mapping": ["Generate anonymization mapping json.",
550
+ [OPT_CONFIG, OPT_DEBUG],
551
+ db_anonymize_mapping],
552
+ "clean": ["Remove database and related files to initialize.",
553
+ [OPT_CONFIG, OPT_DEBUG],
554
+ clean],
555
+ "show-db-info": ["Show abstruction of database status.",
556
+ [OPT_CONFIG, OPT_DEBUG, OPT_TERM],
557
+ show_db_info],
558
+ "show-lt": ["Show all log templates in database.",
559
+ [OPT_CONFIG, OPT_DEBUG,
560
+ [["-s", "--simple"],
561
+ {"dest": "simple", "action": "store_true",
562
+ "help": "only show log templates"}]],
563
+ show_lt],
564
+ "show-ltg": ["Show all log template groups and their members.",
565
+ [OPT_CONFIG, OPT_DEBUG,
566
+ [["-g", "--group"],
567
+ {"dest": "group", "action": "store", "default": None,
568
+ "help": "show members of given labeling group"}]],
569
+ show_ltg],
570
+ "show-tag": ["Show all tags and their templates.",
571
+ [OPT_CONFIG, OPT_DEBUG,
572
+ [["tag"],
573
+ {"metavar": "TAG", "action": "store",
574
+ "nargs": "?", "default": None,
575
+ "help": "tag to search (if not given, show all tags)"}]],
576
+ show_tag],
577
+ "show-tag-stats": ["Show all tags and their templates.",
578
+ [OPT_CONFIG, OPT_DEBUG],
579
+ show_tag_stats],
580
+ "show-lt-import": ["Output log template definitions in importable format.",
581
+ [OPT_CONFIG, OPT_DEBUG,
582
+ [["-e", "--external"],
583
+ {"dest": "external", "action": "store_true",
584
+ "help": "output in non-amulog RE parser format"}]],
585
+ show_lt_import],
586
+ "show-host": ["Show all hostnames in database.",
587
+ [OPT_CONFIG, OPT_DEBUG],
588
+ show_host],
589
+ "show-log": ["Search and show log messages.",
590
+ [OPT_CONFIG, OPT_DEBUG,
591
+ [["--lid"],
592
+ {"dest": "lid", "action": "store_true",
593
+ "help": "show lid"}],
594
+ ARG_DBSEARCH],
595
+ show_log],
596
+ "show-line": ["Show log line with given log message identifier",
597
+ [OPT_CONFIG, OPT_DEBUG, ARG_LID],
598
+ show_line],
599
+ "conf-defaults": ["Show default configurations.",
600
+ [],
601
+ conf_defaults],
602
+ "conf-diff": ["Show differences of 2 configuration files.",
603
+ [OPT_CONFIG_SET,
604
+ [["files"],
605
+ {"metavar": "FILENAME", "nargs": "*",
606
+ "help": "configuration file"}]],
607
+ conf_diff],
608
+ "conf-minimum": ["Remove default options and comments.",
609
+ [[["-o", "--overwrite"],
610
+ {"dest": "overwrite", "action": "store_true",
611
+ "help": "overwrite file instead of stdout dumping"}],
612
+ [["conf_path"],
613
+ {"metavar": "PATH",
614
+ "help": "config filepath to load"}]],
615
+ conf_minimum],
616
+ "conf-group-edit": ["Edit configuration files in a config group.",
617
+ [[["configset"],
618
+ {"metavar": "CONFIG_SET",
619
+ "help": "config group definition file to use"}],
620
+ [["key"],
621
+ {"metavar": "KEY",
622
+ "help": "\"SECTION.OPTION\" to edit"}],
623
+ [["rule"],
624
+ {"metavar": "RULE",
625
+ "help": ("Value specification rule "
626
+ "defined in function-like format. "
627
+ "Example: \"List(1,10,100,1000)\" "
628
+ "Available format: "
629
+ "list(values for each config), "
630
+ "range(START,STEP), "
631
+ "power(START,STEP), "
632
+ "withconf(NAME),"
633
+ "namerange(NAME)."
634
+ "Note that 1 rule for 1 execution.")}]],
635
+ conf_set_edit],
636
+ "conf-shadow": ["Copy configuration files.",
637
+ [OPT_CONFIG,
638
+ [["-f", "--force"],
639
+ {"dest": "force", "action": "store_true",
640
+ "help": "Ignore overwrite of output file"}],
641
+ [["-n", "--number"],
642
+ {"dest": "number", "metavar": "INT",
643
+ "action": "store", "type": int, "default": 1,
644
+ "help": "number of files to generate"}],
645
+ [["-o", "--output"],
646
+ {"dest": "output", "metavar": "FILENAME",
647
+ "action": "store", "type": str, "default": None,
648
+ "help": "basic output filename"}],
649
+ [["-s", "--configset"],
650
+ {"dest": "configset", "metavar": "CONFIG_SET",
651
+ "default": None,
652
+ "help": ("define config group "
653
+ "and dump it in given filename")}],
654
+ [["rules"],
655
+ {"metavar": "RULES", "nargs": "*",
656
+ "help": ("Rules to replace options. You can indicate "
657
+ "option, or option and its value with =. "
658
+ "You can use both of them together. "
659
+ "For example: \"general.import=hoge.conf "
660
+ "general.logging\"")}]],
661
+ conf_shadow],
662
+ }
663
+
664
+ ALIASES = {
665
+ "dump-lt": "show-lt-import",
666
+ }
667
+
668
+
669
+ def main():
670
+ cli.main(DICT_ARGSET, ALIASES, SUBLIB)
671
+
672
+
673
+ if __name__ == "__main__":
674
+ main()
675
+
676
+ # import cProfile
677
+ # cProfile.run('_main()', filename='main.prof')
amulog/alg/__init__.py ADDED
@@ -0,0 +1 @@
1
+ from .meta import is_online
@@ -0,0 +1,2 @@
1
+ from .lt_crf import LTGenCRF as LTGen
2
+ from .lt_crf import init_ltgen_crf as init_ltgen