smf2db 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smf2db/__init__.py +30 -0
- smf2db/__main__.py +583 -0
- smf2db/api/__init__.py +0 -0
- smf2db/api/api_1101.py +209 -0
- smf2db/api/api_1101db.py +81 -0
- smf2db/api/api_1101db_sum.py +121 -0
- smf2db/api/api_1102.py +602 -0
- smf2db/api/api_1102db.py +89 -0
- smf2db/api/api_1102db_sum.py +133 -0
- smf2db/api/api_123.py +168 -0
- smf2db/api/api_123db.py +187 -0
- smf2db/api/api_123db_sum.py +241 -0
- smf2db/api/api_30.py +1012 -0
- smf2db/api/api_30db.py +423 -0
- smf2db/api/api_30db_sum.py +577 -0
- smf2db/api/api_70.py +1392 -0
- smf2db/api/api_70db.py +165 -0
- smf2db/api/api_70db_sum.py +776 -0
- smf2db/api/api_71.py +202 -0
- smf2db/api/api_71db.py +99 -0
- smf2db/api/api_71db_sum.py +211 -0
- smf2db/api/api_72.py +2045 -0
- smf2db/api/api_72db.py +414 -0
- smf2db/api/api_72db_sum.py +763 -0
- smf2db/api/api_73.py +402 -0
- smf2db/api/api_73db.py +113 -0
- smf2db/api/api_73db_sum.py +275 -0
- smf2db/api/api_74.py +3699 -0
- smf2db/api/api_74db.py +429 -0
- smf2db/api/api_74db_sum.py +903 -0
- smf2db/api/api_75.py +183 -0
- smf2db/api/api_75db.py +103 -0
- smf2db/api/api_75db_sum.py +168 -0
- smf2db/api/api_77.py +211 -0
- smf2db/api/api_77db.py +104 -0
- smf2db/api/api_77db_sum.py +149 -0
- smf2db/api/api_78.py +1154 -0
- smf2db/api/api_78db.py +118 -0
- smf2db/api/api_78db_sum.py +854 -0
- smf2db/api/report_util.py +7995 -0
- smf2db/api/util.py +851 -0
- smf2db/config.py +60 -0
- smf2db/db_models/__init__.py +0 -0
- smf2db/db_models/smf1101_15m_model.py +36 -0
- smf2db/db_models/smf1101_agg_dict.py +102 -0
- smf2db/db_models/smf1101_base.py +965 -0
- smf2db/db_models/smf1101_da_model.py +35 -0
- smf2db/db_models/smf1101_hr_model.py +36 -0
- smf2db/db_models/smf1101_model.py +42 -0
- smf2db/db_models/smf1101_rename.py +412 -0
- smf2db/db_models/smf1102_agg_dict.py +673 -0
- smf2db/db_models/smf1102_base.py +2782 -0
- smf2db/db_models/smf1102_da_model.py +1613 -0
- smf2db/db_models/smf1102_hr_model.py +1694 -0
- smf2db/db_models/smf1102_model.py +1517 -0
- smf2db/db_models/smf123_15m_model.py +92 -0
- smf2db/db_models/smf123_base.py +1122 -0
- smf2db/db_models/smf123_da_model.py +89 -0
- smf2db/db_models/smf123_hr_model.py +92 -0
- smf2db/db_models/smf123_model.py +109 -0
- smf2db/db_models/smf30_base.py +596 -0
- smf2db/db_models/smf30_da_model.py +380 -0
- smf2db/db_models/smf30_hr_model.py +382 -0
- smf2db/db_models/smf30_model.py +784 -0
- smf2db/db_models/smf70_base.py +1351 -0
- smf2db/db_models/smf70_da_model.py +372 -0
- smf2db/db_models/smf70_hr_model.py +373 -0
- smf2db/db_models/smf70_model.py +394 -0
- smf2db/db_models/smf71_base.py +808 -0
- smf2db/db_models/smf71_da_model.py +54 -0
- smf2db/db_models/smf71_hr_model.py +55 -0
- smf2db/db_models/smf71_model.py +54 -0
- smf2db/db_models/smf72_base.py +1018 -0
- smf2db/db_models/smf72_da_model.py +886 -0
- smf2db/db_models/smf72_hr_model.py +890 -0
- smf2db/db_models/smf72_model.py +976 -0
- smf2db/db_models/smf73_base.py +363 -0
- smf2db/db_models/smf73_da_model.py +278 -0
- smf2db/db_models/smf73_hr_model.py +279 -0
- smf2db/db_models/smf73_model.py +287 -0
- smf2db/db_models/smf74_base.py +2498 -0
- smf2db/db_models/smf74_da_model.py +1279 -0
- smf2db/db_models/smf74_hr_model.py +1285 -0
- smf2db/db_models/smf74_model.py +1387 -0
- smf2db/db_models/smf75_base.py +150 -0
- smf2db/db_models/smf75_da_model.py +57 -0
- smf2db/db_models/smf75_hr_model.py +58 -0
- smf2db/db_models/smf75_model.py +61 -0
- smf2db/db_models/smf77_base.py +145 -0
- smf2db/db_models/smf77_da_model.py +80 -0
- smf2db/db_models/smf77_hr_model.py +81 -0
- smf2db/db_models/smf77_model.py +118 -0
- smf2db/db_models/smf78_base.py +1616 -0
- smf2db/db_models/smf78_da_model.py +316 -0
- smf2db/db_models/smf78_hr_model.py +318 -0
- smf2db/db_models/smf78_model.py +340 -0
- smf2db/db_models/smf7x_model.py +113 -0
- smf2db/print/__init__.py +0 -0
- smf2db/print/commands.py +382 -0
- smf2db/schemas/schema.py +113 -0
- smf2db/sumup/__init__.py +0 -0
- smf2db/sumup/commands.py +930 -0
- smf2db/upload/__init__.py +0 -0
- smf2db/upload/commands.py +1079 -0
- smf2db-0.1.0.dist-info/METADATA +176 -0
- smf2db-0.1.0.dist-info/RECORD +110 -0
- smf2db-0.1.0.dist-info/WHEEL +5 -0
- smf2db-0.1.0.dist-info/entry_points.txt +2 -0
- smf2db-0.1.0.dist-info/licenses/LICENSE.txt +19 -0
- smf2db-0.1.0.dist-info/top_level.txt +1 -0
smf2db/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Top-level package for Smf2db."""
|
|
2
|
+
# smf2db/__init__.py
|
|
3
|
+
|
|
4
|
+
__app_name__ = "smf2db"
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
|
|
7
|
+
(
|
|
8
|
+
SUCCESS,
|
|
9
|
+
DIR_ERROR,
|
|
10
|
+
FILE_ERROR,
|
|
11
|
+
DB_READ_ERROR,
|
|
12
|
+
DB_WRITE_ERROR,
|
|
13
|
+
JSON_ERROR,
|
|
14
|
+
UPLOAD_ERROR,
|
|
15
|
+
DB_CONNECTION_ERROR,
|
|
16
|
+
) = range(8)
|
|
17
|
+
|
|
18
|
+
ERRORS = {
|
|
19
|
+
DIR_ERROR: "config directory error",
|
|
20
|
+
FILE_ERROR: "config file error",
|
|
21
|
+
DB_READ_ERROR: "database read error",
|
|
22
|
+
DB_WRITE_ERROR: "database write error",
|
|
23
|
+
JSON_ERROR: "Invalid JSON format error",
|
|
24
|
+
UPLOAD_ERROR: "uploading error",
|
|
25
|
+
DB_CONNECTION_ERROR: "database connection error",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
DEFAULT_DB_HOST = 'localhost'
|
|
29
|
+
DEFAULT_DB_PORT = 5432
|
|
30
|
+
DEFAULT_DB_USER = 'postgres'
|
smf2db/__main__.py
ADDED
|
@@ -0,0 +1,583 @@
|
|
|
1
|
+
# smf2db/__main__.py
|
|
2
|
+
|
|
3
|
+
"""This module provides the Smf2db CLI."""
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
import jsonschema
|
|
10
|
+
|
|
11
|
+
from smf2db import (
|
|
12
|
+
__version__, ERRORS, SUCCESS, DEFAULT_DB_HOST, DEFAULT_DB_PORT, DEFAULT_DB_USER
|
|
13
|
+
)
|
|
14
|
+
from smf2db.config import ConfigManager
|
|
15
|
+
from smf2db.print import commands as print_commands
|
|
16
|
+
from smf2db.schemas.schema import schema
|
|
17
|
+
from smf2db.sumup import commands as sumup_commands
|
|
18
|
+
from smf2db.sumup.commands import init_summary
|
|
19
|
+
from smf2db.upload import commands as upload_commands
|
|
20
|
+
from smf2db.upload.commands import init_database, validate_config_file, supports_psycopg2, supports_sqlite, \
|
|
21
|
+
supports_sshtunnel, supports_pg8000
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@click.group()
|
|
25
|
+
@click.version_option(version=__version__)
|
|
26
|
+
@click.pass_context
|
|
27
|
+
def cli(ctx):
|
|
28
|
+
"""A CLI application that upload data to database or print the reports using the JSON files."""
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
@cli.group("db")
|
|
32
|
+
def db():
|
|
33
|
+
"""A CLI application that uploads data to database."""
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
@cli.group("report")
|
|
37
|
+
def report():
|
|
38
|
+
"""Print report from JSON files"""
|
|
39
|
+
|
|
40
|
+
@db.group("upload")
|
|
41
|
+
def upload():
|
|
42
|
+
"""Upload JSON files to database"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@db.group("sumup")
|
|
46
|
+
def sumup():
|
|
47
|
+
"""Summing up database to 15min, hourly or daily database"""
|
|
48
|
+
|
|
49
|
+
def required_with_initcfg(ctx, param, db_driver):
|
|
50
|
+
if db_driver == 'psycopg2' and not supports_psycopg2(): #PSYCOPG2_SUPPORT:
|
|
51
|
+
click.secho(
|
|
52
|
+
'Cannot import psycopg2, "psycopg2" package was not found. '
|
|
53
|
+
"Please install smf2db with `pip install smf2db[psycopg2]` if you want psycopg2 support and ensure psycopg2 is supported on this platform.",
|
|
54
|
+
err=True,
|
|
55
|
+
fg="red",
|
|
56
|
+
)
|
|
57
|
+
raise SystemExit(1)
|
|
58
|
+
if db_driver == 'pg8000' and not supports_pg8000(): #PSYCOPG2_SUPPORT:
|
|
59
|
+
click.secho(
|
|
60
|
+
'Cannot import pg8000, "pg8000" package was not found. '
|
|
61
|
+
"Please install smf2db with `pip install smf2db[pg8000]` if you want pg8000 support.",
|
|
62
|
+
err=True,
|
|
63
|
+
fg="red",
|
|
64
|
+
)
|
|
65
|
+
raise SystemExit(1)
|
|
66
|
+
if db_driver == 'sqlite' and not supports_sqlite(): #SQLITE_SUPPORT:
|
|
67
|
+
click.secho(
|
|
68
|
+
'SQLite version need to be above 3.32 to continue the database operation.',
|
|
69
|
+
err=True,
|
|
70
|
+
fg="red",
|
|
71
|
+
)
|
|
72
|
+
raise SystemExit(1)
|
|
73
|
+
if db_driver in ['psycopg2', 'pg8000']:
|
|
74
|
+
host = ctx.params.get("host")
|
|
75
|
+
if not host:
|
|
76
|
+
# ctx.params['host'] = click.prompt('Host address of the database', default=DEFAULT_DB_HOST)
|
|
77
|
+
host_value = click.prompt('Host address of the database', default=DEFAULT_DB_HOST)
|
|
78
|
+
ctx.params["host"] = validate_hostname(ctx, param, host_value)
|
|
79
|
+
|
|
80
|
+
port = ctx.params.get("port")
|
|
81
|
+
if not port:
|
|
82
|
+
# ctx.params['port'] = click.prompt('Port number at which the database instance is listening', default=DEFAULT_DB_PORT)
|
|
83
|
+
port_value = click.prompt('Port number at which the database instance is listening',
|
|
84
|
+
default=DEFAULT_DB_PORT)
|
|
85
|
+
ctx.params["port"] = validate_port(ctx, param, port_value)
|
|
86
|
+
|
|
87
|
+
elif db_driver == 'sqlite':
|
|
88
|
+
sqlite_path = ctx.params.get('sqlite_path')
|
|
89
|
+
if not sqlite_path:
|
|
90
|
+
ctx.params['sqlite_path'] = click.prompt('Sqlite filepath',
|
|
91
|
+
type=click.Path(exists=True, file_okay=False, readable=True))
|
|
92
|
+
return db_driver
|
|
93
|
+
|
|
94
|
+
def required_with_initdb(ctx, param, smf):
|
|
95
|
+
config_file_path = ctx.params.get('config_file')
|
|
96
|
+
if not config_file_path:
|
|
97
|
+
ctx.params['config_file'] = click.prompt('Config file name', type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True))
|
|
98
|
+
config_file_path = ctx.params.get('config_file')
|
|
99
|
+
|
|
100
|
+
config_mgr = ConfigManager(config_file_path)
|
|
101
|
+
cfg = config_mgr.load_config()
|
|
102
|
+
if isinstance(cfg, tuple):
|
|
103
|
+
click.secho(
|
|
104
|
+
'Cannot open config file. '
|
|
105
|
+
"Please run `smf2db db initcfg` to create the config file first before you initialize the database.",
|
|
106
|
+
err=True,
|
|
107
|
+
fg="red",
|
|
108
|
+
)
|
|
109
|
+
raise SystemExit(1)
|
|
110
|
+
|
|
111
|
+
# Validate config file
|
|
112
|
+
try:
|
|
113
|
+
jsonschema.validate(cfg, schema=schema)
|
|
114
|
+
except jsonschema.exceptions.ValidationError as ex:
|
|
115
|
+
click.secho(
|
|
116
|
+
f'Invalid config file: {ex} '
|
|
117
|
+
"Please run `smf2db db initcfg` to create the config file first.",
|
|
118
|
+
err=True,
|
|
119
|
+
fg="red",
|
|
120
|
+
)
|
|
121
|
+
raise SystemExit(1)
|
|
122
|
+
|
|
123
|
+
db_driver = config_mgr.get('database.driver')
|
|
124
|
+
ssh_host = config_mgr.get('database.ssh_host')
|
|
125
|
+
if ssh_host:
|
|
126
|
+
use_ssh = True
|
|
127
|
+
else:
|
|
128
|
+
use_ssh = False
|
|
129
|
+
|
|
130
|
+
if db_driver == 'psycopg2' and not supports_psycopg2(): # PSYCOPG2_SUPPORT:
|
|
131
|
+
click.secho(
|
|
132
|
+
'Cannot import psycopg2, "psycopg2" package was not found. '
|
|
133
|
+
"Please install smf2db with `pip install smf2db[psycopg2]` if you want psycopg2 support and ensure psycopg2 is supported on this platform.",
|
|
134
|
+
err=True,
|
|
135
|
+
fg="red",
|
|
136
|
+
)
|
|
137
|
+
raise SystemExit(1)
|
|
138
|
+
if db_driver == 'pg8000' and not supports_pg8000(): # PSYCOPG2_SUPPORT:
|
|
139
|
+
click.secho(
|
|
140
|
+
'Cannot import pg8000, "pg8000" package was not found. '
|
|
141
|
+
"Please install smf2db with `pip install smf2db[pg8000]` if you want pg8000 support.",
|
|
142
|
+
err=True,
|
|
143
|
+
fg="red",
|
|
144
|
+
)
|
|
145
|
+
raise SystemExit(1)
|
|
146
|
+
if db_driver == 'sqlite' and not supports_sqlite():
|
|
147
|
+
click.secho(
|
|
148
|
+
'SQLite version need to be above 3.32 to continue the database operation.',
|
|
149
|
+
err=True,
|
|
150
|
+
fg="red",
|
|
151
|
+
)
|
|
152
|
+
raise SystemExit(1)
|
|
153
|
+
if use_ssh and not supports_sshtunnel():
|
|
154
|
+
click.secho(
|
|
155
|
+
'Cannot open SSH tunnel, "sshtunnel" package was not found. '
|
|
156
|
+
"Please install smf2db with `pip install smf2db[sshtunnel]` if you want SSH tunnel support and SSH tunnel is supported on this platform.",
|
|
157
|
+
err=True,
|
|
158
|
+
fg="red",
|
|
159
|
+
)
|
|
160
|
+
raise SystemExit(1)
|
|
161
|
+
|
|
162
|
+
if 'psycopg2' in db_driver or 'pg8000' in db_driver:
|
|
163
|
+
user = ctx.params.get('username')
|
|
164
|
+
if not user:
|
|
165
|
+
ctx.params['username'] = click.prompt('Username to connect to the database.', default=DEFAULT_DB_USER)
|
|
166
|
+
|
|
167
|
+
password = ctx.params.get('password')
|
|
168
|
+
if not password:
|
|
169
|
+
ctx.params['password'] = click.prompt('Password to connect to the database', hide_input=True,)
|
|
170
|
+
|
|
171
|
+
if ('psycopg2' in db_driver or 'pg8000' in db_driver) and use_ssh:
|
|
172
|
+
user = ctx.params.get('ssh_user')
|
|
173
|
+
if not user:
|
|
174
|
+
ctx.params["ssh_user"] = click.prompt('SSH User')
|
|
175
|
+
|
|
176
|
+
password = ctx.params.get('ssh_password')
|
|
177
|
+
if not password:
|
|
178
|
+
ctx.params["ssh_password"] = click.prompt('SSH Password', hide_input=True)
|
|
179
|
+
|
|
180
|
+
return smf
|
|
181
|
+
|
|
182
|
+
def prompt_ssh_with_initcfg(ctx, param, ssh):
|
|
183
|
+
if ssh and not supports_sshtunnel(): #SSH_TUNNEL_SUPPORT:
|
|
184
|
+
click.secho(
|
|
185
|
+
'Cannot open SSH tunnel, "sshtunnel" package was not found. '
|
|
186
|
+
"Please install smf2db with `pip install smf2db[sshtunnel]` if you want SSH tunnel support and SSH tunnel is supported on this platform.",
|
|
187
|
+
err=True,
|
|
188
|
+
fg="red",
|
|
189
|
+
)
|
|
190
|
+
raise SystemExit(1)
|
|
191
|
+
|
|
192
|
+
if ctx.params.get("db_driver") != 'sqlite' and ssh:
|
|
193
|
+
host = ctx.params.get("ssh_host")
|
|
194
|
+
if not host:
|
|
195
|
+
# ctx.params["ssh_host"] = click.prompt("SSH Host")
|
|
196
|
+
ssh_host_value = click.prompt("SSH Host")
|
|
197
|
+
ctx.params["ssh_host"] = validate_hostname(ctx, param, ssh_host_value)
|
|
198
|
+
|
|
199
|
+
port = ctx.params.get("ssh_port")
|
|
200
|
+
if not port:
|
|
201
|
+
# ctx.params["ssh_port"] = click.prompt("SSH Port", default="")
|
|
202
|
+
ctx.params["ssh_port"] = None
|
|
203
|
+
|
|
204
|
+
elif ctx.params.get("db_driver") == 'sqlite' and ssh:
|
|
205
|
+
raise click.BadParameter("--ssh is not supported using sqlite.")
|
|
206
|
+
|
|
207
|
+
return ssh
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def validate_hostname(ctx, param, value):
|
|
211
|
+
"""
|
|
212
|
+
Validate hostname according to RFC 1035
|
|
213
|
+
- Maximum length: 255 characters
|
|
214
|
+
- Labels separated by dots
|
|
215
|
+
- Each label: 1-63 characters
|
|
216
|
+
- Labels can contain letters, digits, hyphens
|
|
217
|
+
- Labels cannot start or end with hyphen
|
|
218
|
+
"""
|
|
219
|
+
if value:
|
|
220
|
+
if len(value) > 255:
|
|
221
|
+
raise click.BadParameter("Host name too long.")
|
|
222
|
+
|
|
223
|
+
# Remove trailing dot if present
|
|
224
|
+
if value[-1] == ".":
|
|
225
|
+
value = value[:-1]
|
|
226
|
+
|
|
227
|
+
# Check each label
|
|
228
|
+
allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
|
|
229
|
+
result = all(allowed.match(x) for x in value.split("."))
|
|
230
|
+
if not result:
|
|
231
|
+
raise click.BadParameter("Host name invalid.")
|
|
232
|
+
else:
|
|
233
|
+
return value
|
|
234
|
+
else:
|
|
235
|
+
return value
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def validate_port(ctx, param, value):
|
|
239
|
+
if value:
|
|
240
|
+
if isinstance(value, int) and 0 < value <= 65535:
|
|
241
|
+
return value
|
|
242
|
+
try:
|
|
243
|
+
if value[:2].lower() == "0x":
|
|
244
|
+
return int(value[2:], 16)
|
|
245
|
+
elif value[:1] == "0":
|
|
246
|
+
return int(value, 8)
|
|
247
|
+
return int(value, 10)
|
|
248
|
+
except ValueError:
|
|
249
|
+
raise click.BadParameter("Port number invalid.")
|
|
250
|
+
return value
|
|
251
|
+
|
|
252
|
+
def validate_filename(ctx, param, value: os.PathLike):
|
|
253
|
+
# Define a regular expression pattern to match forbidden characters
|
|
254
|
+
if Path(value).suffix != '.yaml':
|
|
255
|
+
raise click.BadParameter("Filename must be a YAML file.")
|
|
256
|
+
filename = Path(value).stem
|
|
257
|
+
ILLEGAL_NTFS_CHARS = r'[<>:/\\|?*\"]|[\0-\31]'
|
|
258
|
+
# Define a list of forbidden names
|
|
259
|
+
FORBIDDEN_NAMES = ['CON', 'PRN', 'AUX', 'NUL',
|
|
260
|
+
'COM1', 'COM2', 'COM3', 'COM4', 'COM5',
|
|
261
|
+
'COM6', 'COM7', 'COM8', 'COM9',
|
|
262
|
+
'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5',
|
|
263
|
+
'LPT6', 'LPT7', 'LPT8', 'LPT9']
|
|
264
|
+
# Check for forbidden characters
|
|
265
|
+
match = re.search(ILLEGAL_NTFS_CHARS, filename)
|
|
266
|
+
if match:
|
|
267
|
+
raise click.BadParameter(
|
|
268
|
+
f"Invalid character '{match[0]}' for filename {filename}.")
|
|
269
|
+
# Check for forbidden names
|
|
270
|
+
if filename.upper() in FORBIDDEN_NAMES:
|
|
271
|
+
click.BadParameter(f"{filename} is a reserved folder name.")
|
|
272
|
+
# Check for empty name (disallowed in Windows)
|
|
273
|
+
if filename.strip() == "":
|
|
274
|
+
raise click.BadParameter("Empty file name not allowed.")
|
|
275
|
+
# Check for names starting or ending with dot or space
|
|
276
|
+
match = re.match(r'^[. ]|.*[. ]$', filename)
|
|
277
|
+
if match:
|
|
278
|
+
raise click.BadParameter(
|
|
279
|
+
f"Invalid start or end character ('{match[0]}')"
|
|
280
|
+
f" in file name {filename}"
|
|
281
|
+
)
|
|
282
|
+
return value
|
|
283
|
+
|
|
284
|
+
def validate_dbname(ctx, param, value):
|
|
285
|
+
if value:
|
|
286
|
+
if len(value) > 50:
|
|
287
|
+
raise click.BadParameter("Db prefix cannot be longer than 50 characters.")
|
|
288
|
+
|
|
289
|
+
result = re.fullmatch(r"[A-Za-z0-9_]+", value) is not None
|
|
290
|
+
if not result:
|
|
291
|
+
raise click.BadParameter("Invalid character(s). Only alphanumeric characters and underscore are allowed.")
|
|
292
|
+
else:
|
|
293
|
+
return value
|
|
294
|
+
else:
|
|
295
|
+
return value
|
|
296
|
+
|
|
297
|
+
@db.command()
|
|
298
|
+
@click.option("--config_file", prompt="Configuration file name", required=True, prompt_required=False,
|
|
299
|
+
type=click.Path(file_okay=True, dir_okay=False, readable=True), callback=validate_filename)
|
|
300
|
+
@click.option("--db_driver", default="sqlite", prompt="Database driver", help="Database driver",
|
|
301
|
+
required=True, prompt_required=False, type=click.Choice(["sqlite", "psycopg2", "pg8000"]),
|
|
302
|
+
callback=required_with_initcfg)
|
|
303
|
+
@click.option("-x", "--db_prefix", default="", prompt="Database prefix", required=True, prompt_required=False,
|
|
304
|
+
help="Database prefix(e.g. db_ will create database db_smf30)", callback=validate_dbname)
|
|
305
|
+
@click.option("--partitions", default="no partition", prompt="Database partition scheme", required=True, prompt_required=False,
|
|
306
|
+
help="Database partition scheme", type=click.Choice(["no partition", "weekday", "day of month", "week number"]))
|
|
307
|
+
@click.option("-h", "--host", is_eager=True, default=None, help="Host address of the database.", callback=validate_hostname)
|
|
308
|
+
@click.option("-p", "--port", is_eager=True, default=None, callback=validate_port,
|
|
309
|
+
help="Port number at which the database instance is listening.")
|
|
310
|
+
@click.option('--ssh', is_flag=True, default=False, callback=prompt_ssh_with_initcfg,
|
|
311
|
+
help="Use SSH connection.")
|
|
312
|
+
@click.option("--sqlite_path", is_eager=True, type=click.Path(exists=True, file_okay=False, readable=True),
|
|
313
|
+
default=None, help="Path to sqlite file")
|
|
314
|
+
@click.option('--ssh_host', is_eager=True, default=None, help="SSH host", callback=validate_hostname)
|
|
315
|
+
@click.option('--ssh_port', is_eager=True, default=None, required=False, callback=validate_port,
|
|
316
|
+
help="SSH port")
|
|
317
|
+
def initcfg(config_file: os.PathLike, host: str, port: str, db_prefix: str, ssh: bool, ssh_host: str = None,
|
|
318
|
+
ssh_port: str = None, db_driver: str = 'sqlite', partitions: str = 'weekday', sqlite_path: str = '') -> int:
|
|
319
|
+
"""Initialize the configuration file."""
|
|
320
|
+
predefined_smf_list = ['smf', '30', '70', '71', '72', '73', '74', '75', '77', '78', '110_1', '110_2', '123']
|
|
321
|
+
partitions_scheme_dict = {'no partition': 'single', 'weekday': 'weekday', 'day of month': 'day',
|
|
322
|
+
'week number': 'week'}
|
|
323
|
+
partition_scheme = partitions_scheme_dict[partitions]
|
|
324
|
+
|
|
325
|
+
config_mgr = ConfigManager(config_file)
|
|
326
|
+
cfg = config_mgr.load_config()
|
|
327
|
+
if not isinstance(cfg, tuple):
|
|
328
|
+
click.confirm(
|
|
329
|
+
'This config file is already exist and it will be overwritten. If there is any inconsistent databases exist. they may be deleted. Do you want to continue?',
|
|
330
|
+
abort=True)
|
|
331
|
+
|
|
332
|
+
smf_feature_list = []
|
|
333
|
+
|
|
334
|
+
for smf_type in predefined_smf_list:
|
|
335
|
+
if smf_type != 'smf':
|
|
336
|
+
if smf_type in ['110_1', '110_2']:
|
|
337
|
+
schema = 'smf110'
|
|
338
|
+
else:
|
|
339
|
+
schema = f'smf{smf_type}'
|
|
340
|
+
smf = {'type': smf_type,
|
|
341
|
+
'enabled': False,
|
|
342
|
+
'dbname': f'{db_prefix}smf{smf_type}',
|
|
343
|
+
'schema': schema,
|
|
344
|
+
'summary': {'15min': False, 'hourly': False, 'daily': False}
|
|
345
|
+
}
|
|
346
|
+
else:
|
|
347
|
+
smf = {'type': smf_type,
|
|
348
|
+
'enabled': True,
|
|
349
|
+
'dbname': f'{db_prefix}{smf_type}',
|
|
350
|
+
'schema': f'{smf_type}',
|
|
351
|
+
'summary': {'15min': False, 'hourly': False, 'daily': False}
|
|
352
|
+
}
|
|
353
|
+
smf_feature_list.append(smf)
|
|
354
|
+
if db_driver != 'sqlite':
|
|
355
|
+
db_driver = f'postgresql+{db_driver}'
|
|
356
|
+
|
|
357
|
+
app_config = {
|
|
358
|
+
'database': {
|
|
359
|
+
'host': host,
|
|
360
|
+
'port': port,
|
|
361
|
+
'ssh_host': ssh_host,
|
|
362
|
+
'ssh_port': ssh_port,
|
|
363
|
+
'driver': db_driver,
|
|
364
|
+
'partition_scheme': partition_scheme,
|
|
365
|
+
'sqlite_path': sqlite_path
|
|
366
|
+
},
|
|
367
|
+
'smf': smf_feature_list
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
# Write to file
|
|
371
|
+
config_mgr.save_config(app_config)
|
|
372
|
+
return SUCCESS
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
@db.command()
|
|
376
|
+
@click.argument("smf",
|
|
377
|
+
callback=required_with_initdb,
|
|
378
|
+
nargs=-1,
|
|
379
|
+
required=True,
|
|
380
|
+
type=click.Choice(['30', '70', '71', '72', '73', '74', '75', '77', '78', '110_1', '110_2', '123']))
|
|
381
|
+
@click.option("--config_file", required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), callback=validate_config_file)
|
|
382
|
+
@click.option("-U", "--username", is_eager=True, default=None, help="Username to connect to the database.")
|
|
383
|
+
@click.option("-W", "--password", is_eager=True, default=None, help="Force password prompt.")
|
|
384
|
+
@click.option('--ssh_user', is_eager=True, default=None, help="SSH user")
|
|
385
|
+
@click.option('--ssh_password', is_eager=True, default=None, help="SSH password")
|
|
386
|
+
def initdb(smf: list, config_file: Path, username: str, password: str,
|
|
387
|
+
ssh_user=None, ssh_password=None) -> None:
|
|
388
|
+
"""Initialize the smf2db database."""
|
|
389
|
+
config_mgr = ConfigManager(config_file)
|
|
390
|
+
cfg = config_mgr.load_config()
|
|
391
|
+
|
|
392
|
+
smf_feature_list = cfg['smf']
|
|
393
|
+
smf_70_feature = {}
|
|
394
|
+
for smf_feature in smf_feature_list:
|
|
395
|
+
if smf_feature['type'] == '70':
|
|
396
|
+
smf_70_feature = smf_feature
|
|
397
|
+
break
|
|
398
|
+
|
|
399
|
+
if '70' not in smf and not smf_70_feature['enabled']:
|
|
400
|
+
for smf_type in smf:
|
|
401
|
+
if smf_type.startswith('7') or smf_type.startswith('3'):
|
|
402
|
+
click.secho(
|
|
403
|
+
'70 is the prerequisite of all other 7x and 30 smf types. '
|
|
404
|
+
"Please initialize 70 first by running `smf2db db initdb 70` before you initialize other 7x or 30 databases.",
|
|
405
|
+
err=True,
|
|
406
|
+
fg="red",
|
|
407
|
+
)
|
|
408
|
+
raise SystemExit(1)
|
|
409
|
+
|
|
410
|
+
if config_mgr.get('database.driver') == 'sqlite':
|
|
411
|
+
# Valid its path is a valid path
|
|
412
|
+
if not Path(config_mgr.get('database.sqlite_path')).exists():
|
|
413
|
+
click.secho(f"The database path {config_mgr.get('database.sqlite_path')} does not exist. Please create it.", err=True, fg="red")
|
|
414
|
+
raise SystemExit(1)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
db_list = []
|
|
418
|
+
partitions_scheme = config_mgr.get('database.partition_scheme')
|
|
419
|
+
if partitions_scheme == 'weekday':
|
|
420
|
+
partitions = range(1, 8)
|
|
421
|
+
elif partitions_scheme == 'day':
|
|
422
|
+
partitions = range(1, 32)
|
|
423
|
+
elif partitions_scheme == 'week':
|
|
424
|
+
partitions = range(1, 53)
|
|
425
|
+
else:
|
|
426
|
+
partitions = range(1, 2)
|
|
427
|
+
|
|
428
|
+
for smf_feature in smf_feature_list:
|
|
429
|
+
if smf_feature['type'] in smf:
|
|
430
|
+
original_status = smf_feature['enabled']
|
|
431
|
+
smf_feature['enabled'] = True
|
|
432
|
+
dbname_prefix = smf_feature['dbname']
|
|
433
|
+
for part in partitions:
|
|
434
|
+
db_list.append(f"{smf_feature['dbname']}_{part}")
|
|
435
|
+
if original_status:
|
|
436
|
+
click.confirm(f"This smf {smf_feature['type']} is already enabled. The database(s) with prefix {dbname_prefix} will be dropped if exist and recreated. Do you want to continue?", abort=True)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
if len(db_list) > 0:
|
|
440
|
+
click.confirm(f'The following databases will be created:\n {", ".join(db_list)}.\n Do you want to continue?', abort=True)
|
|
441
|
+
|
|
442
|
+
cfg['smf'] = smf_feature_list
|
|
443
|
+
# Write to file
|
|
444
|
+
config_mgr.save_config(cfg)
|
|
445
|
+
|
|
446
|
+
for smf_type in smf:
|
|
447
|
+
db_init_error = init_database(smf_type, config_file, username, password, ssh_user, ssh_password)
|
|
448
|
+
if db_init_error:
|
|
449
|
+
print('Database initialization failed.')
|
|
450
|
+
click.echo(f'Creating database for {smf_type} failed with "{ERRORS[db_init_error]}"')
|
|
451
|
+
raise SystemExit(1)
|
|
452
|
+
else:
|
|
453
|
+
click.echo(f"The required databases for {smf_type} have been created")
|
|
454
|
+
else:
|
|
455
|
+
click.echo('No databases were created.')
|
|
456
|
+
|
|
457
|
+
@db.command()
|
|
458
|
+
@click.argument("smf", callback=required_with_initdb,
|
|
459
|
+
type=click.Choice(['30', '70', '71', '72', '73', '74', '75', '77', '78', '110_1', '110_2', '123']))
|
|
460
|
+
@click.argument(
|
|
461
|
+
"summary",
|
|
462
|
+
type=click.Choice(['15min', 'hourly', 'daily']),
|
|
463
|
+
nargs=-1,
|
|
464
|
+
required=True,
|
|
465
|
+
)
|
|
466
|
+
@click.option("--config_file", required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), callback=validate_config_file)
|
|
467
|
+
@click.option("-U", "--username", is_eager=True, default=None, help="Username to connect to the database.")
|
|
468
|
+
@click.option("-W", "--password", is_eager=True, default=None, help="Force password prompt.")
|
|
469
|
+
@click.option('--ssh_user', is_eager=True, default=None, help="SSH user")
|
|
470
|
+
@click.option('--ssh_password', is_eager=True, default=None, help="SSH password")
|
|
471
|
+
def initsum(smf: str, config_file: Path, summary: list, username: str, password: str,
|
|
472
|
+
ssh_user=None, ssh_password=None) -> None:
|
|
473
|
+
"""Initialize the smf2db database for summarization."""
|
|
474
|
+
config_mgr = ConfigManager(config_file)
|
|
475
|
+
cfg = config_mgr.load_config()
|
|
476
|
+
if isinstance(cfg, tuple):
|
|
477
|
+
click.secho(
|
|
478
|
+
'Cannot open config file. '
|
|
479
|
+
"Please run `smf2db db initcfg` to create the config file first before you initialize the database.",
|
|
480
|
+
err=True,
|
|
481
|
+
fg="red",
|
|
482
|
+
)
|
|
483
|
+
raise SystemExit(1)
|
|
484
|
+
if config_mgr.get('database.driver') == 'sqlite':
|
|
485
|
+
# Valid its path is a valid path
|
|
486
|
+
if not Path(config_mgr.get('database.sqlite_path')).exists():
|
|
487
|
+
click.secho(f"The database path {config_mgr.get('database.sqlite_path')} does not exist. Please create it.", err=True, fg="red")
|
|
488
|
+
raise SystemExit(1)
|
|
489
|
+
|
|
490
|
+
smf_feature_list = cfg['smf']
|
|
491
|
+
target_feature = None
|
|
492
|
+
summary_db = {'15min': '15m', 'hourly': 'hr', 'daily': 'da'}
|
|
493
|
+
for smf_feature in smf_feature_list:
|
|
494
|
+
if smf_feature['type'] == smf:
|
|
495
|
+
target_feature = smf_feature
|
|
496
|
+
break
|
|
497
|
+
|
|
498
|
+
if not target_feature['enabled']:
|
|
499
|
+
click.secho(
|
|
500
|
+
'This smf type is not enabled. '
|
|
501
|
+
"Please initialize the db first by running `smf2db db initdb` before enable the summary databases.",
|
|
502
|
+
err=True,
|
|
503
|
+
fg="red",
|
|
504
|
+
)
|
|
505
|
+
raise SystemExit(1)
|
|
506
|
+
|
|
507
|
+
if smf.startswith('7') or smf.startswith('3'):
|
|
508
|
+
if '15min' in summary:
|
|
509
|
+
click.secho(
|
|
510
|
+
'15min summary level is not supported for all 7x and 30 databases. Please try again without 15min summary level.',
|
|
511
|
+
err=True,
|
|
512
|
+
fg="red",
|
|
513
|
+
)
|
|
514
|
+
raise SystemExit(1)
|
|
515
|
+
db_list = []
|
|
516
|
+
db_name_prefix = target_feature['dbname']
|
|
517
|
+
confirmed = False
|
|
518
|
+
for smf_feature in smf_feature_list:
|
|
519
|
+
if smf_feature['type'] == smf:
|
|
520
|
+
for summary_level in smf_feature['summary']:
|
|
521
|
+
if summary_level in summary:
|
|
522
|
+
db_list.append(f"{db_name_prefix}_{summary_db[summary_level]}")
|
|
523
|
+
if smf_feature['summary'][summary_level]:
|
|
524
|
+
confirmed = True
|
|
525
|
+
click.confirm(
|
|
526
|
+
f'The summary level `{summary_level}` for this smf type is already enabled. The database {db_name_prefix}_{summary_db[summary_level]} will be dropped if exist and recreated. Do you want to continue?',
|
|
527
|
+
abort=True)
|
|
528
|
+
smf_feature['summary'][summary_level] = True
|
|
529
|
+
break
|
|
530
|
+
if not confirmed:
|
|
531
|
+
click.confirm(
|
|
532
|
+
f'The following database(s) will be dropped if exist and recreated:\n {", ".join(db_list)}.\n Do you want to continue?',
|
|
533
|
+
abort=True)
|
|
534
|
+
|
|
535
|
+
cfg['smf'] = smf_feature_list
|
|
536
|
+
# Write to file
|
|
537
|
+
config_mgr.save_config(cfg)
|
|
538
|
+
|
|
539
|
+
db_init_error = init_summary(smf, config_file, summary, username, password, ssh_user, ssh_password)
|
|
540
|
+
if db_init_error:
|
|
541
|
+
click.echo(f'Creating summary database for {smf} failed with "{ERRORS[db_init_error]}"')
|
|
542
|
+
raise SystemExit(1)
|
|
543
|
+
else:
|
|
544
|
+
click.echo(f"The required summary databases for {smf} have been created")
|
|
545
|
+
|
|
546
|
+
upload.add_command(upload_commands.upload_30)
|
|
547
|
+
upload.add_command(upload_commands.upload_70)
|
|
548
|
+
upload.add_command(upload_commands.upload_71)
|
|
549
|
+
upload.add_command(upload_commands.upload_72)
|
|
550
|
+
upload.add_command(upload_commands.upload_73)
|
|
551
|
+
upload.add_command(upload_commands.upload_74)
|
|
552
|
+
upload.add_command(upload_commands.upload_75)
|
|
553
|
+
upload.add_command(upload_commands.upload_77)
|
|
554
|
+
upload.add_command(upload_commands.upload_78)
|
|
555
|
+
upload.add_command(upload_commands.upload_123)
|
|
556
|
+
upload.add_command(upload_commands.upload_110_1)
|
|
557
|
+
upload.add_command(upload_commands.upload_110_2)
|
|
558
|
+
|
|
559
|
+
report.add_command(print_commands.print_30)
|
|
560
|
+
report.add_command(print_commands.print_70)
|
|
561
|
+
report.add_command(print_commands.print_71)
|
|
562
|
+
report.add_command(print_commands.print_72)
|
|
563
|
+
report.add_command(print_commands.print_73)
|
|
564
|
+
report.add_command(print_commands.print_74)
|
|
565
|
+
report.add_command(print_commands.print_75)
|
|
566
|
+
report.add_command(print_commands.print_77)
|
|
567
|
+
report.add_command(print_commands.print_78)
|
|
568
|
+
report.add_command(print_commands.print_123)
|
|
569
|
+
report.add_command(print_commands.print_110_1)
|
|
570
|
+
report.add_command(print_commands.print_110_2)
|
|
571
|
+
|
|
572
|
+
sumup.add_command(sumup_commands.sum_30)
|
|
573
|
+
sumup.add_command(sumup_commands.sum_70)
|
|
574
|
+
sumup.add_command(sumup_commands.sum_71)
|
|
575
|
+
sumup.add_command(sumup_commands.sum_72)
|
|
576
|
+
sumup.add_command(sumup_commands.sum_73)
|
|
577
|
+
sumup.add_command(sumup_commands.sum_74)
|
|
578
|
+
sumup.add_command(sumup_commands.sum_75)
|
|
579
|
+
sumup.add_command(sumup_commands.sum_77)
|
|
580
|
+
sumup.add_command(sumup_commands.sum_78)
|
|
581
|
+
sumup.add_command(sumup_commands.sum_123)
|
|
582
|
+
sumup.add_command(sumup_commands.sum_110_1)
|
|
583
|
+
sumup.add_command(sumup_commands.sum_110_2)
|
smf2db/api/__init__.py
ADDED
|
File without changes
|