qalita 2.3.1__py3-none-any.whl → 2.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qalita/__main__.py +213 -9
- qalita/commands/{agent.py → worker.py} +89 -89
- qalita/internal/config.py +26 -19
- qalita/internal/utils.py +1 -1
- qalita/web/app.py +97 -14
- qalita/web/blueprints/context.py +13 -60
- qalita/web/blueprints/dashboard.py +35 -76
- qalita/web/blueprints/helpers.py +154 -63
- qalita/web/blueprints/sources.py +29 -61
- qalita/web/blueprints/{agents.py → workers.py} +108 -185
- qalita-2.5.2.dist-info/METADATA +66 -0
- qalita-2.5.2.dist-info/RECORD +24 -0
- {qalita-2.3.1.dist-info → qalita-2.5.2.dist-info}/WHEEL +1 -1
- qalita-2.5.2.dist-info/entry_points.txt +2 -0
- qalita/web/blueprints/studio.py +0 -1255
- qalita/web/public/chatgpt.svg +0 -3
- qalita/web/public/claude.png +0 -0
- qalita/web/public/favicon.ico +0 -0
- qalita/web/public/gemini.png +0 -0
- qalita/web/public/logo-no-slogan.png +0 -0
- qalita/web/public/logo-white-no-slogan.svg +0 -11
- qalita/web/public/mistral.svg +0 -1
- qalita/web/public/noise.webp +0 -0
- qalita/web/public/ollama.png +0 -0
- qalita/web/public/platform.png +0 -0
- qalita/web/public/sources-logos/alloy-db.png +0 -0
- qalita/web/public/sources-logos/amazon-athena.png +0 -0
- qalita/web/public/sources-logos/amazon-rds.png +0 -0
- qalita/web/public/sources-logos/api.svg +0 -2
- qalita/web/public/sources-logos/avro.svg +0 -20
- qalita/web/public/sources-logos/azure-database-mysql.png +0 -0
- qalita/web/public/sources-logos/azure-database-postgresql.png +0 -0
- qalita/web/public/sources-logos/azure-sql-database.png +0 -0
- qalita/web/public/sources-logos/azure-sql-managed-instance.png +0 -0
- qalita/web/public/sources-logos/azure-synapse-analytics.png +0 -0
- qalita/web/public/sources-logos/azure_blob.svg +0 -1
- qalita/web/public/sources-logos/bigquery.png +0 -0
- qalita/web/public/sources-logos/cassandra.svg +0 -254
- qalita/web/public/sources-logos/clickhouse.png +0 -0
- qalita/web/public/sources-logos/cloud-sql.png +0 -0
- qalita/web/public/sources-logos/cockroach-db.png +0 -0
- qalita/web/public/sources-logos/csv.svg +0 -1
- qalita/web/public/sources-logos/database.svg +0 -3
- qalita/web/public/sources-logos/databricks.png +0 -0
- qalita/web/public/sources-logos/duckdb.png +0 -0
- qalita/web/public/sources-logos/elasticsearch.svg +0 -1
- qalita/web/public/sources-logos/excel.svg +0 -1
- qalita/web/public/sources-logos/file.svg +0 -1
- qalita/web/public/sources-logos/folder.svg +0 -6
- qalita/web/public/sources-logos/gcs.png +0 -0
- qalita/web/public/sources-logos/hdfs.svg +0 -1
- qalita/web/public/sources-logos/ibm-db2.png +0 -0
- qalita/web/public/sources-logos/json.png +0 -0
- qalita/web/public/sources-logos/maria-db.png +0 -0
- qalita/web/public/sources-logos/mongodb.svg +0 -1
- qalita/web/public/sources-logos/mssql.svg +0 -1
- qalita/web/public/sources-logos/mysql.svg +0 -7
- qalita/web/public/sources-logos/oracle.svg +0 -4
- qalita/web/public/sources-logos/parquet.svg +0 -16
- qalita/web/public/sources-logos/picture.png +0 -0
- qalita/web/public/sources-logos/postgresql.svg +0 -22
- qalita/web/public/sources-logos/questdb.png +0 -0
- qalita/web/public/sources-logos/redshift.png +0 -0
- qalita/web/public/sources-logos/s3.svg +0 -34
- qalita/web/public/sources-logos/sap-hana.png +0 -0
- qalita/web/public/sources-logos/sftp.png +0 -0
- qalita/web/public/sources-logos/single-store.png +0 -0
- qalita/web/public/sources-logos/snowflake.png +0 -0
- qalita/web/public/sources-logos/sqlite.svg +0 -104
- qalita/web/public/sources-logos/sqlserver.png +0 -0
- qalita/web/public/sources-logos/starburst.png +0 -0
- qalita/web/public/sources-logos/stream.png +0 -0
- qalita/web/public/sources-logos/teradata.png +0 -0
- qalita/web/public/sources-logos/timescale.png +0 -0
- qalita/web/public/sources-logos/xls.svg +0 -1
- qalita/web/public/sources-logos/xlsx.svg +0 -1
- qalita/web/public/sources-logos/yugabyte-db.png +0 -0
- qalita/web/public/studio-logo.svg +0 -10
- qalita/web/public/studio.css +0 -304
- qalita/web/public/studio.png +0 -0
- qalita/web/public/styles.css +0 -682
- qalita/web/templates/dashboard.html +0 -373
- qalita/web/templates/navbar.html +0 -40
- qalita/web/templates/sources/added.html +0 -57
- qalita/web/templates/sources/edit.html +0 -411
- qalita/web/templates/sources/select-source.html +0 -128
- qalita/web/templates/studio/agent-panel.html +0 -769
- qalita/web/templates/studio/context-panel.html +0 -300
- qalita/web/templates/studio/index.html +0 -79
- qalita/web/templates/studio/navbar.html +0 -14
- qalita/web/templates/studio/view-panel.html +0 -529
- qalita-2.3.1.dist-info/METADATA +0 -58
- qalita-2.3.1.dist-info/RECORD +0 -101
- qalita-2.3.1.dist-info/entry_points.txt +0 -3
- {qalita-2.3.1.dist-info → qalita-2.5.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -33,30 +33,30 @@ ROUTINE_LAST_SCHEDULED_UTC = {}
|
|
|
33
33
|
"-n",
|
|
34
34
|
"--name",
|
|
35
35
|
help="The name of the agent, it will be used to identify the agent in the qalita platform",
|
|
36
|
-
envvar="
|
|
36
|
+
envvar="QALITA_WORKER_NAME",
|
|
37
37
|
)
|
|
38
38
|
@click.option(
|
|
39
39
|
"-m",
|
|
40
40
|
"--mode",
|
|
41
41
|
type=click.Choice(["job", "worker"], case_sensitive=False),
|
|
42
42
|
help="The mode of the agent, <worker/job> if you run the agent in worker mode, the agent will loop until it gets a job to do, in job mode it will immediately do a job",
|
|
43
|
-
envvar="
|
|
43
|
+
envvar="QALITA_WORKER_MODE",
|
|
44
44
|
)
|
|
45
45
|
@click.option(
|
|
46
46
|
"-t",
|
|
47
47
|
"--token",
|
|
48
|
-
help="The API token from the qalita platform, it is user scoped. Make sure you have at least the Data Engineer role to have the ability to register
|
|
49
|
-
envvar="
|
|
48
|
+
help="The API token from the qalita platform, it is user scoped. Make sure you have at least the Data Engineer role to have the ability to register workers.",
|
|
49
|
+
envvar="QALITA_WORKER_TOKEN",
|
|
50
50
|
)
|
|
51
51
|
@click.option(
|
|
52
52
|
"-u",
|
|
53
53
|
"--url",
|
|
54
54
|
help="The URL to the qalita backend the agent have to register exemple : http://backend:3080",
|
|
55
|
-
envvar="
|
|
55
|
+
envvar="QALITA_WORKER_ENDPOINT",
|
|
56
56
|
)
|
|
57
57
|
@pass_config
|
|
58
|
-
def
|
|
59
|
-
"""Manage QALITA Platform
|
|
58
|
+
def worker(config, name, mode, token, url):
|
|
59
|
+
"""Manage QALITA Platform Workers"""
|
|
60
60
|
|
|
61
61
|
all_check_pass = True
|
|
62
62
|
|
|
@@ -76,13 +76,13 @@ def agent(config, name, mode, token, url):
|
|
|
76
76
|
# Read the first found file
|
|
77
77
|
env_file = env_files[0]
|
|
78
78
|
abs_env_file = os.path.abspath(env_file)
|
|
79
|
-
logger.info(f"Using
|
|
79
|
+
logger.info(f"Using worker configuration file: [{abs_env_file}]")
|
|
80
80
|
|
|
81
81
|
# Load values from the file only if the corresponding command-line option is not provided
|
|
82
82
|
with open(abs_env_file, "r") as file:
|
|
83
83
|
for line in file:
|
|
84
84
|
key, value = line.strip().split("=")
|
|
85
|
-
key = key.lower().replace("
|
|
85
|
+
key = key.lower().replace("qalita_worker_", "")
|
|
86
86
|
if key == "name" and not name:
|
|
87
87
|
name = value
|
|
88
88
|
elif key == "mode" and not mode:
|
|
@@ -94,54 +94,54 @@ def agent(config, name, mode, token, url):
|
|
|
94
94
|
|
|
95
95
|
# Validation of required options
|
|
96
96
|
if not name:
|
|
97
|
-
logger.error("Error:
|
|
97
|
+
logger.error("Error: Worker name is required!")
|
|
98
98
|
logger.info("\tTo do so, you can set an Environment Variable : ")
|
|
99
|
-
logger.info("\t\texport
|
|
99
|
+
logger.info("\t\texport QALITA_WORKER_NAME='agent-1'")
|
|
100
100
|
logger.info("\tor add the name as a commandline argument : ")
|
|
101
|
-
logger.info("\t\tqalita
|
|
101
|
+
logger.info("\t\tqalita worker --name 'agent-1'")
|
|
102
102
|
logger.info(
|
|
103
103
|
"\tthe prefered way is to create a file '.env-file' with the values : "
|
|
104
104
|
)
|
|
105
|
-
logger.info("\t\
|
|
105
|
+
logger.info("\t\tQALITA_WORKER_NAME=agent-1")
|
|
106
106
|
logger.info("\tand source it : ")
|
|
107
107
|
logger.info("\t\texport $(xargs < .env-file)")
|
|
108
108
|
all_check_pass = False
|
|
109
109
|
if not mode:
|
|
110
|
-
logger.error("Error:
|
|
110
|
+
logger.error("Error: Worker Mode is required!")
|
|
111
111
|
logger.info("\tTo do so, you can set an Environment Variable : ")
|
|
112
|
-
logger.info("\t\texport
|
|
112
|
+
logger.info("\t\texport QALITA_WORKER_MODE='job'")
|
|
113
113
|
logger.info("\tor add the mode as a commandline argument : ")
|
|
114
|
-
logger.info("\t\tqalita
|
|
114
|
+
logger.info("\t\tqalita worker --mode 'job'")
|
|
115
115
|
logger.info(
|
|
116
116
|
"\tthe prefered way is to create a file '.env-file' with the values : "
|
|
117
117
|
)
|
|
118
|
-
logger.info("\t\
|
|
118
|
+
logger.info("\t\tQALITA_WORKER_MODE=job")
|
|
119
119
|
logger.info("\tand source it : ")
|
|
120
120
|
logger.info("\t\texport $(xargs < .env-file)")
|
|
121
121
|
all_check_pass = False
|
|
122
122
|
if not token:
|
|
123
|
-
logger.error("Error:
|
|
123
|
+
logger.error("Error: WORKER_TOKEN is required!")
|
|
124
124
|
logger.info("\tTo do so, you can set an Environment Variable : ")
|
|
125
|
-
logger.info("\t\texport
|
|
125
|
+
logger.info("\t\texport QALITA_WORKER_TOKEN='<your_api_token>'")
|
|
126
126
|
logger.info("\tor add the token as a commandline argument : ")
|
|
127
|
-
logger.info("\t\tqalita
|
|
127
|
+
logger.info("\t\tqalita worker --token '<your_api_token>'")
|
|
128
128
|
logger.info(
|
|
129
129
|
"\tthe prefered way is to create a file '.env-file' with the values : "
|
|
130
130
|
)
|
|
131
|
-
logger.info("\t\
|
|
131
|
+
logger.info("\t\tQALITA_WORKER_TOKEN=<your_api_token>")
|
|
132
132
|
logger.info("\tand source it : ")
|
|
133
133
|
logger.info("\t\texport $(xargs < .env-file)")
|
|
134
134
|
all_check_pass = False
|
|
135
135
|
if not url:
|
|
136
|
-
logger.error("Error:
|
|
136
|
+
logger.error("Error: WORKER_ENDPOINT is required!")
|
|
137
137
|
logger.info("\tTo do so, you can set an Environment Variable : ")
|
|
138
|
-
logger.info("\t\texport
|
|
138
|
+
logger.info("\t\texport QALITA_WORKER_ENDPOINT='http://localhost:3080'")
|
|
139
139
|
logger.info("\tor add the url as a commandline argument : ")
|
|
140
|
-
logger.info("\t\tqalita
|
|
140
|
+
logger.info("\t\tqalita worker --url 'agent-1'")
|
|
141
141
|
logger.info(
|
|
142
142
|
"\tthe prefered way is to create a file '.env-file' with the values : "
|
|
143
143
|
)
|
|
144
|
-
logger.info("\t\
|
|
144
|
+
logger.info("\t\tQALITA_WORKER_ENDPOINT=http://localhost:3080")
|
|
145
145
|
logger.info("\tand source it : ")
|
|
146
146
|
logger.info("\t\texport $(xargs < .env-file)")
|
|
147
147
|
all_check_pass = False
|
|
@@ -154,17 +154,17 @@ def agent(config, name, mode, token, url):
|
|
|
154
154
|
return
|
|
155
155
|
|
|
156
156
|
|
|
157
|
-
@
|
|
157
|
+
@worker.command()
|
|
158
158
|
@pass_config
|
|
159
159
|
def info(config):
|
|
160
|
-
"""Display Information about the
|
|
161
|
-
data = config.
|
|
160
|
+
"""Display Information about the worker"""
|
|
161
|
+
data = config.load_worker_config()
|
|
162
162
|
|
|
163
|
-
print("-------------
|
|
163
|
+
print("------------- Worker information -------------")
|
|
164
164
|
print(f"Name : {config.name}")
|
|
165
165
|
print(f"Mode : {config.mode}")
|
|
166
166
|
print(f"Backend URL : {config.url}")
|
|
167
|
-
print(f"Registered
|
|
167
|
+
print(f"Registered Worker Id : {data['context']['remote']['id']}")
|
|
168
168
|
|
|
169
169
|
|
|
170
170
|
@pass_config
|
|
@@ -176,31 +176,31 @@ def send_alive(config, config_file, mode="", status="online"):
|
|
|
176
176
|
remote_agent = config_file["context"]["remote"]
|
|
177
177
|
|
|
178
178
|
if not remote_agent:
|
|
179
|
-
logger.error(f"No remote
|
|
179
|
+
logger.error(f"No remote worker found with name '{config.name}'")
|
|
180
180
|
return
|
|
181
181
|
|
|
182
182
|
"""Send a keep-alive to the backend"""
|
|
183
183
|
try:
|
|
184
184
|
r = send_api_request.__wrapped__(
|
|
185
185
|
config,
|
|
186
|
-
request=f"/api/v2/
|
|
186
|
+
request=f"/api/v2/workers/{remote_agent['id']}",
|
|
187
187
|
mode="put",
|
|
188
188
|
data={"status": status},
|
|
189
189
|
) # type: ignore[attr-defined]
|
|
190
190
|
except Exception:
|
|
191
191
|
r = send_api_request(
|
|
192
|
-
request=f"/api/v2/
|
|
192
|
+
request=f"/api/v2/workers/{remote_agent['id']}",
|
|
193
193
|
mode="put",
|
|
194
194
|
data={"status": status},
|
|
195
195
|
)
|
|
196
196
|
|
|
197
197
|
if r.status_code != 200:
|
|
198
|
-
logger.warning(f"
|
|
198
|
+
logger.warning(f"Worker failed to send alive {r.status_code} - {r.text}")
|
|
199
199
|
|
|
200
200
|
|
|
201
201
|
@pass_config
|
|
202
202
|
def authenticate(config, user_id):
|
|
203
|
-
"""Authenticate the
|
|
203
|
+
"""Authenticate the worker to the QALITA Platform"""
|
|
204
204
|
try:
|
|
205
205
|
r = send_request.__wrapped__(
|
|
206
206
|
config,
|
|
@@ -210,7 +210,7 @@ def authenticate(config, user_id):
|
|
|
210
210
|
except Exception:
|
|
211
211
|
r = send_request(request=f"{config.url}/api/v2/users/{user_id}", mode="get")
|
|
212
212
|
if r.status_code == 200:
|
|
213
|
-
logger.success(f"
|
|
213
|
+
logger.success(f"Worker Authenticated to the platform at {config.url}")
|
|
214
214
|
config_json = {}
|
|
215
215
|
config_json["user"] = r.json()
|
|
216
216
|
try:
|
|
@@ -218,10 +218,10 @@ def authenticate(config, user_id):
|
|
|
218
218
|
except KeyError:
|
|
219
219
|
config_json["context"] = {}
|
|
220
220
|
config_json["context"]["local"] = config.json()
|
|
221
|
-
config.
|
|
221
|
+
config.save_worker_config(config_json)
|
|
222
222
|
else:
|
|
223
223
|
logger.error(
|
|
224
|
-
f"
|
|
224
|
+
f"Worker can't authenticate - HTTP Code : {r.status_code} - {r.text}"
|
|
225
225
|
)
|
|
226
226
|
logger.error(
|
|
227
227
|
"Make sure you have generated an API TOKEN from the qalita platform backend or web app"
|
|
@@ -231,13 +231,13 @@ def authenticate(config, user_id):
|
|
|
231
231
|
try:
|
|
232
232
|
# Récupération de la liste des agents existants
|
|
233
233
|
try:
|
|
234
|
-
r = send_api_request.__wrapped__(config, request="/api/v2/
|
|
234
|
+
r = send_api_request.__wrapped__(config, request="/api/v2/workers", mode="get") # type: ignore[attr-defined]
|
|
235
235
|
except Exception:
|
|
236
|
-
r = send_api_request(request="/api/v2/
|
|
236
|
+
r = send_api_request(request="/api/v2/workers", mode="get")
|
|
237
237
|
|
|
238
238
|
if r.status_code != 200:
|
|
239
239
|
logger.error(
|
|
240
|
-
f"
|
|
240
|
+
f"Worker can't authenticate - HTTP Code: {r.status_code} - {r.text}"
|
|
241
241
|
)
|
|
242
242
|
logger.error(
|
|
243
243
|
"Make sure you have generated an API TOKEN from the qalita platform backend or web app"
|
|
@@ -258,7 +258,7 @@ def authenticate(config, user_id):
|
|
|
258
258
|
try:
|
|
259
259
|
r = send_api_request.__wrapped__(
|
|
260
260
|
config,
|
|
261
|
-
request="/api/v2/
|
|
261
|
+
request="/api/v2/workers/create",
|
|
262
262
|
mode="post",
|
|
263
263
|
data={
|
|
264
264
|
"name": config.name,
|
|
@@ -269,7 +269,7 @@ def authenticate(config, user_id):
|
|
|
269
269
|
) # type: ignore[attr-defined]
|
|
270
270
|
except Exception:
|
|
271
271
|
r = send_api_request(
|
|
272
|
-
request="/api/v2/
|
|
272
|
+
request="/api/v2/workers/create",
|
|
273
273
|
mode="post",
|
|
274
274
|
data={
|
|
275
275
|
"name": config.name,
|
|
@@ -284,9 +284,9 @@ def authenticate(config, user_id):
|
|
|
284
284
|
|
|
285
285
|
# Fetch the full list again
|
|
286
286
|
try:
|
|
287
|
-
r = send_api_request.__wrapped__(config, request="/api/v2/
|
|
287
|
+
r = send_api_request.__wrapped__(config, request="/api/v2/workers", mode="get") # type: ignore[attr-defined]
|
|
288
288
|
except Exception:
|
|
289
|
-
r = send_api_request(request="/api/v2/
|
|
289
|
+
r = send_api_request(request="/api/v2/workers", mode="get")
|
|
290
290
|
|
|
291
291
|
if r.status_code != 200:
|
|
292
292
|
logger.error(
|
|
@@ -300,10 +300,10 @@ def authenticate(config, user_id):
|
|
|
300
300
|
sys.exit(1)
|
|
301
301
|
|
|
302
302
|
except Exception as exception:
|
|
303
|
-
logger.error(f"
|
|
303
|
+
logger.error(f"Worker can't communicate with backend: {exception}")
|
|
304
304
|
sys.exit(1)
|
|
305
305
|
|
|
306
|
-
config_json = config.
|
|
306
|
+
config_json = config.load_worker_config()
|
|
307
307
|
|
|
308
308
|
# Remove 'jobs' from each agent in the API response before updating the config
|
|
309
309
|
cleaned_agents = [
|
|
@@ -317,7 +317,7 @@ def authenticate(config, user_id):
|
|
|
317
317
|
config_json["context"]["remote"] = agent_exists
|
|
318
318
|
|
|
319
319
|
# Save the updated config
|
|
320
|
-
config.
|
|
320
|
+
config.save_worker_config(config_json)
|
|
321
321
|
|
|
322
322
|
r = send_api_request(request=f"/api/v2/registries", mode="get")
|
|
323
323
|
|
|
@@ -328,7 +328,7 @@ def authenticate(config, user_id):
|
|
|
328
328
|
pass
|
|
329
329
|
else:
|
|
330
330
|
logger.error(
|
|
331
|
-
f"
|
|
331
|
+
f"Worker can't fetch registry - HTTP Code : {r.status_code} - {r.text}"
|
|
332
332
|
)
|
|
333
333
|
logger.error(
|
|
334
334
|
"Make sure you have generated an API TOKEN from the qalita platform backend or web app"
|
|
@@ -336,16 +336,16 @@ def authenticate(config, user_id):
|
|
|
336
336
|
sys.exit(1)
|
|
337
337
|
|
|
338
338
|
registry_data = r.json()
|
|
339
|
-
config_json = config.
|
|
339
|
+
config_json = config.load_worker_config()
|
|
340
340
|
config_json["registries"] = registry_data
|
|
341
|
-
config.
|
|
341
|
+
config.save_worker_config(config_json)
|
|
342
342
|
|
|
343
343
|
|
|
344
|
-
@
|
|
344
|
+
@worker.command()
|
|
345
345
|
@pass_config
|
|
346
346
|
def login(config):
|
|
347
347
|
"""
|
|
348
|
-
Register the
|
|
348
|
+
Register the worker to the QALITA Platform
|
|
349
349
|
"""
|
|
350
350
|
if config.verbose:
|
|
351
351
|
logger.info("Verbose mode enabled")
|
|
@@ -362,47 +362,47 @@ def login(config):
|
|
|
362
362
|
"Make sure you are using compatible versions for the platform and the cli,\n\t> check compatibility matrix on the documentation <"
|
|
363
363
|
)
|
|
364
364
|
authenticate(validated_info["user_id"])
|
|
365
|
-
agent_conf = config.
|
|
365
|
+
agent_conf = config.load_worker_config()
|
|
366
366
|
send_alive(config_file=agent_conf)
|
|
367
367
|
|
|
368
368
|
|
|
369
|
-
@
|
|
369
|
+
@worker.command()
|
|
370
370
|
@pass_config
|
|
371
371
|
@click.option(
|
|
372
372
|
"-s",
|
|
373
373
|
"--source-id",
|
|
374
374
|
help="The source ID to run the job against, to get the source ID, run qalita source list",
|
|
375
|
-
envvar="
|
|
375
|
+
envvar="QALITA_WORKER_JOB_SOURCE",
|
|
376
376
|
)
|
|
377
377
|
@click.option(
|
|
378
378
|
"-sv",
|
|
379
379
|
"--source-version",
|
|
380
380
|
help="The source Version to run the job against, to get the source version, run qalita source -s <source_id> versions, default to latest",
|
|
381
|
-
envvar="
|
|
381
|
+
envvar="QALITA_WORKER_JOB_SOURCE_VERSION",
|
|
382
382
|
)
|
|
383
383
|
@click.option(
|
|
384
384
|
"-t",
|
|
385
385
|
"--target-id",
|
|
386
386
|
help="The target ID to run the job against, to get the target ID, run qalita source list",
|
|
387
|
-
envvar="
|
|
387
|
+
envvar="QALITA_WORKER_JOB_SOURCE",
|
|
388
388
|
)
|
|
389
389
|
@click.option(
|
|
390
390
|
"-tv",
|
|
391
391
|
"--target-version",
|
|
392
392
|
help="The target Version to run the job against, to get the target version, run qalita source -s <target_id> versions, default to latest",
|
|
393
|
-
envvar="
|
|
393
|
+
envvar="QALITA_WORKER_JOB_SOURCE_VERSION",
|
|
394
394
|
)
|
|
395
395
|
@click.option(
|
|
396
396
|
"-p",
|
|
397
397
|
"--pack-id",
|
|
398
398
|
help="The pack ID to run the job against, to get the pack ID, run qalita pack list",
|
|
399
|
-
envvar="
|
|
399
|
+
envvar="QALITA_WORKER_JOB_PACK",
|
|
400
400
|
)
|
|
401
401
|
@click.option(
|
|
402
402
|
"-pv",
|
|
403
403
|
"--pack-version",
|
|
404
404
|
help="The pack Version to run the job against, to get the pack version, run qalita pack -p <pack_id> versions, default to latest",
|
|
405
|
-
envvar="
|
|
405
|
+
envvar="QALITA_WORKER_JOB_PACK_VERSION",
|
|
406
406
|
)
|
|
407
407
|
def run(
|
|
408
408
|
config, source_id, source_version, target_id, target_version, pack_id, pack_version
|
|
@@ -411,31 +411,31 @@ def run(
|
|
|
411
411
|
# Pre-checks
|
|
412
412
|
if config.mode == "job":
|
|
413
413
|
if source_id is None:
|
|
414
|
-
logger.error("
|
|
414
|
+
logger.error("Worker can't run job without source")
|
|
415
415
|
logger.error(
|
|
416
|
-
"Please configure a source with --source or -s or
|
|
416
|
+
"Please configure a source with --source or -s or QALITA_WORKER_JOB_SOURCE"
|
|
417
417
|
)
|
|
418
418
|
logger.error("To get the source ID, run qalita source list")
|
|
419
419
|
sys.exit(1)
|
|
420
420
|
if pack_id is None:
|
|
421
|
-
logger.error("
|
|
421
|
+
logger.error("Worker can't run job without pack")
|
|
422
422
|
logger.error(
|
|
423
|
-
"Please configure a pack with --pack or -p or
|
|
423
|
+
"Please configure a pack with --pack or -p or QALITA_WORKER_JOB_PACK"
|
|
424
424
|
)
|
|
425
425
|
logger.error("To get the pack ID, run qalita pack list")
|
|
426
426
|
sys.exit(1)
|
|
427
|
-
logger.info("-------------
|
|
427
|
+
logger.info("------------- Worker Authenticate -------------")
|
|
428
428
|
validated_info = validate_token(config.token)
|
|
429
429
|
authenticate(validated_info["user_id"])
|
|
430
|
-
logger.info("-------------
|
|
431
|
-
agent_conf = config.
|
|
432
|
-
logger.info(f"
|
|
433
|
-
logger.info(f"
|
|
430
|
+
logger.info("------------- Worker Run -------------")
|
|
431
|
+
agent_conf = config.load_worker_config()
|
|
432
|
+
logger.info(f"Worker ID : {agent_conf['context']['remote']['id']}")
|
|
433
|
+
logger.info(f"Worker Mode : {config.mode}")
|
|
434
434
|
|
|
435
|
-
# Create a temp folder named "
|
|
436
|
-
|
|
437
|
-
if not os.path.exists(
|
|
438
|
-
os.makedirs(
|
|
435
|
+
# Create a temp folder named "jobs" if it doesn't already exist
|
|
436
|
+
jobs_path = config.get_worker_run_path()
|
|
437
|
+
if not os.path.exists(jobs_path):
|
|
438
|
+
os.makedirs(jobs_path)
|
|
439
439
|
|
|
440
440
|
last_alive_time = time.time()
|
|
441
441
|
|
|
@@ -469,7 +469,7 @@ def run(
|
|
|
469
469
|
claim_unassigned_jobs(config)
|
|
470
470
|
|
|
471
471
|
check_job = send_api_request(
|
|
472
|
-
request=f'/api/v1/
|
|
472
|
+
request=f'/api/v1/workers/{agent_conf["context"]["remote"]["id"]}/jobs/next',
|
|
473
473
|
mode="get",
|
|
474
474
|
)
|
|
475
475
|
if check_job.status_code == 200:
|
|
@@ -568,18 +568,18 @@ def pull_pack(config, pack_id, pack_version=None):
|
|
|
568
568
|
else:
|
|
569
569
|
logger.error(f"Failed to fetch pack asset: {r.text}")
|
|
570
570
|
|
|
571
|
-
|
|
571
|
+
jobs_path = config.get_worker_run_path()
|
|
572
572
|
# Système de caching, on regarde si le pack est déjà présent dans le cache sinon on le télécharge
|
|
573
573
|
file_name = pack_url.split("/")[-1]
|
|
574
574
|
bucket_name = pack_url.split("/")[3]
|
|
575
575
|
s3_folder = "/".join(pack_url.split("/")[4:-1])
|
|
576
|
-
local_path = f"{
|
|
576
|
+
local_path = f"{jobs_path}/{bucket_name}/{s3_folder}/{file_name}"
|
|
577
577
|
|
|
578
578
|
if os.path.exists(local_path):
|
|
579
579
|
logger.info(f"Using CACHED Pack at : {local_path}")
|
|
580
580
|
return local_path, pack_version
|
|
581
|
-
if not os.path.exists(f"{
|
|
582
|
-
os.makedirs(f"{
|
|
581
|
+
if not os.path.exists(f"{jobs_path}/{bucket_name}/{s3_folder}"):
|
|
582
|
+
os.makedirs(f"{jobs_path}/{bucket_name}/{s3_folder}")
|
|
583
583
|
|
|
584
584
|
# Fetch the pack from api
|
|
585
585
|
response = send_api_request(f"/api/v1/assets/{pack_asset_id}/fetch", "get")
|
|
@@ -615,7 +615,7 @@ def job_run(
|
|
|
615
615
|
logger.info(f"Pack {pack_id}:{pack_version_id}")
|
|
616
616
|
|
|
617
617
|
"""Runs a job"""
|
|
618
|
-
agent_conf = config.
|
|
618
|
+
agent_conf = config.load_worker_config()
|
|
619
619
|
send_alive(config_file=agent_conf, mode="job", status="starting")
|
|
620
620
|
|
|
621
621
|
# Get Source Version & Version ID
|
|
@@ -668,8 +668,8 @@ def job_run(
|
|
|
668
668
|
random.choice(string.ascii_lowercase + string.digits) for _ in range(5)
|
|
669
669
|
)
|
|
670
670
|
|
|
671
|
-
|
|
672
|
-
temp_folder_name = f"{
|
|
671
|
+
jobs_path = config.get_worker_run_path()
|
|
672
|
+
temp_folder_name = f"{jobs_path}/{datetime_string}_{random_seed}"
|
|
673
673
|
os.makedirs(temp_folder_name)
|
|
674
674
|
|
|
675
675
|
# Copy the downloaded pack to the temp folder
|
|
@@ -871,7 +871,7 @@ def post_run(
|
|
|
871
871
|
):
|
|
872
872
|
logger.info("------------- Job Post Run -------------")
|
|
873
873
|
|
|
874
|
-
agent_conf = config.
|
|
874
|
+
agent_conf = config.load_worker_config()
|
|
875
875
|
|
|
876
876
|
#########################################################
|
|
877
877
|
## LOGS
|
|
@@ -1016,12 +1016,12 @@ def post_run(
|
|
|
1016
1016
|
return logs_id
|
|
1017
1017
|
|
|
1018
1018
|
|
|
1019
|
-
@
|
|
1019
|
+
@worker.command()
|
|
1020
1020
|
@pass_config
|
|
1021
1021
|
def joblist(config):
|
|
1022
1022
|
"""Jobs are the tasks that the agent will execute"""
|
|
1023
1023
|
tab_jobs = []
|
|
1024
|
-
agent_conf = config.
|
|
1024
|
+
agent_conf = config.load_worker_config()
|
|
1025
1025
|
headers = [
|
|
1026
1026
|
"ID",
|
|
1027
1027
|
"Name",
|
|
@@ -1035,7 +1035,7 @@ def joblist(config):
|
|
|
1035
1035
|
]
|
|
1036
1036
|
|
|
1037
1037
|
r = send_request(
|
|
1038
|
-
request=f"{agent_conf['context']['local']['url']}/api/v2/
|
|
1038
|
+
request=f"{agent_conf['context']['local']['url']}/api/v2/workers/{agent_conf['context']['remote']['id']}",
|
|
1039
1039
|
mode="get",
|
|
1040
1040
|
)
|
|
1041
1041
|
if r.status_code == 200:
|
|
@@ -1057,7 +1057,7 @@ def joblist(config):
|
|
|
1057
1057
|
|
|
1058
1058
|
else:
|
|
1059
1059
|
logger.error(
|
|
1060
|
-
f"Error cannot fetch job list, make sure you are logged in with > qalita
|
|
1060
|
+
f"Error cannot fetch job list, make sure you are logged in with > qalita worker login : {r.status_code} - {r.reason}"
|
|
1061
1061
|
)
|
|
1062
1062
|
return
|
|
1063
1063
|
|
|
@@ -1146,7 +1146,7 @@ def check_routines(config, agent_start_datetime):
|
|
|
1146
1146
|
7. Continue ....
|
|
1147
1147
|
"""
|
|
1148
1148
|
source_conf = config.load_source_config(verbose=False)
|
|
1149
|
-
agent_conf = config.
|
|
1149
|
+
agent_conf = config.load_worker_config()
|
|
1150
1150
|
extract_ids = lambda x: [
|
|
1151
1151
|
source["id"] for source in x["sources"] if "id" in source
|
|
1152
1152
|
] # Skip if no ID
|
|
@@ -1254,7 +1254,7 @@ def claim_unassigned_jobs(config):
|
|
|
1254
1254
|
"""
|
|
1255
1255
|
try:
|
|
1256
1256
|
source_conf = config.load_source_config(verbose=False)
|
|
1257
|
-
agent_conf = config.
|
|
1257
|
+
agent_conf = config.load_worker_config()
|
|
1258
1258
|
local_source_ids = [s["id"] for s in source_conf["sources"] if "id" in s]
|
|
1259
1259
|
|
|
1260
1260
|
r = send_api_request(
|
qalita/internal/config.py
CHANGED
|
@@ -19,7 +19,7 @@ class Config(object):
|
|
|
19
19
|
self.token = ""
|
|
20
20
|
self.url = ""
|
|
21
21
|
self.verbose = False
|
|
22
|
-
self.
|
|
22
|
+
self.worker_id = None
|
|
23
23
|
self.config = None
|
|
24
24
|
self._qalita_home = os.path.expanduser("~/.qalita")
|
|
25
25
|
|
|
@@ -67,42 +67,49 @@ class Config(object):
|
|
|
67
67
|
self.save_source_config()
|
|
68
68
|
return self.config
|
|
69
69
|
|
|
70
|
-
def
|
|
71
|
-
"""Get the path for the
|
|
72
|
-
return os.path.join(self._qalita_home, ".
|
|
70
|
+
def get_worker_file_path(self):
|
|
71
|
+
"""Get the path for the worker file based on QALITA_HOME env or default."""
|
|
72
|
+
return os.path.join(self._qalita_home, ".worker")
|
|
73
73
|
|
|
74
|
-
def
|
|
75
|
-
"""Get the path for the
|
|
76
|
-
return os.path.join(self._qalita_home, "
|
|
74
|
+
def get_worker_run_path(self):
|
|
75
|
+
"""Get the path for the worker run folder based on QALITA_HOME env or default."""
|
|
76
|
+
return os.path.join(self._qalita_home, "jobs")
|
|
77
77
|
|
|
78
|
-
def
|
|
79
|
-
"""Save the
|
|
80
|
-
|
|
78
|
+
def save_worker_config(self, data):
|
|
79
|
+
"""Save the worker config in file to persist between context."""
|
|
80
|
+
worker_file_path = self.get_worker_file_path()
|
|
81
81
|
|
|
82
82
|
# Ensure the directory exists before saving the file
|
|
83
|
-
os.makedirs(os.path.dirname(
|
|
83
|
+
os.makedirs(os.path.dirname(worker_file_path), exist_ok=True)
|
|
84
84
|
|
|
85
|
-
with open(
|
|
85
|
+
with open(worker_file_path, "wb") as file: # open in binary mode
|
|
86
86
|
json_str = json.dumps(data, indent=4) # convert to json string
|
|
87
87
|
json_bytes = json_str.encode("utf-8") # convert to bytes
|
|
88
88
|
base64_bytes = base64.b64encode(json_bytes) # encode to base64
|
|
89
89
|
file.write(base64_bytes)
|
|
90
90
|
|
|
91
|
-
def
|
|
92
|
-
|
|
91
|
+
def load_worker_config(self):
|
|
92
|
+
worker_file_path = self.get_worker_file_path()
|
|
93
93
|
try:
|
|
94
|
-
with open(
|
|
94
|
+
with open(worker_file_path, "rb") as file: # open in binary mode
|
|
95
95
|
base64_bytes = file.read() # read base64
|
|
96
96
|
json_bytes = base64.b64decode(base64_bytes) # decode from base64
|
|
97
97
|
json_str = json_bytes.decode("utf-8") # convert to string
|
|
98
98
|
return json.loads(json_str) # parse json
|
|
99
99
|
except FileNotFoundError as exception:
|
|
100
|
-
logger.error(f"
|
|
101
|
-
logger.error("Make sure you have logged in before > qalita
|
|
100
|
+
logger.error(f"Worker can't load data file : {exception}")
|
|
101
|
+
logger.error("Make sure you have logged in before > qalita worker login")
|
|
102
102
|
sys.exit(1)
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
|
|
104
|
+
# Alias for backward compatibility (agent -> worker refactoring)
|
|
105
|
+
def load_agent_config(self):
|
|
106
|
+
return self.load_worker_config()
|
|
107
|
+
|
|
108
|
+
def get_agent_run_path(self):
|
|
109
|
+
return self.get_worker_run_path()
|
|
110
|
+
|
|
111
|
+
def set_worker_id(self, worker_id):
|
|
112
|
+
self.worker_id = worker_id
|
|
106
113
|
|
|
107
114
|
def json(self):
|
|
108
115
|
data = {
|