regscale-cli 6.20.1.1__py3-none-any.whl → 6.20.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of regscale-cli might be problematic. Click here for more details.
- regscale/__init__.py +1 -1
- regscale/core/app/utils/variables.py +5 -3
- regscale/integrations/commercial/__init__.py +2 -0
- regscale/integrations/commercial/burp.py +14 -0
- regscale/integrations/commercial/grype/commands.py +8 -1
- regscale/integrations/commercial/grype/scanner.py +2 -1
- regscale/integrations/commercial/jira.py +290 -133
- regscale/integrations/commercial/opentext/commands.py +14 -5
- regscale/integrations/commercial/opentext/scanner.py +3 -2
- regscale/integrations/commercial/qualys/__init__.py +3 -3
- regscale/integrations/commercial/stigv2/click_commands.py +6 -37
- regscale/integrations/commercial/tenablev2/commands.py +12 -4
- regscale/integrations/commercial/tenablev2/sc_scanner.py +21 -1
- regscale/integrations/commercial/tenablev2/sync_compliance.py +3 -0
- regscale/integrations/commercial/trivy/commands.py +11 -4
- regscale/integrations/commercial/trivy/scanner.py +2 -1
- regscale/integrations/jsonl_scanner_integration.py +8 -1
- regscale/integrations/public/cisa.py +58 -63
- regscale/integrations/public/fedramp/fedramp_cis_crm.py +88 -93
- regscale/integrations/scanner_integration.py +22 -6
- regscale/models/app_models/click.py +49 -1
- regscale/models/integration_models/burp.py +11 -8
- regscale/models/integration_models/cisa_kev_data.json +142 -21
- regscale/models/integration_models/flat_file_importer/__init__.py +36 -176
- regscale/models/integration_models/jira_task_sync.py +27 -0
- regscale/models/integration_models/qualys.py +6 -7
- regscale/models/integration_models/synqly_models/capabilities.json +1 -1
- regscale/models/regscale_models/control_implementation.py +39 -2
- regscale/models/regscale_models/regscale_model.py +49 -1
- regscale/models/regscale_models/task.py +1 -0
- regscale/regscale.py +1 -4
- regscale/utils/string.py +13 -0
- {regscale_cli-6.20.1.1.dist-info → regscale_cli-6.20.2.0.dist-info}/METADATA +1 -1
- {regscale_cli-6.20.1.1.dist-info → regscale_cli-6.20.2.0.dist-info}/RECORD +38 -38
- regscale/integrations/commercial/synqly_jira.py +0 -840
- {regscale_cli-6.20.1.1.dist-info → regscale_cli-6.20.2.0.dist-info}/LICENSE +0 -0
- {regscale_cli-6.20.1.1.dist-info → regscale_cli-6.20.2.0.dist-info}/WHEEL +0 -0
- {regscale_cli-6.20.1.1.dist-info → regscale_cli-6.20.2.0.dist-info}/entry_points.txt +0 -0
- {regscale_cli-6.20.1.1.dist-info → regscale_cli-6.20.2.0.dist-info}/top_level.txt +0 -0
|
@@ -8,10 +8,11 @@ import tempfile
|
|
|
8
8
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
9
9
|
from datetime import datetime, timedelta
|
|
10
10
|
from io import BytesIO
|
|
11
|
-
from
|
|
12
|
-
from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
|
|
11
|
+
from typing import TYPE_CHECKING, Any, Optional, Tuple, Union, Literal
|
|
13
12
|
from urllib.parse import urljoin
|
|
14
13
|
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
15
16
|
if TYPE_CHECKING:
|
|
16
17
|
from regscale.core.app.application import Application
|
|
17
18
|
|
|
@@ -21,6 +22,7 @@ from jira import Issue as jiraIssue
|
|
|
21
22
|
from jira import JIRAError
|
|
22
23
|
from rich.progress import Progress
|
|
23
24
|
|
|
25
|
+
from regscale.core.app.utils.parser_utils import safe_datetime_str
|
|
24
26
|
from regscale.core.app.api import Api
|
|
25
27
|
from regscale.core.app.logz import create_logger
|
|
26
28
|
from regscale.core.app.utils.app_utils import (
|
|
@@ -125,6 +127,16 @@ def issues(
|
|
|
125
127
|
prompt="Enter the name of the project in Jira",
|
|
126
128
|
required=True,
|
|
127
129
|
)
|
|
130
|
+
@click.option(
|
|
131
|
+
"--sync_attachments",
|
|
132
|
+
type=click.BOOL,
|
|
133
|
+
help=(
|
|
134
|
+
"Whether RegScale will sync the attachments for the issue "
|
|
135
|
+
"in the provided Jira project and vice versa. Defaults to True."
|
|
136
|
+
),
|
|
137
|
+
required=False,
|
|
138
|
+
default=True,
|
|
139
|
+
)
|
|
128
140
|
@click.option(
|
|
129
141
|
"--token_auth",
|
|
130
142
|
"-t",
|
|
@@ -135,6 +147,7 @@ def tasks(
|
|
|
135
147
|
regscale_id: int,
|
|
136
148
|
regscale_module: str,
|
|
137
149
|
jira_project: str,
|
|
150
|
+
sync_attachments: bool = True,
|
|
138
151
|
token_auth: bool = False,
|
|
139
152
|
):
|
|
140
153
|
"""Sync tasks from Jira into RegScale."""
|
|
@@ -143,12 +156,44 @@ def tasks(
|
|
|
143
156
|
parent_module=regscale_module,
|
|
144
157
|
jira_project=jira_project,
|
|
145
158
|
jira_issue_type="Task",
|
|
146
|
-
sync_attachments=
|
|
159
|
+
sync_attachments=sync_attachments,
|
|
147
160
|
sync_tasks_only=True,
|
|
148
161
|
token_auth=token_auth,
|
|
149
162
|
)
|
|
150
163
|
|
|
151
164
|
|
|
165
|
+
def get_regscale_data_and_attachments(
|
|
166
|
+
parent_id: int, parent_module: str, sync_attachments: bool = True, sync_tasks_only: bool = False
|
|
167
|
+
) -> Tuple[list[Union[Issue, Task]], dict[int, list[File]]]:
|
|
168
|
+
"""
|
|
169
|
+
Get the RegScale data and attachments for the given parent ID and module
|
|
170
|
+
|
|
171
|
+
:param int parent_id: The ID of the parent
|
|
172
|
+
:param str parent_module: The module of the parent
|
|
173
|
+
:param bool sync_attachments: Whether to sync attachments
|
|
174
|
+
:param bool sync_tasks_only: Whether to sync tasks only
|
|
175
|
+
:return: Tuple of RegScale issues, RegScale attachments
|
|
176
|
+
:rtype: Tuple[list[Union[Issue, Task]], dict[int, list[File]]]
|
|
177
|
+
"""
|
|
178
|
+
if sync_tasks_only and sync_attachments:
|
|
179
|
+
regscale_issues, regscale_attachments = Task.get_objects_and_attachments_by_parent(
|
|
180
|
+
parent_id=parent_id,
|
|
181
|
+
parent_module=parent_module,
|
|
182
|
+
)
|
|
183
|
+
elif sync_tasks_only and not sync_attachments:
|
|
184
|
+
regscale_issues = Task.get_all_by_parent(parent_id, parent_module)
|
|
185
|
+
regscale_attachments = []
|
|
186
|
+
elif sync_attachments:
|
|
187
|
+
regscale_issues, regscale_attachments = Issue.get_objects_and_attachments_by_parent(
|
|
188
|
+
parent_id=parent_id,
|
|
189
|
+
parent_module=parent_module,
|
|
190
|
+
)
|
|
191
|
+
else:
|
|
192
|
+
regscale_issues = Issue.get_all_by_parent(parent_id, parent_module)
|
|
193
|
+
regscale_attachments = []
|
|
194
|
+
return regscale_issues, regscale_attachments
|
|
195
|
+
|
|
196
|
+
|
|
152
197
|
def sync_regscale_and_jira(
|
|
153
198
|
parent_id: int,
|
|
154
199
|
parent_module: str,
|
|
@@ -180,20 +225,36 @@ def sync_regscale_and_jira(
|
|
|
180
225
|
# create Jira client
|
|
181
226
|
jira_client = create_jira_client(config, token_auth)
|
|
182
227
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
228
|
+
jql_str = (
|
|
229
|
+
f"project = {jira_project} AND issueType = {jira_issue_type}"
|
|
230
|
+
if sync_tasks_only
|
|
231
|
+
else f"project = {jira_project}"
|
|
232
|
+
)
|
|
233
|
+
regscale_objects, regscale_attachments = get_regscale_data_and_attachments(
|
|
234
|
+
parent_id=parent_id,
|
|
235
|
+
parent_module=parent_module,
|
|
236
|
+
sync_attachments=sync_attachments,
|
|
237
|
+
sync_tasks_only=sync_tasks_only,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
output_str = "task" if sync_tasks_only else "issue"
|
|
241
|
+
|
|
242
|
+
# write regscale data to a json file
|
|
243
|
+
check_file_path("artifacts")
|
|
244
|
+
file_name = f"existingRegScale{output_str}s.json"
|
|
245
|
+
file_path = Path("./artifacts") / file_name
|
|
246
|
+
save_data_to(
|
|
247
|
+
file=file_path,
|
|
248
|
+
data=[issue.dict() for issue in regscale_objects],
|
|
249
|
+
output_log=False,
|
|
250
|
+
)
|
|
251
|
+
logger.info(
|
|
252
|
+
"Saved RegScale %s(s) for %s #%i, see %s",
|
|
253
|
+
output_str,
|
|
254
|
+
parent_module,
|
|
255
|
+
parent_id,
|
|
256
|
+
str(file_path.absolute()),
|
|
257
|
+
)
|
|
197
258
|
|
|
198
259
|
jira_objects = fetch_jira_objects(
|
|
199
260
|
jira_client=jira_client,
|
|
@@ -203,10 +264,10 @@ def sync_regscale_and_jira(
|
|
|
203
264
|
sync_tasks_only=sync_tasks_only,
|
|
204
265
|
)
|
|
205
266
|
|
|
206
|
-
if
|
|
267
|
+
if regscale_objects:
|
|
207
268
|
# sync RegScale issues to Jira
|
|
208
|
-
if
|
|
209
|
-
|
|
269
|
+
if regscale_objects_to_update := sync_regscale_to_jira(
|
|
270
|
+
regscale_objects=regscale_objects,
|
|
210
271
|
jira_client=jira_client,
|
|
211
272
|
jira_project=jira_project,
|
|
212
273
|
jira_issue_type=jira_issue_type,
|
|
@@ -217,34 +278,37 @@ def sync_regscale_and_jira(
|
|
|
217
278
|
with job_progress:
|
|
218
279
|
# create task to update RegScale issues
|
|
219
280
|
updating_issues = job_progress.add_task(
|
|
220
|
-
f"[#f8b737]Updating {len(
|
|
221
|
-
total=len(
|
|
281
|
+
f"[#f8b737]Updating {len(regscale_objects_to_update)} RegScale {output_str}(s) from Jira...",
|
|
282
|
+
total=len(regscale_objects_to_update),
|
|
222
283
|
)
|
|
223
284
|
# create threads to analyze Jira issues and RegScale issues
|
|
224
285
|
create_threads(
|
|
225
286
|
process=update_regscale_issues,
|
|
226
287
|
args=(
|
|
227
|
-
|
|
228
|
-
api,
|
|
288
|
+
regscale_objects_to_update,
|
|
229
289
|
updating_issues,
|
|
230
290
|
),
|
|
231
|
-
thread_count=len(
|
|
291
|
+
thread_count=len(regscale_objects_to_update),
|
|
232
292
|
)
|
|
233
293
|
# output the final result
|
|
234
294
|
logger.info(
|
|
235
|
-
"%i/%i
|
|
236
|
-
len(
|
|
295
|
+
"%i/%i %s(s) updated in RegScale.",
|
|
296
|
+
len(regscale_objects_to_update),
|
|
237
297
|
len(update_counter),
|
|
298
|
+
output_str,
|
|
238
299
|
)
|
|
239
|
-
|
|
240
|
-
|
|
300
|
+
if sync_tasks_only:
|
|
301
|
+
# set the updated flag to True for the tasks that were updated, to prevent them from being updated again during the sync_regscale_objects_to_jira function
|
|
302
|
+
for task in regscale_objects_to_update:
|
|
303
|
+
task.extra_data["updated"] = True
|
|
304
|
+
else:
|
|
305
|
+
logger.info("No %s(s) need to be updated in RegScale.", output_str)
|
|
241
306
|
|
|
242
307
|
if jira_objects:
|
|
243
|
-
sync_regscale_objects_to_jira(
|
|
244
|
-
jira_objects,
|
|
308
|
+
return sync_regscale_objects_to_jira(
|
|
309
|
+
jira_objects, regscale_objects, sync_attachments, app, parent_id, parent_module, sync_tasks_only
|
|
245
310
|
)
|
|
246
|
-
|
|
247
|
-
logger.info(f"No {'tasks' if sync_tasks_only else 'issues'} need to be analyzed from Jira.")
|
|
311
|
+
logger.info("No %s need to be analyzed from Jira.", output_str)
|
|
248
312
|
|
|
249
313
|
|
|
250
314
|
def sync_regscale_objects_to_jira(
|
|
@@ -280,6 +344,7 @@ def sync_regscale_objects_to_jira(
|
|
|
280
344
|
tasks_inserted, tasks_updated, tasks_closed = create_and_update_regscale_tasks(
|
|
281
345
|
jira_issues=jira_issues,
|
|
282
346
|
existing_tasks=regscale_objects,
|
|
347
|
+
jira_client=jira_client,
|
|
283
348
|
parent_id=parent_id,
|
|
284
349
|
parent_module=parent_module,
|
|
285
350
|
progress=job_progress,
|
|
@@ -349,7 +414,6 @@ def update_regscale_issues(args: Tuple, thread: int) -> None:
|
|
|
349
414
|
# set up local variables from the passed args
|
|
350
415
|
(
|
|
351
416
|
regscale_issues,
|
|
352
|
-
app,
|
|
353
417
|
task,
|
|
354
418
|
) = args
|
|
355
419
|
# find which records should be executed by the current thread
|
|
@@ -425,6 +489,7 @@ def create_regscale_task_from_jira(config: dict, jira_issue: jiraIssue, parent_i
|
|
|
425
489
|
dateClosed=date_closed,
|
|
426
490
|
percentComplete=percent_complete,
|
|
427
491
|
otherIdentifier=jira_issue.key,
|
|
492
|
+
extra_data={"jiraIssue": jira_issue}, # type: ignore
|
|
428
493
|
)
|
|
429
494
|
|
|
430
495
|
|
|
@@ -447,74 +512,140 @@ def check_and_close_tasks(existing_tasks: list[Task], all_jira_titles: set[str])
|
|
|
447
512
|
return close_tasks
|
|
448
513
|
|
|
449
514
|
|
|
450
|
-
def
|
|
515
|
+
def process_tasks_for_sync(
|
|
516
|
+
config: dict,
|
|
451
517
|
jira_issues: list[jiraIssue],
|
|
452
518
|
existing_tasks: list[Task],
|
|
453
519
|
parent_id: int,
|
|
454
520
|
parent_module: str,
|
|
455
521
|
progress: Progress,
|
|
456
522
|
progress_task: Any,
|
|
457
|
-
) -> tuple[
|
|
523
|
+
) -> tuple[list[Task], list[Task], list[Task]]:
|
|
458
524
|
"""
|
|
459
|
-
Function to create
|
|
525
|
+
Function to create lists of Tasks that need to be created, updated, and closed in RegScale from Jira
|
|
460
526
|
|
|
527
|
+
:param dict config: Application config
|
|
461
528
|
:param list[jiraIssue] jira_issues: List of Jira issues to create or update in RegScale
|
|
462
529
|
:param list[Task] existing_tasks: List of existing tasks in RegScale
|
|
463
530
|
:param int parent_id: Parent record ID in RegScale
|
|
464
531
|
:param str parent_module: Parent record module in RegScale
|
|
465
532
|
:param Progress progress: Job progress object to use for updating the progress bar
|
|
466
533
|
:param Any progress_task: Task object to update the progress bar
|
|
467
|
-
:return: A tuple of
|
|
468
|
-
:rtype: tuple[
|
|
534
|
+
:return: A tuple of lists of Tasks to create, update, and close
|
|
535
|
+
:rtype: tuple[list[Task], list[Task], list[Task]]
|
|
469
536
|
"""
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
app = Application()
|
|
473
|
-
config = app.config
|
|
537
|
+
closed_statuses = ["Closed", "Cancelled"]
|
|
474
538
|
insert_tasks = []
|
|
475
539
|
update_tasks = []
|
|
476
|
-
|
|
540
|
+
close_tasks = []
|
|
477
541
|
for jira_issue in jira_issues:
|
|
478
542
|
task = create_regscale_task_from_jira(config, jira_issue, parent_id, parent_module)
|
|
479
543
|
if task not in existing_tasks:
|
|
480
|
-
# set due date to today if not provided
|
|
481
544
|
insert_tasks.append(task)
|
|
482
545
|
else:
|
|
483
|
-
existing_task = next(
|
|
546
|
+
existing_task = next(
|
|
547
|
+
(t for t in existing_tasks if t == task and not t.extra_data.get("updated", False)), None
|
|
548
|
+
)
|
|
484
549
|
task.id = existing_task.id
|
|
485
|
-
|
|
550
|
+
if (jira_issue.fields.status.name.lower() == "done" and task.status not in closed_statuses) or (
|
|
551
|
+
task.status in closed_statuses and task != existing_task
|
|
552
|
+
):
|
|
553
|
+
task.status = "Closed"
|
|
554
|
+
task.percentComplete = 100
|
|
555
|
+
task.dateClosed = safe_datetime_str(jira_issue.fields.statuscategorychangedate)
|
|
556
|
+
close_tasks.append(task)
|
|
557
|
+
elif task != existing_task:
|
|
558
|
+
update_tasks.append(task)
|
|
486
559
|
progress.update(progress_task, advance=1)
|
|
487
|
-
|
|
560
|
+
return insert_tasks, update_tasks, close_tasks
|
|
561
|
+
|
|
488
562
|
|
|
563
|
+
def create_and_update_regscale_tasks(
|
|
564
|
+
jira_issues: list[jiraIssue],
|
|
565
|
+
existing_tasks: list[Task],
|
|
566
|
+
jira_client: JIRA,
|
|
567
|
+
parent_id: int,
|
|
568
|
+
parent_module: str,
|
|
569
|
+
progress: Progress,
|
|
570
|
+
progress_task: Any,
|
|
571
|
+
) -> tuple[int, int, int]:
|
|
572
|
+
"""
|
|
573
|
+
Function to create or update Tasks in RegScale from Jira
|
|
574
|
+
|
|
575
|
+
:param list[jiraIssue] jira_issues: List of Jira issues to create or update in RegScale
|
|
576
|
+
:param list[Task] existing_tasks: List of existing tasks in RegScale
|
|
577
|
+
:param JIRA jira_client: Jira client to use for the request
|
|
578
|
+
:param int parent_id: Parent record ID in RegScale
|
|
579
|
+
:param str parent_module: Parent record module in RegScale
|
|
580
|
+
:param Progress progress: Job progress object to use for updating the progress bar
|
|
581
|
+
:param Any progress_task: Task object to update the progress bar
|
|
582
|
+
:return: A tuple of counts
|
|
583
|
+
:rtype: tuple[int, int, int]
|
|
584
|
+
"""
|
|
585
|
+
from regscale.core.app.api import Api
|
|
586
|
+
from regscale.models.integration_models.jira_task_sync import TaskSync
|
|
587
|
+
|
|
588
|
+
api = Api()
|
|
589
|
+
insert_tasks, update_tasks, close_tasks = process_tasks_for_sync(
|
|
590
|
+
config=api.app.config,
|
|
591
|
+
jira_issues=jira_issues,
|
|
592
|
+
existing_tasks=existing_tasks,
|
|
593
|
+
parent_id=parent_id,
|
|
594
|
+
parent_module=parent_module,
|
|
595
|
+
progress=progress,
|
|
596
|
+
progress_task=progress_task,
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
task_sync_operations: list[TaskSync] = []
|
|
600
|
+
if insert_tasks:
|
|
601
|
+
task_sync_operations.append(TaskSync(insert_tasks, "create"))
|
|
602
|
+
if update_tasks:
|
|
603
|
+
task_sync_operations.append(TaskSync(update_tasks, "update"))
|
|
604
|
+
if close_tasks:
|
|
605
|
+
task_sync_operations.append(TaskSync(close_tasks, "close"))
|
|
489
606
|
with progress:
|
|
490
607
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
total=len(insert_tasks),
|
|
495
|
-
)
|
|
496
|
-
create_futures = {executor.submit(task.create) for task in insert_tasks}
|
|
497
|
-
for _ in as_completed(create_futures):
|
|
498
|
-
progress.update(creating_tasks, advance=1)
|
|
499
|
-
if update_tasks:
|
|
500
|
-
update_task = progress.add_task(
|
|
501
|
-
f"[#f8b737]Updating {len(update_tasks)} task(s) in RegScale...",
|
|
502
|
-
total=len(update_tasks),
|
|
608
|
+
for task_sync_operation in task_sync_operations:
|
|
609
|
+
progress_task = progress.add_task(
|
|
610
|
+
task_sync_operation.progress_message, total=len(task_sync_operation.tasks)
|
|
503
611
|
)
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
close_futures = {executor.submit(task.save) for task in close_tasks}
|
|
513
|
-
for _ in as_completed(close_futures):
|
|
514
|
-
progress.update(closing_tasks, advance=1)
|
|
612
|
+
task_futures = {
|
|
613
|
+
executor.submit(
|
|
614
|
+
task_and_attachments_sync, operation=task_sync_operation.operation, task=task, jira_client=jira_client, api=api # type: ignore
|
|
615
|
+
)
|
|
616
|
+
for task in task_sync_operation.tasks
|
|
617
|
+
}
|
|
618
|
+
for _ in as_completed(task_futures):
|
|
619
|
+
progress.update(progress_task, advance=1)
|
|
515
620
|
return len(insert_tasks), len(update_tasks), len(close_tasks)
|
|
516
621
|
|
|
517
622
|
|
|
623
|
+
def task_and_attachments_sync(
|
|
624
|
+
operation: Literal["create", "update", "close"], task: Task, jira_client: JIRA, api: Api
|
|
625
|
+
) -> None:
|
|
626
|
+
"""
|
|
627
|
+
Function to create, update and close tasks as well as attachments between RegScale and Jira
|
|
628
|
+
|
|
629
|
+
:param Literal["create", "update", "close"] operation: Operation to perform on the tasks
|
|
630
|
+
:param Task task: Task to perform the operation on
|
|
631
|
+
:param JIRA jira_client: Jira client to use for the request
|
|
632
|
+
:param Api api: API object to use for the request
|
|
633
|
+
:rtype: None
|
|
634
|
+
"""
|
|
635
|
+
task_to_sync = None
|
|
636
|
+
if operation == "create":
|
|
637
|
+
task_to_sync = task.create()
|
|
638
|
+
elif operation in ["update", "close"]:
|
|
639
|
+
task_to_sync = task.save()
|
|
640
|
+
if task_to_sync:
|
|
641
|
+
compare_files_for_dupes_and_upload(
|
|
642
|
+
jira_issue=task.extra_data["jiraIssue"],
|
|
643
|
+
regscale_object=task_to_sync,
|
|
644
|
+
jira_client=jira_client,
|
|
645
|
+
api=api,
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
|
|
518
649
|
def create_and_update_regscale_issues(args: Tuple, thread: int) -> None:
|
|
519
650
|
"""
|
|
520
651
|
Function to create or update issues in RegScale from Jira
|
|
@@ -570,7 +701,7 @@ def create_and_update_regscale_issues(args: Tuple, thread: int) -> None:
|
|
|
570
701
|
# getting the hashes of all Jira & RegScale attachments
|
|
571
702
|
compare_files_for_dupes_and_upload(
|
|
572
703
|
jira_issue=jira_issue,
|
|
573
|
-
|
|
704
|
+
regscale_object=regscale_issue,
|
|
574
705
|
jira_client=jira_client,
|
|
575
706
|
api=Api(),
|
|
576
707
|
)
|
|
@@ -579,40 +710,45 @@ def create_and_update_regscale_issues(args: Tuple, thread: int) -> None:
|
|
|
579
710
|
|
|
580
711
|
|
|
581
712
|
def sync_regscale_to_jira(
|
|
582
|
-
|
|
713
|
+
regscale_objects: list[Union[Issue, Task]],
|
|
583
714
|
jira_client: JIRA,
|
|
584
715
|
jira_project: str,
|
|
585
716
|
jira_issue_type: str,
|
|
586
717
|
sync_attachments: bool = True,
|
|
587
718
|
attachments: Optional[dict] = None,
|
|
588
719
|
api: Optional[Api] = None,
|
|
589
|
-
) -> list[Issue]:
|
|
720
|
+
) -> list[Union[Issue, Task]]:
|
|
590
721
|
"""
|
|
591
|
-
Sync issues from RegScale to Jira
|
|
722
|
+
Sync issues or tasks from RegScale to Jira
|
|
592
723
|
|
|
593
|
-
:param list[Issue] regscale_issues: list of RegScale issues to sync to Jira
|
|
724
|
+
:param list[Union[Issue, Task]] regscale_issues: list of RegScale issues or tasks to sync to Jira
|
|
594
725
|
:param JIRA jira_client: Jira client to use for issue creation in Jira
|
|
595
726
|
:param str jira_project: Jira Project to create the issues in
|
|
596
727
|
:param str jira_issue_type: Type of issue to create in Jira
|
|
597
728
|
:param bool sync_attachments: Sync attachments from RegScale to Jira, defaults to True
|
|
598
729
|
:param Optional[dict] attachments: Dict of attachments to sync from RegScale to Jira, defaults to None
|
|
599
730
|
:param Optional[Api] api: API object to download attachments, defaults to None
|
|
600
|
-
:return: list of RegScale issues that need to be updated
|
|
601
|
-
:rtype: list[Issue]
|
|
731
|
+
:return: list of RegScale issues or tasks that need to be updated
|
|
732
|
+
:rtype: list[Union[Issue, Task]]
|
|
602
733
|
"""
|
|
603
734
|
new_issue_counter = 0
|
|
604
|
-
|
|
735
|
+
regscale_objects_to_update = []
|
|
605
736
|
with job_progress:
|
|
737
|
+
output_str = "issue" if jira_issue_type.lower() != "task" else "task"
|
|
606
738
|
# create task to create Jira issues
|
|
607
739
|
creating_issues = job_progress.add_task(
|
|
608
|
-
f"[#f8b737]Verifying {len(
|
|
609
|
-
total=len(
|
|
740
|
+
f"[#f8b737]Verifying {len(regscale_objects)} RegScale {output_str}(s) exist in Jira...",
|
|
741
|
+
total=len(regscale_objects),
|
|
610
742
|
)
|
|
611
|
-
for
|
|
612
|
-
|
|
613
|
-
|
|
743
|
+
for regscale_object in regscale_objects:
|
|
744
|
+
if (
|
|
745
|
+
isinstance(regscale_object, Issue) and (not regscale_object.jiraId or regscale_object.jiraId == "")
|
|
746
|
+
) or (
|
|
747
|
+
isinstance(regscale_object, Task)
|
|
748
|
+
and (not regscale_object.otherIdentifier or regscale_object.otherIdentifier == "")
|
|
749
|
+
):
|
|
614
750
|
new_issue = create_issue_in_jira(
|
|
615
|
-
|
|
751
|
+
regscale_object=regscale_object,
|
|
616
752
|
jira_client=jira_client,
|
|
617
753
|
jira_project=jira_project,
|
|
618
754
|
issue_type=jira_issue_type,
|
|
@@ -625,13 +761,16 @@ def sync_regscale_to_jira(
|
|
|
625
761
|
# get the Jira ID
|
|
626
762
|
jira_id = new_issue.key
|
|
627
763
|
# update the RegScale issue for the Jira link
|
|
628
|
-
|
|
764
|
+
if isinstance(regscale_object, Issue):
|
|
765
|
+
regscale_object.jiraId = jira_id
|
|
766
|
+
elif isinstance(regscale_object, Task):
|
|
767
|
+
regscale_object.otherIdentifier = jira_id
|
|
629
768
|
# add the issue to the update_issues global list
|
|
630
|
-
|
|
769
|
+
regscale_objects_to_update.append(regscale_object)
|
|
631
770
|
job_progress.update(creating_issues, advance=1)
|
|
632
771
|
# output the final result
|
|
633
|
-
logger.info("%i new
|
|
634
|
-
return
|
|
772
|
+
logger.info("%i new %s(s) opened in Jira.", new_issue_counter, output_str)
|
|
773
|
+
return regscale_objects_to_update
|
|
635
774
|
|
|
636
775
|
|
|
637
776
|
def fetch_jira_objects(
|
|
@@ -747,24 +886,30 @@ def map_jira_due_date(jira_issue: Optional[jiraIssue], config: dict) -> str:
|
|
|
747
886
|
return due_date
|
|
748
887
|
|
|
749
888
|
|
|
750
|
-
def _generate_jira_comment(
|
|
889
|
+
def _generate_jira_comment(regscale_object: Union[Issue, Task]) -> str:
|
|
751
890
|
"""
|
|
752
891
|
Generate a Jira comment from a RegScale issue and it's populated fields
|
|
753
892
|
|
|
754
|
-
:param Issue
|
|
893
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task to generate a Jira comment from
|
|
755
894
|
:return: Jira comment
|
|
756
895
|
:rtype: str
|
|
757
896
|
"""
|
|
897
|
+
exclude_fields = [
|
|
898
|
+
"createdById",
|
|
899
|
+
"lastUpdatedById",
|
|
900
|
+
"issueOwnerId",
|
|
901
|
+
"assignedToId",
|
|
902
|
+
"uuid",
|
|
903
|
+
] + regscale_object._exclude_graphql_fields
|
|
758
904
|
comment = ""
|
|
759
|
-
|
|
760
|
-
for field_name, field_value in issue.__dict__.items():
|
|
905
|
+
for field_name, field_value in regscale_object.__dict__.items():
|
|
761
906
|
if field_value and field_name not in exclude_fields:
|
|
762
907
|
comment += f"**{field_name}:** {field_value}\n"
|
|
763
908
|
return comment
|
|
764
909
|
|
|
765
910
|
|
|
766
911
|
def create_issue_in_jira(
|
|
767
|
-
|
|
912
|
+
regscale_object: Union[Issue, Task],
|
|
768
913
|
jira_client: JIRA,
|
|
769
914
|
jira_project: str,
|
|
770
915
|
issue_type: str,
|
|
@@ -775,7 +920,7 @@ def create_issue_in_jira(
|
|
|
775
920
|
"""
|
|
776
921
|
Create a new issue in Jira
|
|
777
922
|
|
|
778
|
-
:param Issue
|
|
923
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task object
|
|
779
924
|
:param JIRA jira_client: Jira client to use for issue creation in Jira
|
|
780
925
|
:param str jira_project: Project name in Jira to create the issue in
|
|
781
926
|
:param str issue_type: The type of issue to create in Jira
|
|
@@ -788,12 +933,12 @@ def create_issue_in_jira(
|
|
|
788
933
|
if not api:
|
|
789
934
|
api = Api()
|
|
790
935
|
try:
|
|
791
|
-
|
|
792
|
-
logger.debug("Creating Jira issue: %s",
|
|
936
|
+
regscale_object_url = f"RegScale {regscale_object.get_module_string().title()} #{regscale_object.id}: {urljoin(api.config['domain'], f'/form/{regscale_object.get_module_string()}/{regscale_object.id}')}\n\n"
|
|
937
|
+
logger.debug("Creating Jira issue: %s", regscale_object.title)
|
|
793
938
|
new_issue = jira_client.create_issue(
|
|
794
939
|
project=jira_project,
|
|
795
|
-
summary=
|
|
796
|
-
description=
|
|
940
|
+
summary=regscale_object.title,
|
|
941
|
+
description=regscale_object_url + regscale_object.description,
|
|
797
942
|
issuetype=issue_type,
|
|
798
943
|
)
|
|
799
944
|
logger.debug("Jira issue created: %s", new_issue.key)
|
|
@@ -801,7 +946,7 @@ def create_issue_in_jira(
|
|
|
801
946
|
logger.debug("Adding comment to Jira issue: %s", new_issue.key)
|
|
802
947
|
_ = jira_client.add_comment(
|
|
803
948
|
issue=new_issue,
|
|
804
|
-
body=
|
|
949
|
+
body=regscale_object_url + _generate_jira_comment(regscale_object),
|
|
805
950
|
)
|
|
806
951
|
logger.debug("Comment added to Jira issue: %s", new_issue.key)
|
|
807
952
|
except JIRAError as ex:
|
|
@@ -810,7 +955,7 @@ def create_issue_in_jira(
|
|
|
810
955
|
if add_attachments and attachments:
|
|
811
956
|
compare_files_for_dupes_and_upload(
|
|
812
957
|
jira_issue=new_issue,
|
|
813
|
-
|
|
958
|
+
regscale_object=regscale_object,
|
|
814
959
|
jira_client=jira_client,
|
|
815
960
|
api=api,
|
|
816
961
|
)
|
|
@@ -818,13 +963,13 @@ def create_issue_in_jira(
|
|
|
818
963
|
|
|
819
964
|
|
|
820
965
|
def compare_files_for_dupes_and_upload(
|
|
821
|
-
jira_issue: jiraIssue,
|
|
966
|
+
jira_issue: jiraIssue, regscale_object: Union[Issue, Task], jira_client: JIRA, api: Api
|
|
822
967
|
) -> None:
|
|
823
968
|
"""
|
|
824
969
|
Compare files for duplicates and upload them to Jira and RegScale
|
|
825
970
|
|
|
826
971
|
:param jiraIssue jira_issue: Jira issue to upload the attachments to
|
|
827
|
-
:param Issue
|
|
972
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task to upload the attachments from
|
|
828
973
|
:param JIRA jira_client: Jira client to use for uploading the attachments
|
|
829
974
|
:param Api api: Api object to use for interacting with RegScale
|
|
830
975
|
:rtype: None
|
|
@@ -833,10 +978,10 @@ def compare_files_for_dupes_and_upload(
|
|
|
833
978
|
jira_uploaded_attachments = []
|
|
834
979
|
regscale_uploaded_attachments = []
|
|
835
980
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
836
|
-
jira_dir, regscale_dir =
|
|
981
|
+
jira_dir, regscale_dir = download_regscale_attachments_to_directory(
|
|
837
982
|
directory=temp_dir,
|
|
838
983
|
jira_issue=jira_issue,
|
|
839
|
-
|
|
984
|
+
regscale_object=regscale_object,
|
|
840
985
|
api=api,
|
|
841
986
|
)
|
|
842
987
|
jira_attachment_hashes = compute_hashes_in_directory(jira_dir)
|
|
@@ -846,22 +991,22 @@ def compare_files_for_dupes_and_upload(
|
|
|
846
991
|
jira_attachment_hashes,
|
|
847
992
|
regscale_attachment_hashes,
|
|
848
993
|
jira_issue,
|
|
849
|
-
|
|
994
|
+
regscale_object,
|
|
850
995
|
jira_client,
|
|
851
996
|
jira_uploaded_attachments,
|
|
852
997
|
)
|
|
853
998
|
upload_files_to_regscale(
|
|
854
|
-
jira_attachment_hashes, regscale_attachment_hashes,
|
|
999
|
+
jira_attachment_hashes, regscale_attachment_hashes, regscale_object, api, regscale_uploaded_attachments
|
|
855
1000
|
)
|
|
856
1001
|
|
|
857
|
-
log_upload_results(regscale_uploaded_attachments, jira_uploaded_attachments,
|
|
1002
|
+
log_upload_results(regscale_uploaded_attachments, jira_uploaded_attachments, regscale_object, jira_issue)
|
|
858
1003
|
|
|
859
1004
|
|
|
860
1005
|
def upload_files_to_jira(
|
|
861
1006
|
jira_attachment_hashes: dict,
|
|
862
1007
|
regscale_attachment_hashes: dict,
|
|
863
1008
|
jira_issue: jiraIssue,
|
|
864
|
-
|
|
1009
|
+
regscale_object: Union[Issue, Task],
|
|
865
1010
|
jira_client: JIRA,
|
|
866
1011
|
jira_uploaded_attachments: list,
|
|
867
1012
|
) -> None:
|
|
@@ -871,7 +1016,7 @@ def upload_files_to_jira(
|
|
|
871
1016
|
:param dict jira_attachment_hashes: Dictionary of Jira attachment hashes
|
|
872
1017
|
:param dict regscale_attachment_hashes: Dictionary of RegScale attachment hashes
|
|
873
1018
|
:param jiraIssue jira_issue: Jira issue to upload the attachments to
|
|
874
|
-
:param Issue
|
|
1019
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task to upload the attachments from
|
|
875
1020
|
:param JIRA jira_client: Jira client to use for uploading the attachments
|
|
876
1021
|
:param list jira_uploaded_attachments: List of Jira attachments that were uploaded
|
|
877
1022
|
:rtype: None
|
|
@@ -884,7 +1029,10 @@ def upload_files_to_jira(
|
|
|
884
1029
|
jira_client.add_attachment(
|
|
885
1030
|
issue=jira_issue.id,
|
|
886
1031
|
attachment=BytesIO(in_file.read()), # type: ignore
|
|
887
|
-
filename=
|
|
1032
|
+
filename=(
|
|
1033
|
+
f"RegScale_{regscale_object.get_module_string().title()}_{regscale_object.id}_"
|
|
1034
|
+
f"{Path(file).name}"
|
|
1035
|
+
),
|
|
888
1036
|
)
|
|
889
1037
|
jira_uploaded_attachments.append(file)
|
|
890
1038
|
except JIRAError as ex:
|
|
@@ -906,7 +1054,7 @@ def upload_files_to_jira(
|
|
|
906
1054
|
def upload_files_to_regscale(
|
|
907
1055
|
jira_attachment_hashes: dict,
|
|
908
1056
|
regscale_attachment_hashes: dict,
|
|
909
|
-
|
|
1057
|
+
regscale_object: Union[Issue, Task],
|
|
910
1058
|
api: Api,
|
|
911
1059
|
regscale_uploaded_attachments: list,
|
|
912
1060
|
) -> None:
|
|
@@ -915,7 +1063,7 @@ def upload_files_to_regscale(
|
|
|
915
1063
|
|
|
916
1064
|
:param dict jira_attachment_hashes: Dictionary of Jira attachment hashes
|
|
917
1065
|
:param dict regscale_attachment_hashes: Dictionary of RegScale attachment hashes
|
|
918
|
-
:param Issue
|
|
1066
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task to upload the attachments to
|
|
919
1067
|
:param Api api: Api object to use for interacting with RegScale
|
|
920
1068
|
:param list regscale_uploaded_attachments: List of RegScale attachments that were uploaded
|
|
921
1069
|
:rtype: None
|
|
@@ -926,57 +1074,66 @@ def upload_files_to_regscale(
|
|
|
926
1074
|
with open(file, "rb") as in_file:
|
|
927
1075
|
if File.upload_file_to_regscale(
|
|
928
1076
|
file_name=f"Jira_attachment_{Path(file).name}",
|
|
929
|
-
parent_id=
|
|
930
|
-
parent_module=
|
|
1077
|
+
parent_id=regscale_object.id,
|
|
1078
|
+
parent_module=regscale_object.get_module_string(),
|
|
931
1079
|
api=api,
|
|
932
1080
|
file_data=in_file.read(),
|
|
933
1081
|
):
|
|
934
1082
|
regscale_uploaded_attachments.append(file)
|
|
935
1083
|
logger.debug(
|
|
936
|
-
"Uploaded %s to RegScale
|
|
1084
|
+
"Uploaded %s to RegScale %s #%i.",
|
|
937
1085
|
Path(file).name,
|
|
938
|
-
|
|
1086
|
+
regscale_object.get_module_string().title(),
|
|
1087
|
+
regscale_object.id,
|
|
939
1088
|
)
|
|
940
1089
|
else:
|
|
941
1090
|
logger.warning(
|
|
942
|
-
"Unable to upload %s to RegScale
|
|
1091
|
+
"Unable to upload %s to RegScale %s #%i.",
|
|
943
1092
|
Path(file).name,
|
|
944
|
-
|
|
1093
|
+
regscale_object.get_module_string().title(),
|
|
1094
|
+
regscale_object.id,
|
|
945
1095
|
)
|
|
946
1096
|
|
|
947
1097
|
|
|
948
1098
|
def log_upload_results(
|
|
949
|
-
regscale_uploaded_attachments: list,
|
|
1099
|
+
regscale_uploaded_attachments: list,
|
|
1100
|
+
jira_uploaded_attachments: list,
|
|
1101
|
+
regscale_object: Union[Issue, Task],
|
|
1102
|
+
jira_issue: jiraIssue,
|
|
950
1103
|
) -> None:
|
|
951
1104
|
"""
|
|
952
1105
|
Log the results of the upload process
|
|
953
1106
|
|
|
954
1107
|
:param list regscale_uploaded_attachments: List of RegScale attachments that were uploaded
|
|
955
1108
|
:param list jira_uploaded_attachments: List of Jira attachments that were uploaded
|
|
956
|
-
:param Issue
|
|
1109
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task that the attachments were uploaded to
|
|
957
1110
|
:param jiraIssue jira_issue: Jira issue that the attachments were uploaded to
|
|
958
1111
|
:rtype: None
|
|
959
1112
|
:return: None
|
|
960
1113
|
"""
|
|
961
1114
|
if regscale_uploaded_attachments and jira_uploaded_attachments:
|
|
962
1115
|
logger.info(
|
|
963
|
-
"%i file(s) uploaded to RegScale
|
|
1116
|
+
"%i file(s) uploaded to RegScale %s #%i and %i file(s) uploaded to Jira %s %s.",
|
|
964
1117
|
len(regscale_uploaded_attachments),
|
|
965
|
-
|
|
1118
|
+
regscale_object.get_module_string().title(),
|
|
1119
|
+
regscale_object.id,
|
|
966
1120
|
len(jira_uploaded_attachments),
|
|
1121
|
+
jira_issue.fields.issuetype.name,
|
|
967
1122
|
jira_issue.key,
|
|
968
1123
|
)
|
|
969
1124
|
elif jira_uploaded_attachments:
|
|
970
1125
|
logger.info(
|
|
971
|
-
"%i file(s) uploaded to Jira
|
|
1126
|
+
"%i file(s) uploaded to Jira %s %s.",
|
|
972
1127
|
len(jira_uploaded_attachments),
|
|
1128
|
+
jira_issue.fields.issuetype.name,
|
|
973
1129
|
jira_issue.key,
|
|
974
1130
|
)
|
|
975
1131
|
elif regscale_uploaded_attachments:
|
|
976
1132
|
logger.info(
|
|
977
|
-
"%i file(s) uploaded to RegScale
|
|
1133
|
+
"%i file(s) uploaded to RegScale %s #%i.",
|
|
978
1134
|
len(regscale_uploaded_attachments),
|
|
979
|
-
|
|
1135
|
+
regscale_object.get_module_string().title(),
|
|
1136
|
+
regscale_object.id,
|
|
980
1137
|
)
|
|
981
1138
|
|
|
982
1139
|
|
|
@@ -999,10 +1156,10 @@ def validate_issue_type(jira_client: JIRA, issue_type: str) -> Any:
|
|
|
999
1156
|
error_and_exit(error_desc=message)
|
|
1000
1157
|
|
|
1001
1158
|
|
|
1002
|
-
def
|
|
1159
|
+
def download_regscale_attachments_to_directory(
|
|
1003
1160
|
directory: str,
|
|
1004
1161
|
jira_issue: jiraIssue,
|
|
1005
|
-
|
|
1162
|
+
regscale_object: Union[Issue, Task],
|
|
1006
1163
|
api: Api,
|
|
1007
1164
|
) -> tuple[str, str]:
|
|
1008
1165
|
"""
|
|
@@ -1010,7 +1167,7 @@ def download_issue_attachments_to_directory(
|
|
|
1010
1167
|
|
|
1011
1168
|
:param str directory: Directory to store the files in
|
|
1012
1169
|
:param jiraIssue jira_issue: Jira issue to download the attachments for
|
|
1013
|
-
:param Issue
|
|
1170
|
+
:param Union[Issue, Task] regscale_object: RegScale issue or task to download the attachments for
|
|
1014
1171
|
:param Api api: Api object to use for interacting with RegScale
|
|
1015
1172
|
:return: Tuple of strings containing the Jira and RegScale directories
|
|
1016
1173
|
:rtype: tuple[str, str]
|
|
@@ -1025,8 +1182,8 @@ def download_issue_attachments_to_directory(
|
|
|
1025
1182
|
# get the regscale issue attachments
|
|
1026
1183
|
regscale_issue_attachments = File.get_files_for_parent_from_regscale(
|
|
1027
1184
|
api=api,
|
|
1028
|
-
parent_id=
|
|
1029
|
-
parent_module=
|
|
1185
|
+
parent_id=regscale_object.id,
|
|
1186
|
+
parent_module=regscale_object.get_module_string(),
|
|
1030
1187
|
)
|
|
1031
1188
|
# create a directory for the regscale attachments
|
|
1032
1189
|
regscale_dir = os.path.join(directory, "regscale")
|
|
@@ -1037,8 +1194,8 @@ def download_issue_attachments_to_directory(
|
|
|
1037
1194
|
file.write(
|
|
1038
1195
|
File.download_file_from_regscale_to_memory(
|
|
1039
1196
|
api=api,
|
|
1040
|
-
record_id=
|
|
1041
|
-
module=
|
|
1197
|
+
record_id=regscale_object.id,
|
|
1198
|
+
module=regscale_object.get_module_string(),
|
|
1042
1199
|
stored_name=attachment.trustedStorageName,
|
|
1043
1200
|
file_hash=(attachment.fileHash if attachment.fileHash else attachment.shaHash),
|
|
1044
1201
|
)
|