sirmordred 1.1.2__tar.gz → 1.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sirmordred-1.1.2 → sirmordred-1.1.3}/PKG-INFO +1 -1
- {sirmordred-1.1.2 → sirmordred-1.1.3}/pyproject.toml +1 -1
- sirmordred-1.1.3/sirmordred/_version.py +2 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_collection.py +2 -2
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_identities.py +2 -2
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_panels.py +4 -4
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/micro.py +42 -6
- sirmordred-1.1.2/sirmordred/_version.py +0 -2
- {sirmordred-1.1.2 → sirmordred-1.1.3}/LICENSE +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/README.md +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/menu.yaml +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/__init__.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/bin/sirmordred.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/config.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/error.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/github.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/sirmordred.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_autorefresh.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_enrich.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_manager.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/task_projects.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/find_affiliation_conflicts.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/grimoirelab_valid.yml +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/healthcheck.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/logs/all.log +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/panels_config.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/projects_json2yml.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/sirmordred/utils/setup.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/aliases.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/askbot.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/bugzilla.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/bugzillarest.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/confluence.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/crates.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/discourse.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/dockerhub.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/functest.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/gerrit.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/github-issue.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/github-pull.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/gitlab-issue.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/gitlab-merge.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/googlehits.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/jenkins.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/jira.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/launchpad.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/mediawiki-1.23.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/mediawiki-1.28.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/meetup.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/mozillaclub.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/nntp.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/phabricator.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/redmine.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/remo.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/rss.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/slack.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/stackexchange.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/telegram.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives/twitter.archive +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives-test-projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/archives-test.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/eclipse-projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/healthcheck_cache_invalid.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/healthcheck_cache_valid.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/healthcheck_cache_wrong_key.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/mordred.log +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/orgs_sortinghat.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/perceval_identities_sortinghat.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/remote_identities_sortinghat.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/task-identities-data.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/task-params-expected +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/task-params-test-projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/task-params-test.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/data/url-projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/run_tests.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/sortinghat_settings.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test-no-collection.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test-no-sh.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test-projects-no-collection.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test-projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test-repos-projects.json +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test-repos.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_config.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_github.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_healthcheck.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_sirmordred.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_studies.cfg +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_autorefresh.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_collection.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_enrich.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_identities.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_manager.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_panels.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_task_projects.py +0 -0
- {sirmordred-1.1.2 → sirmordred-1.1.3}/tests/test_wrong.cfg +0 -0
@@ -41,8 +41,8 @@ logger = logging.getLogger(__name__)
|
|
41
41
|
class TaskRawDataCollection(Task):
|
42
42
|
""" Basic class shared by all collection tasks """
|
43
43
|
|
44
|
-
def __init__(self, config, backend_section=None, allowed_repos=None):
|
45
|
-
super().__init__(config)
|
44
|
+
def __init__(self, config, sortinghat_client=None, backend_section=None, allowed_repos=None):
|
45
|
+
super().__init__(config, sortinghat_client)
|
46
46
|
|
47
47
|
self.backend_section = backend_section
|
48
48
|
self.allowed_repos = set(allowed_repos) if allowed_repos else None
|
@@ -38,8 +38,8 @@ logger = logging.getLogger(__name__)
|
|
38
38
|
class TaskIdentitiesMerge(Task):
|
39
39
|
""" Task for processing identities in SortingHat """
|
40
40
|
|
41
|
-
def __init__(self, conf,
|
42
|
-
super().__init__(conf,
|
41
|
+
def __init__(self, conf, sortinghat_client):
|
42
|
+
super().__init__(conf, sortinghat_client)
|
43
43
|
self.last_autorefresh = datetime.utcnow() # Last autorefresh date
|
44
44
|
|
45
45
|
def is_backend_task(self):
|
@@ -264,8 +264,8 @@ class TaskPanels(Task):
|
|
264
264
|
# Panels to be uploaded always, no matter the data sources configured
|
265
265
|
panels_common = panels_multi_ds + ["panels/json/about.json"]
|
266
266
|
|
267
|
-
def __init__(self, conf):
|
268
|
-
super().__init__(conf)
|
267
|
+
def __init__(self, conf, sortinghat_client=None):
|
268
|
+
super().__init__(conf, sortinghat_client)
|
269
269
|
# Read panels and menu description from yaml file
|
270
270
|
with open(self.conf['general']['menu_file'], 'r') as f:
|
271
271
|
try:
|
@@ -473,8 +473,8 @@ class TaskPanelsMenu(Task):
|
|
473
473
|
}
|
474
474
|
}
|
475
475
|
|
476
|
-
def __init__(self, conf):
|
477
|
-
super().__init__(conf)
|
476
|
+
def __init__(self, conf, sortinghat_client=None):
|
477
|
+
super().__init__(conf, sortinghat_client)
|
478
478
|
# Read panels and menu description from yaml file """
|
479
479
|
with open(self.conf['general']['menu_file'], 'r') as f:
|
480
480
|
try:
|
@@ -33,6 +33,7 @@ from sirmordred.task_identities import TaskIdentitiesMerge
|
|
33
33
|
from sirmordred.task_enrich import TaskEnrich
|
34
34
|
from sirmordred.task_panels import TaskPanels, TaskPanelsMenu
|
35
35
|
from sirmordred.task_projects import TaskProjects
|
36
|
+
from sortinghat.cli.client import SortingHatClient
|
36
37
|
|
37
38
|
COLOR_LOG_FORMAT_SUFFIX = "\033[1m %(log_color)s "
|
38
39
|
LOG_COLORS = {'DEBUG': 'white', 'INFO': 'cyan', 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red,bg_white'}
|
@@ -50,6 +51,34 @@ def main():
|
|
50
51
|
args.panels)
|
51
52
|
|
52
53
|
|
54
|
+
def create_sortinghat_client(config):
|
55
|
+
"""Create a SortingHat client"""
|
56
|
+
|
57
|
+
conf = config.get_conf()
|
58
|
+
|
59
|
+
sortinghat = conf.get('sortinghat', None)
|
60
|
+
if not sortinghat:
|
61
|
+
return None
|
62
|
+
|
63
|
+
db_user = sortinghat['user'] if sortinghat else None
|
64
|
+
db_password = sortinghat['password'] if sortinghat else None
|
65
|
+
db_host = sortinghat['host'] if sortinghat else '127.0.0.1'
|
66
|
+
db_path = sortinghat.get('path', None) if sortinghat else None
|
67
|
+
db_port = sortinghat.get('port', None) if sortinghat else None
|
68
|
+
db_ssl = sortinghat.get('ssl', False) if sortinghat else False
|
69
|
+
db_verify_ssl = sortinghat.get('verify_ssl', True) if sortinghat else True
|
70
|
+
db_tenant = sortinghat.get('tenant', True) if sortinghat else None
|
71
|
+
|
72
|
+
client = SortingHatClient(host=db_host, port=db_port,
|
73
|
+
path=db_path, ssl=db_ssl,
|
74
|
+
user=db_user, password=db_password,
|
75
|
+
verify_ssl=db_verify_ssl,
|
76
|
+
tenant=db_tenant)
|
77
|
+
client.connect()
|
78
|
+
|
79
|
+
return client
|
80
|
+
|
81
|
+
|
53
82
|
def micro_mordred(config, backend_sections, repos_to_check, raw, identities_merge, enrich, panels):
|
54
83
|
"""Execute the Mordred tasks using the configuration file.
|
55
84
|
|
@@ -66,12 +95,17 @@ def micro_mordred(config, backend_sections, repos_to_check, raw, identities_merg
|
|
66
95
|
for backend in backend_sections:
|
67
96
|
get_raw(config, backend, repos_to_check)
|
68
97
|
|
98
|
+
if identities_merge or enrich:
|
99
|
+
sortinghat_client = create_sortinghat_client(config)
|
100
|
+
else:
|
101
|
+
sortinghat_client = None
|
102
|
+
|
69
103
|
if identities_merge:
|
70
|
-
get_identities_merge(config)
|
104
|
+
get_identities_merge(config, sortinghat_client)
|
71
105
|
|
72
106
|
if enrich:
|
73
107
|
for backend in backend_sections:
|
74
|
-
get_enrich(config, backend, repos_to_check)
|
108
|
+
get_enrich(config, sortinghat_client, backend, repos_to_check)
|
75
109
|
|
76
110
|
if panels:
|
77
111
|
get_panels(config)
|
@@ -97,29 +131,31 @@ def get_raw(config, backend_section, repos_to_check=None):
|
|
97
131
|
sys.exit(-1)
|
98
132
|
|
99
133
|
|
100
|
-
def get_identities_merge(config):
|
134
|
+
def get_identities_merge(config, sortinghat_client):
|
101
135
|
"""Execute the merge identities phase
|
102
136
|
|
137
|
+
:param sortinghat_client: a SortingHat client
|
103
138
|
:param config: a Mordred config object
|
104
139
|
"""
|
105
140
|
TaskProjects(config).execute()
|
106
|
-
task = TaskIdentitiesMerge(config)
|
141
|
+
task = TaskIdentitiesMerge(config, sortinghat_client)
|
107
142
|
task.execute()
|
108
143
|
logging.info("Merging identities finished!")
|
109
144
|
|
110
145
|
|
111
|
-
def get_enrich(config, backend_section, repos_to_check=None):
|
146
|
+
def get_enrich(config, sortinghat_client, backend_section, repos_to_check=None):
|
112
147
|
"""Execute the enrich phase for a given backend section
|
113
148
|
|
114
149
|
Repos are only checked if they are in BOTH `repos_to_check` and the `projects.json`
|
115
150
|
|
116
151
|
:param config: a Mordred config object
|
152
|
+
:param sortinghat_client: a SortingHat client
|
117
153
|
:param backend_section: the backend section where the enrich phase is executed
|
118
154
|
:param repos_to_check: A list of repo URLs to check, or None to check all repos
|
119
155
|
"""
|
120
156
|
|
121
157
|
TaskProjects(config).execute()
|
122
|
-
task = TaskEnrich(config, backend_section=backend_section, allowed_repos=repos_to_check)
|
158
|
+
task = TaskEnrich(config, sortinghat_client, backend_section=backend_section, allowed_repos=repos_to_check)
|
123
159
|
try:
|
124
160
|
task.execute()
|
125
161
|
logging.info("Loading enriched data finished!")
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|