advisory-parser 1.12__tar.gz → 1.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {advisory-parser-1.12/advisory_parser.egg-info → advisory_parser-1.14}/PKG-INFO +18 -3
- {advisory-parser-1.12 → advisory_parser-1.14}/README.rst +1 -1
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/__init__.py +1 -2
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/flaw.py +0 -1
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/parser.py +2 -2
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/parsers/__init__.py +1 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/parsers/chrome.py +20 -7
- advisory_parser-1.14/advisory_parser/parsers/jenkins.py +177 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/parsers/mysql.py +59 -20
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/parsers/utils.py +31 -9
- {advisory-parser-1.12 → advisory_parser-1.14/advisory_parser.egg-info}/PKG-INFO +18 -3
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser.egg-info/SOURCES.txt +3 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser.egg-info/requires.txt +1 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/setup.py +5 -3
- advisory_parser-1.14/tests/test_data/jenkins_2023-04-12.html +1229 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_flash_parser.py +0 -1
- advisory_parser-1.14/tests/test_jenkins_parser.py +198 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_mysql_parser.py +25 -12
- {advisory-parser-1.12 → advisory_parser-1.14}/tox.ini +1 -1
- {advisory-parser-1.12 → advisory_parser-1.14}/COPYRIGHT +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/LICENSE +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/MANIFEST.in +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/exceptions.py +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser/parsers/flash.py +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser.egg-info/dependency_links.txt +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/advisory_parser.egg-info/top_level.txt +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/pyproject.toml +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/setup.cfg +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_chrome_parser.py +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_data/chrome_2017-06-15.txt +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_data/chrome_2020-02-04.txt +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_data/flash_apsb17-21.html +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_data/flash_apsb17-23.html +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_data/flash_apsb17-33.html +0 -0
- {advisory-parser-1.12 → advisory_parser-1.14}/tests/test_data/mysql_cpu-apr-2019.html +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: advisory-parser
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.14
|
|
4
4
|
Summary: Security flaw parser for upstream security advisories
|
|
5
5
|
Home-page: https://github.com/RedHatProductSecurity/advisory-parser
|
|
6
6
|
Author: Martin Prpič, Red Hat Product Security
|
|
@@ -16,7 +16,22 @@ Classifier: Programming Language :: Python :: 3.6
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.7
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.8
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
22
|
License-File: LICENSE
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.0.0
|
|
24
|
+
Requires-Dist: cvss>=2.0
|
|
25
|
+
Dynamic: author
|
|
26
|
+
Dynamic: author-email
|
|
27
|
+
Dynamic: classifier
|
|
28
|
+
Dynamic: description
|
|
29
|
+
Dynamic: home-page
|
|
30
|
+
Dynamic: keywords
|
|
31
|
+
Dynamic: license
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
Dynamic: requires-dist
|
|
34
|
+
Dynamic: summary
|
|
20
35
|
|
|
21
36
|
Advisory Parser
|
|
22
37
|
===============
|
|
@@ -54,7 +69,7 @@ Currently available parsers include:
|
|
|
54
69
|
|
|
55
70
|
"Google Chrome", `<https://chromereleases.googleblog.com/2017/06/stable-channel-update-for-desktop_15.html>`_
|
|
56
71
|
"Adobe Flash", `<https://helpx.adobe.com/security/products/flash-player/apsb17-17.html>`_
|
|
57
|
-
"Jenkins",
|
|
72
|
+
"Jenkins", `<https://www.jenkins.io/security/advisory/2023-04-12>`_
|
|
58
73
|
"MySQL", `<http://www.oracle.com/technetwork/security-advisory/cpujul2017verbose-3236625.html>`_
|
|
59
74
|
"phpMyAdmin", ""
|
|
60
75
|
"Wireshark", ""
|
|
@@ -34,7 +34,7 @@ Currently available parsers include:
|
|
|
34
34
|
|
|
35
35
|
"Google Chrome", `<https://chromereleases.googleblog.com/2017/06/stable-channel-update-for-desktop_15.html>`_
|
|
36
36
|
"Adobe Flash", `<https://helpx.adobe.com/security/products/flash-player/apsb17-17.html>`_
|
|
37
|
-
"Jenkins",
|
|
37
|
+
"Jenkins", `<https://www.jenkins.io/security/advisory/2023-04-12>`_
|
|
38
38
|
"MySQL", `<http://www.oracle.com/technetwork/security-advisory/cpujul2017verbose-3236625.html>`_
|
|
39
39
|
"phpMyAdmin", ""
|
|
40
40
|
"Wireshark", ""
|
|
@@ -3,12 +3,15 @@
|
|
|
3
3
|
# License: LGPLv3+
|
|
4
4
|
|
|
5
5
|
import re
|
|
6
|
+
import logging
|
|
6
7
|
from datetime import datetime
|
|
7
8
|
|
|
8
9
|
from advisory_parser.exceptions import AdvisoryParserTextException
|
|
9
10
|
from advisory_parser.flaw import Flaw
|
|
10
11
|
from .utils import get_text_from_url, CVE_REGEX
|
|
11
12
|
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
12
15
|
# Chromium does not publish CVSS scores with their CVEs so these values are
|
|
13
16
|
# best-effort guesses based on impact.
|
|
14
17
|
CVSS3_MAP = {
|
|
@@ -18,6 +21,13 @@ CVSS3_MAP = {
|
|
|
18
21
|
"low": "4.3/CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L",
|
|
19
22
|
}
|
|
20
23
|
|
|
24
|
+
# Constants for Chrome advisory parsing
|
|
25
|
+
SECURITY_FIXES_HEADER = "Security Fixes"
|
|
26
|
+
LABELS_FOOTER = "Labels:\nStable updates"
|
|
27
|
+
DATE_FORMAT = "%A, %B %d, %Y"
|
|
28
|
+
CHROME_VERSION_REGEX = r"\d{2,3}\.\d\.\d{4}\.\d{2,3}"
|
|
29
|
+
CHROMIUM_ISSUE_BASE_URL = "https://code.google.com/p/chromium/issues/detail?id="
|
|
30
|
+
|
|
21
31
|
|
|
22
32
|
def parse_chrome_advisory(url):
|
|
23
33
|
advisory_text = get_text_from_url(url)
|
|
@@ -25,12 +35,15 @@ def parse_chrome_advisory(url):
|
|
|
25
35
|
# Workaround for advisories that do not use <div>s for each CVE entry. E.g.:
|
|
26
36
|
# https://chromereleases.googleblog.com/2018/04/stable-channel-update-for-desktop.html
|
|
27
37
|
advisory_text = re.sub(r"(.)\[\$", r"\1\n[$", advisory_text)
|
|
38
|
+
# 2026+ posts concatenate CVE entries on one line using [TBD][bugid],
|
|
39
|
+
# [N/A][bugid], or [$reward][bugid]. Split them into separate lines.
|
|
40
|
+
advisory_text = re.sub(r"(?<=\S)(\[(?:TBD|N/A|\$\d+)\]\[)", r"\n\1", advisory_text)
|
|
28
41
|
|
|
29
|
-
if
|
|
42
|
+
if SECURITY_FIXES_HEADER not in advisory_text:
|
|
30
43
|
raise AdvisoryParserTextException("No security fixes found in {}".format(url))
|
|
31
44
|
|
|
32
45
|
# Throw away parts of the text after the blog post
|
|
33
|
-
flaws_text = advisory_text.split(
|
|
46
|
+
flaws_text = advisory_text.split(LABELS_FOOTER)[0].strip()
|
|
34
47
|
|
|
35
48
|
# Parse out public date
|
|
36
49
|
match = re.search("^Stable Channel Update for Desktop\n(.+)", flaws_text, re.MULTILINE)
|
|
@@ -38,7 +51,7 @@ def parse_chrome_advisory(url):
|
|
|
38
51
|
raise AdvisoryParserTextException("Could not find public date in {}".format(url))
|
|
39
52
|
|
|
40
53
|
try:
|
|
41
|
-
public_date = datetime.strptime(match.group(1),
|
|
54
|
+
public_date = datetime.strptime(match.group(1), DATE_FORMAT)
|
|
42
55
|
except ValueError:
|
|
43
56
|
raise AdvisoryParserTextException(
|
|
44
57
|
"Could not parse public date ({}) from {}".format(match.group(1), url)
|
|
@@ -46,8 +59,8 @@ def parse_chrome_advisory(url):
|
|
|
46
59
|
|
|
47
60
|
# Find Chrome version, e.g. 46.0.2490.71
|
|
48
61
|
try:
|
|
49
|
-
fixed_in = re.search(
|
|
50
|
-
except ValueError:
|
|
62
|
+
fixed_in = re.search(CHROME_VERSION_REGEX, flaws_text).group(0)
|
|
63
|
+
except (AttributeError, ValueError):
|
|
51
64
|
raise AdvisoryParserTextException("Could not find fixed-in version in {}".format(url))
|
|
52
65
|
|
|
53
66
|
# Filter out lines that contain CVEs
|
|
@@ -72,7 +85,7 @@ def parse_chrome_advisory(url):
|
|
|
72
85
|
else:
|
|
73
86
|
match = re.search(r"(Critical|High|Medium|Low)", metadata)
|
|
74
87
|
if not match:
|
|
75
|
-
|
|
88
|
+
logger.warning("Could not find impact; skipping: %s", line)
|
|
76
89
|
continue
|
|
77
90
|
else:
|
|
78
91
|
impact = match.group(1)
|
|
@@ -110,7 +123,7 @@ def parse_chrome_advisory(url):
|
|
|
110
123
|
|
|
111
124
|
description += "\n\nUpstream bug(s):\n"
|
|
112
125
|
for bug in bug_ids:
|
|
113
|
-
description += "\
|
|
126
|
+
description += "\n" + CHROMIUM_ISSUE_BASE_URL + bug
|
|
114
127
|
|
|
115
128
|
com_url = (
|
|
116
129
|
url if "blogspot.com" in url else re.sub(r"blogspot\.[^/]*/", "blogspot.com/", url)
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# Copyright (c) 2019 Red Hat, Inc.
|
|
2
|
+
# Author: Shubh Bapna
|
|
3
|
+
# License: LGPLv3+
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
from advisory_parser.exceptions import AdvisoryParserTextException
|
|
9
|
+
from advisory_parser.flaw import Flaw
|
|
10
|
+
from .utils import find_tag_by_text, CVE_REGEX, find_tag_by_id, get_request
|
|
11
|
+
from cvss import CVSS3
|
|
12
|
+
|
|
13
|
+
TO_RH_SEVERITY = {
|
|
14
|
+
"None": "low",
|
|
15
|
+
"Low": "low",
|
|
16
|
+
"Medium": "moderate",
|
|
17
|
+
"High": "important",
|
|
18
|
+
"Critical": "critical",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse_jenkins_advisory(url):
|
|
23
|
+
severity_to_cvss3_map = extract_severity_to_cvss3_map(url)
|
|
24
|
+
fixes = extract_fixes(url)
|
|
25
|
+
|
|
26
|
+
# extract date from the url
|
|
27
|
+
public_date_match = re.compile(r"\d{4}-\d{2}-\d{2}").search(url).group()
|
|
28
|
+
try:
|
|
29
|
+
public_date = datetime.strptime(public_date_match, "%Y-%m-%d")
|
|
30
|
+
except ValueError:
|
|
31
|
+
raise AdvisoryParserTextException(
|
|
32
|
+
"Could not parse public date ({}) from {}".format(public_date_match, url)
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# split it into chunks of advisories
|
|
36
|
+
advisories = extract_advisories(url)
|
|
37
|
+
if len(advisories) == 0:
|
|
38
|
+
raise AdvisoryParserTextException("No security fixes found in {}".format(url))
|
|
39
|
+
|
|
40
|
+
flaws, warnings = [], []
|
|
41
|
+
for index, advisory in enumerate(advisories):
|
|
42
|
+
severity = re.search(r"(SECURITY-\d+(\s\(\d\))?)", advisory).group(1)
|
|
43
|
+
if severity not in severity_to_cvss3_map.keys():
|
|
44
|
+
warnings.append("Could not find impact or cvss; skipping: {}".format(severity))
|
|
45
|
+
continue
|
|
46
|
+
|
|
47
|
+
cves = list(set(CVE_REGEX.findall(advisory)))
|
|
48
|
+
if len(cves) == 0:
|
|
49
|
+
warnings.append("Could not find CVEs or bugs; skipping: {}".format(severity))
|
|
50
|
+
continue
|
|
51
|
+
|
|
52
|
+
from_url = url + "#" + severity
|
|
53
|
+
summary = advisory.strip().split("\n")[0].strip()
|
|
54
|
+
impact = severity_to_cvss3_map[severity]["impact"]
|
|
55
|
+
cvss3 = severity_to_cvss3_map[severity]["score"]
|
|
56
|
+
description = extract_description(advisory, index == len(advisories) - 1)
|
|
57
|
+
affected_plugins = extract_affected_plugins(advisory)
|
|
58
|
+
affected_plugins_fix = extract_affected_plugins_fixes(
|
|
59
|
+
affected_plugins, fixes, summary, description, severity, warnings
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
flaws.append(
|
|
63
|
+
Flaw(
|
|
64
|
+
from_url=from_url,
|
|
65
|
+
cves=cves,
|
|
66
|
+
summary=f"jenkins-plugin: {', '.join(affected_plugins)}: {summary}",
|
|
67
|
+
public_date=public_date,
|
|
68
|
+
cvss3=cvss3,
|
|
69
|
+
impact=impact,
|
|
70
|
+
fixed_in=affected_plugins_fix,
|
|
71
|
+
description=description,
|
|
72
|
+
advisory_id=severity,
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
return flaws, warnings
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def extract_description(advisory, is_last):
|
|
80
|
+
description = "\n".join(advisory.strip().split("Description:")[-1].strip().split("\n")[:-1])
|
|
81
|
+
# if it is last advisory then we have to remove more stuff from description
|
|
82
|
+
if is_last:
|
|
83
|
+
description = description.split("Severity\n")[0].strip()
|
|
84
|
+
return description
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def extract_affected_plugins(advisory):
|
|
88
|
+
return [
|
|
89
|
+
a.strip()
|
|
90
|
+
for a in re.search(r"Affected plugins?:\n((.*\n)*?.*?)\nDescription", advisory)
|
|
91
|
+
.group(1)
|
|
92
|
+
.strip()
|
|
93
|
+
.split(",")
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def extract_affected_plugins_fixes(
|
|
98
|
+
affected_plugins, fixes, summary, description, severity, warnings
|
|
99
|
+
):
|
|
100
|
+
affected_plugins_fixes = {}
|
|
101
|
+
for fix in fixes:
|
|
102
|
+
if (
|
|
103
|
+
fix.lower() in affected_plugins
|
|
104
|
+
or summary.find(fix) != -1
|
|
105
|
+
or description.find(fix) != -1
|
|
106
|
+
):
|
|
107
|
+
affected_plugins_fixes[fix.lower()] = fixes[fix]
|
|
108
|
+
|
|
109
|
+
for affected_plugin in affected_plugins:
|
|
110
|
+
if affected_plugin not in affected_plugins_fixes:
|
|
111
|
+
warnings.append(
|
|
112
|
+
"{}: Could not find a fixed version for {} plugin".format(severity, affected_plugin)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
return affected_plugins_fixes
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def extract_severity_to_cvss3_map(url):
|
|
119
|
+
severity_list = find_tag_by_text(url, "h2", re.compile(r"\s*Severity\s*")).findNext("ul")
|
|
120
|
+
severity_to_cvss3_map = {}
|
|
121
|
+
|
|
122
|
+
# jenkins marks each severity with a high, medium or low impact and links that with a url to
|
|
123
|
+
# a cvss calculator. depending on the security advisory, each high, medium or low might be
|
|
124
|
+
# different from the next high, medium or low, so we will try to extract the cvss vector to get
|
|
125
|
+
# the most accurate cvss3 score
|
|
126
|
+
for c in severity_list.findChildren("li"):
|
|
127
|
+
severity, _ = list(c.stripped_strings)
|
|
128
|
+
|
|
129
|
+
# the href url contains the cvss vector after the # example:
|
|
130
|
+
# https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H
|
|
131
|
+
css3_vector = c.findNext().attrs["href"].split("#")[-1]
|
|
132
|
+
|
|
133
|
+
# severity contains a colon at the end so have to remove that
|
|
134
|
+
score = CVSS3(css3_vector)
|
|
135
|
+
severity_to_cvss3_map[severity[:-1]] = {
|
|
136
|
+
"score": score.rh_vector(),
|
|
137
|
+
"impact": TO_RH_SEVERITY[score.severities()[0]],
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return severity_to_cvss3_map
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def extract_fixes(url):
|
|
144
|
+
fixes_list = find_tag_by_text(url, "h2", re.compile(r"\s*Fix\s*")).findNext("ul")
|
|
145
|
+
plugins_to_fix = {}
|
|
146
|
+
# jenkins does include fixes in each advisories' description but that makes it harder to extract
|
|
147
|
+
# it also does have a list of fixes for each plugin for all the advisories at the end of the page
|
|
148
|
+
for fix in fixes_list.findChildren("li"):
|
|
149
|
+
plugin = list(fix.stripped_strings)[0].split("Plugin")[0].strip()
|
|
150
|
+
version = list(fix.stripped_strings)[1].split("version")[1].strip()
|
|
151
|
+
plugins_to_fix.setdefault(plugin, []).append(version)
|
|
152
|
+
return plugins_to_fix
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def extract_advisories(url):
|
|
156
|
+
tag_list = find_tag_by_id(url, "h3", re.compile(r"SECURITY-\d+"))
|
|
157
|
+
# append end tag
|
|
158
|
+
tag_list.append(find_tag_by_text(url, "h2", re.compile(r"\s*Severity\s*")))
|
|
159
|
+
|
|
160
|
+
start, end = 0, 1
|
|
161
|
+
all_texts = []
|
|
162
|
+
|
|
163
|
+
while end < len(tag_list):
|
|
164
|
+
curr_text = ""
|
|
165
|
+
curr_tag = tag_list[start]
|
|
166
|
+
end_tag = tag_list[end]
|
|
167
|
+
|
|
168
|
+
while curr_tag != end_tag:
|
|
169
|
+
curr_text += curr_tag.get_text()
|
|
170
|
+
curr_tag = curr_tag.next_sibling
|
|
171
|
+
|
|
172
|
+
# Filter out blank lines and leading/trailing spaces
|
|
173
|
+
all_texts.append("\n".join(line.strip() for line in curr_text.splitlines() if line))
|
|
174
|
+
start = end
|
|
175
|
+
end += 1
|
|
176
|
+
|
|
177
|
+
return all_texts
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# Author: Martin Prpič, Red Hat Product Security
|
|
3
3
|
# License: LGPLv3+
|
|
4
4
|
|
|
5
|
+
import calendar
|
|
5
6
|
import re
|
|
6
7
|
from datetime import datetime, timedelta
|
|
7
8
|
|
|
@@ -14,6 +15,21 @@ from .utils import get_request, get_text_from_url, CVE_REGEX
|
|
|
14
15
|
MARIADB_VULN_PAGE = "https://mariadb.com/kb/en/security/"
|
|
15
16
|
VERSION_REGEX = re.compile(r"(\d\d?\.\d\.\d\d?)")
|
|
16
17
|
|
|
18
|
+
month_to_num = {
|
|
19
|
+
"jan": 1,
|
|
20
|
+
"feb": 2,
|
|
21
|
+
"mar": 3,
|
|
22
|
+
"apr": 4,
|
|
23
|
+
"may": 5,
|
|
24
|
+
"jun": 6,
|
|
25
|
+
"jul": 7,
|
|
26
|
+
"aug": 8,
|
|
27
|
+
"sep": 9,
|
|
28
|
+
"oct": 10,
|
|
29
|
+
"nov": 11,
|
|
30
|
+
"dec": 12,
|
|
31
|
+
}
|
|
32
|
+
|
|
17
33
|
|
|
18
34
|
def _nearest_tuesday(year, month, day=17):
|
|
19
35
|
"""For a given year and month, return nearest Tuesday to the 17th of that month
|
|
@@ -24,25 +40,8 @@ def _nearest_tuesday(year, month, day=17):
|
|
|
24
40
|
July and October."
|
|
25
41
|
[https://www.oracle.com/security-alerts/]
|
|
26
42
|
"""
|
|
27
|
-
month_to_num = {
|
|
28
|
-
"jan": 1,
|
|
29
|
-
"feb": 2,
|
|
30
|
-
"mar": 3,
|
|
31
|
-
"apr": 4,
|
|
32
|
-
"may": 5,
|
|
33
|
-
"jun": 6,
|
|
34
|
-
"jul": 7,
|
|
35
|
-
"aug": 8,
|
|
36
|
-
"sep": 9,
|
|
37
|
-
"oct": 10,
|
|
38
|
-
"nov": 11,
|
|
39
|
-
"dec": 12,
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
if month.lower() not in month_to_num:
|
|
43
|
-
raise AdvisoryParserTextException("Invalid month parsed from advisory URL:", str(month))
|
|
44
43
|
|
|
45
|
-
base_date = datetime(year,
|
|
44
|
+
base_date = datetime(year, month, day)
|
|
46
45
|
|
|
47
46
|
previous_tuesday = base_date - timedelta(days=((base_date.weekday() + 6) % 7))
|
|
48
47
|
next_tuesday = base_date + timedelta(days=((1 - base_date.weekday()) % 7))
|
|
@@ -54,8 +53,35 @@ def _nearest_tuesday(year, month, day=17):
|
|
|
54
53
|
)
|
|
55
54
|
|
|
56
55
|
|
|
57
|
-
def
|
|
56
|
+
def _third_tuesday(year, month):
|
|
57
|
+
"""For a given year and month, return the 3rd Tuesday of that month
|
|
58
|
+
|
|
59
|
+
"Critical Patch Updates provide security patches for supported Oracle on-premises products.
|
|
60
|
+
They are available to customers with valid support contracts. Starting in April 2022,
|
|
61
|
+
Critical Patch Updates are released on the third Tuesday of January, April, July, and October
|
|
62
|
+
(They were previously published on the Tuesday closest to the 17th day of January, April, July,
|
|
63
|
+
and October). The next four dates are:
|
|
64
|
+
|
|
65
|
+
16 July 2024
|
|
66
|
+
15 October 2024
|
|
67
|
+
21 January 2025
|
|
68
|
+
15 April 2025
|
|
69
|
+
"
|
|
70
|
+
[https://www.oracle.com/security-alerts/]
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
c = calendar.Calendar()
|
|
74
|
+
monthcal = c.monthdatescalendar(year, month)
|
|
75
|
+
third_tuesday = [
|
|
76
|
+
day
|
|
77
|
+
for week in monthcal
|
|
78
|
+
for day in week
|
|
79
|
+
if day.weekday() == calendar.TUESDAY and day.month == month
|
|
80
|
+
][2]
|
|
81
|
+
return third_tuesday
|
|
82
|
+
|
|
58
83
|
|
|
84
|
+
def create_mariadb_cve_map():
|
|
59
85
|
# Pull plain text of the MariaDB page since the HTML is invalid: it
|
|
60
86
|
# doesn't define </li> ending tags for list elements. The HTML would
|
|
61
87
|
# have to be parsed with a more lenient parser (html5lib), which is an
|
|
@@ -99,7 +125,20 @@ def parse_mysql_advisory(url):
|
|
|
99
125
|
# Extract the CPU's month and year from the URL since the verbose page has
|
|
100
126
|
# no dates on it
|
|
101
127
|
month, year = url_match.groups()
|
|
102
|
-
|
|
128
|
+
if month.lower() not in month_to_num:
|
|
129
|
+
raise AdvisoryParserTextException("Invalid month parsed from advisory URL:", str(month))
|
|
130
|
+
month_num = month_to_num[month.lower()]
|
|
131
|
+
year_int = int(year)
|
|
132
|
+
# Starting in April 2022, Critical Patch Updates are released on the third Tuesday of January,
|
|
133
|
+
# April, July, and October (They were previously published on the Tuesday closest to the 17th
|
|
134
|
+
# day of January, April, July, and October).
|
|
135
|
+
# [https://www.oracle.com/security-alerts/]
|
|
136
|
+
if year_int < 2022:
|
|
137
|
+
cpu_date = _nearest_tuesday(year_int, month_num)
|
|
138
|
+
elif year_int == 2022 and month_num < 4:
|
|
139
|
+
cpu_date = _nearest_tuesday(year_int, month_num)
|
|
140
|
+
else:
|
|
141
|
+
cpu_date = _third_tuesday(year_int, month_num)
|
|
103
142
|
advisory_id = "CPU {} {}".format(month.capitalize(), year)
|
|
104
143
|
|
|
105
144
|
# Fetch the CPU verbose page
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
# License: LGPLv3+
|
|
4
4
|
|
|
5
5
|
import re
|
|
6
|
+
import gzip
|
|
7
|
+
import logging
|
|
6
8
|
from urllib.error import HTTPError, URLError
|
|
7
9
|
from urllib.request import urlopen, Request
|
|
8
10
|
|
|
@@ -10,27 +12,35 @@ from bs4 import BeautifulSoup
|
|
|
10
12
|
|
|
11
13
|
from advisory_parser.exceptions import AdvisoryParserGetContentException
|
|
12
14
|
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
13
17
|
CVE_REGEX = re.compile(r"CVE-\d{4}-\d{4,}")
|
|
14
18
|
|
|
15
19
|
|
|
16
20
|
def get_request(url):
|
|
17
|
-
|
|
18
|
-
"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
+
headers = {
|
|
22
|
+
"User-Agent": "Advisory-Parser/1.0",
|
|
23
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
24
|
+
"Accept-Encoding": "gzip, deflate",
|
|
25
|
+
}
|
|
21
26
|
request = Request(url, None, headers)
|
|
22
27
|
try:
|
|
23
|
-
res = urlopen(request)
|
|
28
|
+
res = urlopen(request, timeout=30)
|
|
29
|
+
data = res.read()
|
|
30
|
+
|
|
31
|
+
# Handle gzip-compressed responses
|
|
32
|
+
if res.headers.get("Content-Encoding") == "gzip":
|
|
33
|
+
data = gzip.decompress(data)
|
|
24
34
|
except HTTPError as e:
|
|
25
|
-
error_msg = "Failed to GET with status code: {}".format(e.code)
|
|
35
|
+
error_msg = "Failed to GET {} with status code: {}".format(url, e.code)
|
|
26
36
|
raise AdvisoryParserGetContentException(error_msg)
|
|
27
37
|
except URLError as e:
|
|
28
|
-
error_msg = "Failed to establish connection: {}".format(e.reason)
|
|
38
|
+
error_msg = "Failed to establish connection to {}: {}".format(url, e.reason)
|
|
29
39
|
raise AdvisoryParserGetContentException(error_msg)
|
|
30
40
|
except ValueError:
|
|
31
|
-
raise AdvisoryParserGetContentException("Invalid URL specified.
|
|
41
|
+
raise AdvisoryParserGetContentException("Invalid URL specified: {}".format(url))
|
|
32
42
|
else:
|
|
33
|
-
return
|
|
43
|
+
return data
|
|
34
44
|
|
|
35
45
|
|
|
36
46
|
def get_text_from_url(url):
|
|
@@ -47,3 +57,15 @@ def get_text_from_url(url):
|
|
|
47
57
|
text = "\n".join(line.strip() for line in text.splitlines() if line)
|
|
48
58
|
|
|
49
59
|
return text
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def find_tag_by_text(url, tag, text):
|
|
63
|
+
html = get_request(url)
|
|
64
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
65
|
+
return soup.find(tag, text=text)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def find_tag_by_id(url, tag, tag_id):
|
|
69
|
+
html = get_request(url)
|
|
70
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
71
|
+
return soup.findAll(tag, id=tag_id)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: advisory-parser
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.14
|
|
4
4
|
Summary: Security flaw parser for upstream security advisories
|
|
5
5
|
Home-page: https://github.com/RedHatProductSecurity/advisory-parser
|
|
6
6
|
Author: Martin Prpič, Red Hat Product Security
|
|
@@ -16,7 +16,22 @@ Classifier: Programming Language :: Python :: 3.6
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.7
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.8
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
22
|
License-File: LICENSE
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.0.0
|
|
24
|
+
Requires-Dist: cvss>=2.0
|
|
25
|
+
Dynamic: author
|
|
26
|
+
Dynamic: author-email
|
|
27
|
+
Dynamic: classifier
|
|
28
|
+
Dynamic: description
|
|
29
|
+
Dynamic: home-page
|
|
30
|
+
Dynamic: keywords
|
|
31
|
+
Dynamic: license
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
Dynamic: requires-dist
|
|
34
|
+
Dynamic: summary
|
|
20
35
|
|
|
21
36
|
Advisory Parser
|
|
22
37
|
===============
|
|
@@ -54,7 +69,7 @@ Currently available parsers include:
|
|
|
54
69
|
|
|
55
70
|
"Google Chrome", `<https://chromereleases.googleblog.com/2017/06/stable-channel-update-for-desktop_15.html>`_
|
|
56
71
|
"Adobe Flash", `<https://helpx.adobe.com/security/products/flash-player/apsb17-17.html>`_
|
|
57
|
-
"Jenkins",
|
|
72
|
+
"Jenkins", `<https://www.jenkins.io/security/advisory/2023-04-12>`_
|
|
58
73
|
"MySQL", `<http://www.oracle.com/technetwork/security-advisory/cpujul2017verbose-3236625.html>`_
|
|
59
74
|
"phpMyAdmin", ""
|
|
60
75
|
"Wireshark", ""
|
|
@@ -18,14 +18,17 @@ advisory_parser.egg-info/top_level.txt
|
|
|
18
18
|
advisory_parser/parsers/__init__.py
|
|
19
19
|
advisory_parser/parsers/chrome.py
|
|
20
20
|
advisory_parser/parsers/flash.py
|
|
21
|
+
advisory_parser/parsers/jenkins.py
|
|
21
22
|
advisory_parser/parsers/mysql.py
|
|
22
23
|
advisory_parser/parsers/utils.py
|
|
23
24
|
tests/test_chrome_parser.py
|
|
24
25
|
tests/test_flash_parser.py
|
|
26
|
+
tests/test_jenkins_parser.py
|
|
25
27
|
tests/test_mysql_parser.py
|
|
26
28
|
tests/test_data/chrome_2017-06-15.txt
|
|
27
29
|
tests/test_data/chrome_2020-02-04.txt
|
|
28
30
|
tests/test_data/flash_apsb17-21.html
|
|
29
31
|
tests/test_data/flash_apsb17-23.html
|
|
30
32
|
tests/test_data/flash_apsb17-33.html
|
|
33
|
+
tests/test_data/jenkins_2023-04-12.html
|
|
31
34
|
tests/test_data/mysql_cpu-apr-2019.html
|
|
@@ -4,15 +4,14 @@
|
|
|
4
4
|
|
|
5
5
|
from setuptools import setup
|
|
6
6
|
|
|
7
|
-
|
|
8
7
|
with open("README.rst", "r") as f:
|
|
9
8
|
description = f.read()
|
|
10
9
|
|
|
11
|
-
requires = ["beautifulsoup4>=4.0.0"]
|
|
10
|
+
requires = ["beautifulsoup4>=4.0.0", "cvss>=2.0"]
|
|
12
11
|
|
|
13
12
|
setup(
|
|
14
13
|
name="advisory-parser",
|
|
15
|
-
version="1.
|
|
14
|
+
version="1.14",
|
|
16
15
|
description="Security flaw parser for upstream security advisories",
|
|
17
16
|
long_description=description,
|
|
18
17
|
url="https://github.com/RedHatProductSecurity/advisory-parser",
|
|
@@ -29,6 +28,9 @@ setup(
|
|
|
29
28
|
"Programming Language :: Python :: 3.7",
|
|
30
29
|
"Programming Language :: Python :: 3.8",
|
|
31
30
|
"Programming Language :: Python :: 3.9",
|
|
31
|
+
"Programming Language :: Python :: 3.10",
|
|
32
|
+
"Programming Language :: Python :: 3.11",
|
|
33
|
+
"Programming Language :: Python :: 3.12",
|
|
32
34
|
],
|
|
33
35
|
keywords="security advisory parser scraper",
|
|
34
36
|
packages=["advisory_parser", "advisory_parser.parsers"],
|