advisory-parser 1.11__tar.gz → 1.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- advisory_parser-1.13/PKG-INFO +88 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/README.rst +1 -3
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/__init__.py +1 -1
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/flaw.py +0 -1
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/parser.py +2 -2
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/parsers/__init__.py +1 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/parsers/chrome.py +1 -1
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/parsers/flash.py +1 -1
- advisory_parser-1.13/advisory_parser/parsers/jenkins.py +177 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/parsers/mysql.py +69 -22
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/parsers/utils.py +19 -2
- advisory_parser-1.13/advisory_parser.egg-info/PKG-INFO +88 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser.egg-info/SOURCES.txt +4 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser.egg-info/requires.txt +1 -0
- advisory_parser-1.13/pyproject.toml +2 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/setup.py +7 -4
- advisory_parser-1.13/tests/test_data/jenkins_2023-04-12.html +1229 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_flash_parser.py +0 -1
- advisory_parser-1.13/tests/test_jenkins_parser.py +198 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_mysql_parser.py +25 -12
- advisory-parser-1.11/PKG-INFO +0 -85
- advisory-parser-1.11/advisory_parser.egg-info/PKG-INFO +0 -85
- {advisory-parser-1.11 → advisory_parser-1.13}/COPYRIGHT +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/LICENSE +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/MANIFEST.in +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser/exceptions.py +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser.egg-info/dependency_links.txt +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/advisory_parser.egg-info/top_level.txt +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/setup.cfg +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_chrome_parser.py +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_data/chrome_2017-06-15.txt +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_data/chrome_2020-02-04.txt +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_data/flash_apsb17-21.html +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_data/flash_apsb17-23.html +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_data/flash_apsb17-33.html +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tests/test_data/mysql_cpu-apr-2019.html +0 -0
- {advisory-parser-1.11 → advisory_parser-1.13}/tox.ini +0 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: advisory-parser
|
|
3
|
+
Version: 1.13
|
|
4
|
+
Summary: Security flaw parser for upstream security advisories
|
|
5
|
+
Home-page: https://github.com/RedHatProductSecurity/advisory-parser
|
|
6
|
+
Author: Martin Prpič, Red Hat Product Security
|
|
7
|
+
Author-email: secalert@redhat.com
|
|
8
|
+
License: LGPLv3+
|
|
9
|
+
Keywords: security advisory parser scraper
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Security
|
|
13
|
+
Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.0.0
|
|
24
|
+
Requires-Dist: cvss>=2.0
|
|
25
|
+
|
|
26
|
+
Advisory Parser
|
|
27
|
+
===============
|
|
28
|
+
|
|
29
|
+
This library allows you to parse data from security advisories of certain
|
|
30
|
+
projects to extract information about security issues. The parsed
|
|
31
|
+
information includes metadata such as impact, CVSS score, summary,
|
|
32
|
+
description, and others; for a full list, see the
|
|
33
|
+
``advisory_parser/flaw.py`` file.
|
|
34
|
+
|
|
35
|
+
**DISCLAIMER**: Much of the advisory parsing is fairly fragile. Because web
|
|
36
|
+
pages change all the time, it is not uncommon for parsers to break when a
|
|
37
|
+
page is changed in some way. Also, the advisory parsers only work with the
|
|
38
|
+
latest version of the advisory pages.
|
|
39
|
+
|
|
40
|
+
The need for parsing raw security advisories in this way could be avoided
|
|
41
|
+
if vendors provided their security pages in a machine readable (and
|
|
42
|
+
preferably standardized) format. An example of this would be Red Hat's
|
|
43
|
+
security advisories that can be pulled in from a separate Security Data API
|
|
44
|
+
(`RHSA-2016:1883.json <https://access.redhat.com/labs/securitydataapi/cvrf/RHSA-2016:1883.json>`_)
|
|
45
|
+
or downloaded as an XML file
|
|
46
|
+
(`cvrf-rhsa-2016-1883.xml <https://www.redhat.com/security/data/cvrf/2016/cvrf-rhsa-2016-1883.xml>`_),
|
|
47
|
+
or OpenSSL's list of issues available in XML
|
|
48
|
+
(`vulnerabilities.xml <https://www.openssl.org/news/vulnerabilities.xml>`_).
|
|
49
|
+
|
|
50
|
+
If you are a vendor or an upstream project owner interested in providing
|
|
51
|
+
your security advisories in a machine readable format and don't know where
|
|
52
|
+
to start, feel free to reach out to mprpic@redhat.com.
|
|
53
|
+
|
|
54
|
+
Currently available parsers include:
|
|
55
|
+
|
|
56
|
+
.. csv-table::
|
|
57
|
+
:header: "Project", "Example URL"
|
|
58
|
+
:widths: 20, 80
|
|
59
|
+
|
|
60
|
+
"Google Chrome", `<https://chromereleases.googleblog.com/2017/06/stable-channel-update-for-desktop_15.html>`_
|
|
61
|
+
"Adobe Flash", `<https://helpx.adobe.com/security/products/flash-player/apsb17-17.html>`_
|
|
62
|
+
"Jenkins", `<https://www.jenkins.io/security/advisory/2023-04-12>`_
|
|
63
|
+
"MySQL", `<http://www.oracle.com/technetwork/security-advisory/cpujul2017verbose-3236625.html>`_
|
|
64
|
+
"phpMyAdmin", ""
|
|
65
|
+
"Wireshark", ""
|
|
66
|
+
|
|
67
|
+
Installation
|
|
68
|
+
------------
|
|
69
|
+
|
|
70
|
+
::
|
|
71
|
+
|
|
72
|
+
pip install advisory-parser
|
|
73
|
+
|
|
74
|
+
Usage
|
|
75
|
+
-----
|
|
76
|
+
|
|
77
|
+
.. code-block:: python
|
|
78
|
+
|
|
79
|
+
from pprint import pprint
|
|
80
|
+
from advisory_parser import Parser
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
url = 'https://helpx.adobe.com/security/products/flash-player/apsb17-17.html'
|
|
84
|
+
flaws, warnings = Parser.parse_from_url(url)
|
|
85
|
+
|
|
86
|
+
for flaw in flaws:
|
|
87
|
+
print()
|
|
88
|
+
pprint(vars(flaw))
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
Advisory Parser
|
|
2
2
|
===============
|
|
3
3
|
|
|
4
|
-
Upstream repository: `<https://github.com/mprpic/advisory-parser>`_
|
|
5
|
-
|
|
6
4
|
This library allows you to parse data from security advisories of certain
|
|
7
5
|
projects to extract information about security issues. The parsed
|
|
8
6
|
information includes metadata such as impact, CVSS score, summary,
|
|
@@ -36,7 +34,7 @@ Currently available parsers include:
|
|
|
36
34
|
|
|
37
35
|
"Google Chrome", `<https://chromereleases.googleblog.com/2017/06/stable-channel-update-for-desktop_15.html>`_
|
|
38
36
|
"Adobe Flash", `<https://helpx.adobe.com/security/products/flash-player/apsb17-17.html>`_
|
|
39
|
-
"Jenkins",
|
|
37
|
+
"Jenkins", `<https://www.jenkins.io/security/advisory/2023-04-12>`_
|
|
40
38
|
"MySQL", `<http://www.oracle.com/technetwork/security-advisory/cpujul2017verbose-3236625.html>`_
|
|
41
39
|
"phpMyAdmin", ""
|
|
42
40
|
"Wireshark", ""
|
|
@@ -46,7 +46,7 @@ def parse_chrome_advisory(url):
|
|
|
46
46
|
|
|
47
47
|
# Find Chrome version, e.g. 46.0.2490.71
|
|
48
48
|
try:
|
|
49
|
-
fixed_in = re.search(r"\d{2}\.\d\.\d{4}\.\d{2,3}", flaws_text).group(0)
|
|
49
|
+
fixed_in = re.search(r"\d{2,3}\.\d\.\d{4}\.\d{2,3}", flaws_text).group(0)
|
|
50
50
|
except ValueError:
|
|
51
51
|
raise AdvisoryParserTextException("Could not find fixed-in version in {}".format(url))
|
|
52
52
|
|
|
@@ -64,7 +64,7 @@ def parse_flash_advisory(url):
|
|
|
64
64
|
vuln_data = []
|
|
65
65
|
for row in table_rows[1:]:
|
|
66
66
|
vuln_category, vuln_impact, severity, cve = [
|
|
67
|
-
elem.get_text().strip().replace(
|
|
67
|
+
elem.get_text().strip().replace("\xa0", " ") for elem in row.find_all("td")
|
|
68
68
|
]
|
|
69
69
|
vuln_data.append((vuln_impact, vuln_category, severity, cve))
|
|
70
70
|
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# Copyright (c) 2019 Red Hat, Inc.
|
|
2
|
+
# Author: Shubh Bapna
|
|
3
|
+
# License: LGPLv3+
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
from advisory_parser.exceptions import AdvisoryParserTextException
|
|
9
|
+
from advisory_parser.flaw import Flaw
|
|
10
|
+
from .utils import find_tag_by_text, CVE_REGEX, find_tag_by_id, get_request
|
|
11
|
+
from cvss import CVSS3
|
|
12
|
+
|
|
13
|
+
TO_RH_SEVERITY = {
|
|
14
|
+
"None": "low",
|
|
15
|
+
"Low": "low",
|
|
16
|
+
"Medium": "moderate",
|
|
17
|
+
"High": "important",
|
|
18
|
+
"Critical": "critical",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse_jenkins_advisory(url):
|
|
23
|
+
severity_to_cvss3_map = extract_severity_to_cvss3_map(url)
|
|
24
|
+
fixes = extract_fixes(url)
|
|
25
|
+
|
|
26
|
+
# extract date from the url
|
|
27
|
+
public_date_match = re.compile(r"\d{4}-\d{2}-\d{2}").search(url).group()
|
|
28
|
+
try:
|
|
29
|
+
public_date = datetime.strptime(public_date_match, "%Y-%m-%d")
|
|
30
|
+
except ValueError:
|
|
31
|
+
raise AdvisoryParserTextException(
|
|
32
|
+
"Could not parse public date ({}) from {}".format(public_date_match, url)
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# split it into chunks of advisories
|
|
36
|
+
advisories = extract_advisories(url)
|
|
37
|
+
if len(advisories) == 0:
|
|
38
|
+
raise AdvisoryParserTextException("No security fixes found in {}".format(url))
|
|
39
|
+
|
|
40
|
+
flaws, warnings = [], []
|
|
41
|
+
for index, advisory in enumerate(advisories):
|
|
42
|
+
severity = re.search(r"(SECURITY-\d+(\s\(\d\))?)", advisory).group(1)
|
|
43
|
+
if severity not in severity_to_cvss3_map.keys():
|
|
44
|
+
warnings.append("Could not find impact or cvss; skipping: {}".format(severity))
|
|
45
|
+
continue
|
|
46
|
+
|
|
47
|
+
cves = list(set(CVE_REGEX.findall(advisory)))
|
|
48
|
+
if len(cves) == 0:
|
|
49
|
+
warnings.append("Could not find CVEs or bugs; skipping: {}".format(severity))
|
|
50
|
+
continue
|
|
51
|
+
|
|
52
|
+
from_url = url + "#" + severity
|
|
53
|
+
summary = advisory.strip().split("\n")[0].strip()
|
|
54
|
+
impact = severity_to_cvss3_map[severity]["impact"]
|
|
55
|
+
cvss3 = severity_to_cvss3_map[severity]["score"]
|
|
56
|
+
description = extract_description(advisory, index == len(advisories) - 1)
|
|
57
|
+
affected_plugins = extract_affected_plugins(advisory)
|
|
58
|
+
affected_plugins_fix = extract_affected_plugins_fixes(
|
|
59
|
+
affected_plugins, fixes, summary, description, severity, warnings
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
flaws.append(
|
|
63
|
+
Flaw(
|
|
64
|
+
from_url=from_url,
|
|
65
|
+
cves=cves,
|
|
66
|
+
summary=f"jenkins-plugin: {', '.join(affected_plugins)}: {summary}",
|
|
67
|
+
public_date=public_date,
|
|
68
|
+
cvss3=cvss3,
|
|
69
|
+
impact=impact,
|
|
70
|
+
fixed_in=affected_plugins_fix,
|
|
71
|
+
description=description,
|
|
72
|
+
advisory_id=severity,
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
return flaws, warnings
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def extract_description(advisory, is_last):
|
|
80
|
+
description = "\n".join(advisory.strip().split("Description:")[-1].strip().split("\n")[:-1])
|
|
81
|
+
# if it is last advisory then we have to remove more stuff from description
|
|
82
|
+
if is_last:
|
|
83
|
+
description = description.split("Severity\n")[0].strip()
|
|
84
|
+
return description
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def extract_affected_plugins(advisory):
|
|
88
|
+
return [
|
|
89
|
+
a.strip()
|
|
90
|
+
for a in re.search(r"Affected plugins?:\n((.*\n)*?.*?)\nDescription", advisory)
|
|
91
|
+
.group(1)
|
|
92
|
+
.strip()
|
|
93
|
+
.split(",")
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def extract_affected_plugins_fixes(
|
|
98
|
+
affected_plugins, fixes, summary, description, severity, warnings
|
|
99
|
+
):
|
|
100
|
+
affected_plugins_fixes = {}
|
|
101
|
+
for fix in fixes:
|
|
102
|
+
if (
|
|
103
|
+
fix.lower() in affected_plugins
|
|
104
|
+
or summary.find(fix) != -1
|
|
105
|
+
or description.find(fix) != -1
|
|
106
|
+
):
|
|
107
|
+
affected_plugins_fixes[fix.lower()] = fixes[fix]
|
|
108
|
+
|
|
109
|
+
for affected_plugin in affected_plugins:
|
|
110
|
+
if affected_plugin not in affected_plugins_fixes:
|
|
111
|
+
warnings.append(
|
|
112
|
+
"{}: Could not find a fixed version for {} plugin".format(severity, affected_plugin)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
return affected_plugins_fixes
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def extract_severity_to_cvss3_map(url):
|
|
119
|
+
severity_list = find_tag_by_text(url, "h2", re.compile(r"\s*Severity\s*")).findNext("ul")
|
|
120
|
+
severity_to_cvss3_map = {}
|
|
121
|
+
|
|
122
|
+
# jenkins marks each severity with a high, medium or low impact and links that with a url to
|
|
123
|
+
# a cvss calculator. depending on the security advisory, each high, medium or low might be
|
|
124
|
+
# different from the next high, medium or low, so we will try to extract the cvss vector to get
|
|
125
|
+
# the most accurate cvss3 score
|
|
126
|
+
for c in severity_list.findChildren("li"):
|
|
127
|
+
severity, _ = list(c.stripped_strings)
|
|
128
|
+
|
|
129
|
+
# the href url contains the cvss vector after the # example:
|
|
130
|
+
# https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H
|
|
131
|
+
css3_vector = c.findNext().attrs["href"].split("#")[-1]
|
|
132
|
+
|
|
133
|
+
# severity contains a colon at the end so have to remove that
|
|
134
|
+
score = CVSS3(css3_vector)
|
|
135
|
+
severity_to_cvss3_map[severity[:-1]] = {
|
|
136
|
+
"score": score.rh_vector(),
|
|
137
|
+
"impact": TO_RH_SEVERITY[score.severities()[0]],
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return severity_to_cvss3_map
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def extract_fixes(url):
|
|
144
|
+
fixes_list = find_tag_by_text(url, "h2", re.compile(r"\s*Fix\s*")).findNext("ul")
|
|
145
|
+
plugins_to_fix = {}
|
|
146
|
+
# jenkins does include fixes in each advisories' description but that makes it harder to extract
|
|
147
|
+
# it also does have a list of fixes for each plugin for all the advisories at the end of the page
|
|
148
|
+
for fix in fixes_list.findChildren("li"):
|
|
149
|
+
plugin = list(fix.stripped_strings)[0].split("Plugin")[0].strip()
|
|
150
|
+
version = list(fix.stripped_strings)[1].split("version")[1].strip()
|
|
151
|
+
plugins_to_fix.setdefault(plugin, []).append(version)
|
|
152
|
+
return plugins_to_fix
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def extract_advisories(url):
|
|
156
|
+
tag_list = find_tag_by_id(url, "h3", re.compile(r"SECURITY-\d+"))
|
|
157
|
+
# append end tag
|
|
158
|
+
tag_list.append(find_tag_by_text(url, "h2", re.compile(r"\s*Severity\s*")))
|
|
159
|
+
|
|
160
|
+
start, end = 0, 1
|
|
161
|
+
all_texts = []
|
|
162
|
+
|
|
163
|
+
while end < len(tag_list):
|
|
164
|
+
curr_text = ""
|
|
165
|
+
curr_tag = tag_list[start]
|
|
166
|
+
end_tag = tag_list[end]
|
|
167
|
+
|
|
168
|
+
while curr_tag != end_tag:
|
|
169
|
+
curr_text += curr_tag.get_text()
|
|
170
|
+
curr_tag = curr_tag.next_sibling
|
|
171
|
+
|
|
172
|
+
# Filter out blank lines and leading/trailing spaces
|
|
173
|
+
all_texts.append("\n".join(line.strip() for line in curr_text.splitlines() if line))
|
|
174
|
+
start = end
|
|
175
|
+
end += 1
|
|
176
|
+
|
|
177
|
+
return all_texts
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# Author: Martin Prpič, Red Hat Product Security
|
|
3
3
|
# License: LGPLv3+
|
|
4
4
|
|
|
5
|
+
import calendar
|
|
5
6
|
import re
|
|
6
7
|
from datetime import datetime, timedelta
|
|
7
8
|
|
|
@@ -11,9 +12,24 @@ from advisory_parser.exceptions import AdvisoryParserTextException
|
|
|
11
12
|
from advisory_parser.flaw import Flaw
|
|
12
13
|
from .utils import get_request, get_text_from_url, CVE_REGEX
|
|
13
14
|
|
|
14
|
-
MARIADB_VULN_PAGE = "https://mariadb.com/kb/en/
|
|
15
|
+
MARIADB_VULN_PAGE = "https://mariadb.com/kb/en/security/"
|
|
15
16
|
VERSION_REGEX = re.compile(r"(\d\d?\.\d\.\d\d?)")
|
|
16
17
|
|
|
18
|
+
month_to_num = {
|
|
19
|
+
"jan": 1,
|
|
20
|
+
"feb": 2,
|
|
21
|
+
"mar": 3,
|
|
22
|
+
"apr": 4,
|
|
23
|
+
"may": 5,
|
|
24
|
+
"jun": 6,
|
|
25
|
+
"jul": 7,
|
|
26
|
+
"aug": 8,
|
|
27
|
+
"sep": 9,
|
|
28
|
+
"oct": 10,
|
|
29
|
+
"nov": 11,
|
|
30
|
+
"dec": 12,
|
|
31
|
+
}
|
|
32
|
+
|
|
17
33
|
|
|
18
34
|
def _nearest_tuesday(year, month, day=17):
|
|
19
35
|
"""For a given year and month, return nearest Tuesday to the 17th of that month
|
|
@@ -24,25 +40,8 @@ def _nearest_tuesday(year, month, day=17):
|
|
|
24
40
|
July and October."
|
|
25
41
|
[https://www.oracle.com/security-alerts/]
|
|
26
42
|
"""
|
|
27
|
-
month_to_num = {
|
|
28
|
-
"jan": 1,
|
|
29
|
-
"feb": 2,
|
|
30
|
-
"mar": 3,
|
|
31
|
-
"apr": 4,
|
|
32
|
-
"may": 5,
|
|
33
|
-
"jun": 6,
|
|
34
|
-
"jul": 7,
|
|
35
|
-
"aug": 8,
|
|
36
|
-
"sep": 9,
|
|
37
|
-
"oct": 10,
|
|
38
|
-
"nov": 11,
|
|
39
|
-
"dec": 12,
|
|
40
|
-
}
|
|
41
43
|
|
|
42
|
-
|
|
43
|
-
raise AdvisoryParserTextException("Invalid month parsed from advisory URL:", str(month))
|
|
44
|
-
|
|
45
|
-
base_date = datetime(year, month_to_num[month.lower()], day)
|
|
44
|
+
base_date = datetime(year, month, day)
|
|
46
45
|
|
|
47
46
|
previous_tuesday = base_date - timedelta(days=((base_date.weekday() + 6) % 7))
|
|
48
47
|
next_tuesday = base_date + timedelta(days=((1 - base_date.weekday()) % 7))
|
|
@@ -54,8 +53,35 @@ def _nearest_tuesday(year, month, day=17):
|
|
|
54
53
|
)
|
|
55
54
|
|
|
56
55
|
|
|
57
|
-
def
|
|
56
|
+
def _third_tuesday(year, month):
|
|
57
|
+
"""For a given year and month, return the 3rd Tuesday of that month
|
|
58
|
+
|
|
59
|
+
"Critical Patch Updates provide security patches for supported Oracle on-premises products.
|
|
60
|
+
They are available to customers with valid support contracts. Starting in April 2022,
|
|
61
|
+
Critical Patch Updates are released on the third Tuesday of January, April, July, and October
|
|
62
|
+
(They were previously published on the Tuesday closest to the 17th day of January, April, July,
|
|
63
|
+
and October). The next four dates are:
|
|
64
|
+
|
|
65
|
+
16 July 2024
|
|
66
|
+
15 October 2024
|
|
67
|
+
21 January 2025
|
|
68
|
+
15 April 2025
|
|
69
|
+
"
|
|
70
|
+
[https://www.oracle.com/security-alerts/]
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
c = calendar.Calendar()
|
|
74
|
+
monthcal = c.monthdatescalendar(year, month)
|
|
75
|
+
third_tuesday = [
|
|
76
|
+
day
|
|
77
|
+
for week in monthcal
|
|
78
|
+
for day in week
|
|
79
|
+
if day.weekday() == calendar.TUESDAY and day.month == month
|
|
80
|
+
][2]
|
|
81
|
+
return third_tuesday
|
|
58
82
|
|
|
83
|
+
|
|
84
|
+
def create_mariadb_cve_map():
|
|
59
85
|
# Pull plain text of the MariaDB page since the HTML is invalid: it
|
|
60
86
|
# doesn't define </li> ending tags for list elements. The HTML would
|
|
61
87
|
# have to be parsed with a more lenient parser (html5lib), which is an
|
|
@@ -99,7 +125,20 @@ def parse_mysql_advisory(url):
|
|
|
99
125
|
# Extract the CPU's month and year from the URL since the verbose page has
|
|
100
126
|
# no dates on it
|
|
101
127
|
month, year = url_match.groups()
|
|
102
|
-
|
|
128
|
+
if month.lower() not in month_to_num:
|
|
129
|
+
raise AdvisoryParserTextException("Invalid month parsed from advisory URL:", str(month))
|
|
130
|
+
month_num = month_to_num[month.lower()]
|
|
131
|
+
year_int = int(year)
|
|
132
|
+
# Starting in April 2022, Critical Patch Updates are released on the third Tuesday of January,
|
|
133
|
+
# April, July, and October (They were previously published on the Tuesday closest to the 17th
|
|
134
|
+
# day of January, April, July, and October).
|
|
135
|
+
# [https://www.oracle.com/security-alerts/]
|
|
136
|
+
if year_int < 2022:
|
|
137
|
+
cpu_date = _nearest_tuesday(year_int, month_num)
|
|
138
|
+
elif year_int == 2022 and month_num < 4:
|
|
139
|
+
cpu_date = _nearest_tuesday(year_int, month_num)
|
|
140
|
+
else:
|
|
141
|
+
cpu_date = _third_tuesday(year_int, month_num)
|
|
103
142
|
advisory_id = "CPU {} {}".format(month.capitalize(), year)
|
|
104
143
|
|
|
105
144
|
# Fetch the CPU verbose page
|
|
@@ -132,7 +171,15 @@ def parse_mysql_advisory(url):
|
|
|
132
171
|
description = "\n".join(description)
|
|
133
172
|
|
|
134
173
|
# Take the text part only, i.e. anything before the CVSS string
|
|
135
|
-
|
|
174
|
+
desc_cvss = re.split(r"\n\s*CVSS v?3\.[0-9] (?=Base Score)", description)
|
|
175
|
+
if len(desc_cvss) != 2:
|
|
176
|
+
warnings.append(
|
|
177
|
+
"ERROR: Could not identify CVSS score in {}; skipping:\n\n{}\n---".format(
|
|
178
|
+
cve, description
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
continue
|
|
182
|
+
description, cvss_text = desc_cvss
|
|
136
183
|
|
|
137
184
|
# Filter out some whitespace
|
|
138
185
|
description = description.replace("\n", " ").replace(" ", " ").strip()
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import re
|
|
6
6
|
from urllib.error import HTTPError, URLError
|
|
7
|
-
from urllib.request import urlopen
|
|
7
|
+
from urllib.request import urlopen, Request
|
|
8
8
|
|
|
9
9
|
from bs4 import BeautifulSoup
|
|
10
10
|
|
|
@@ -14,8 +14,13 @@ CVE_REGEX = re.compile(r"CVE-\d{4}-\d{4,}")
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def get_request(url):
|
|
17
|
+
user_agent = (
|
|
18
|
+
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7"
|
|
19
|
+
)
|
|
20
|
+
headers = {"User-Agent": user_agent}
|
|
21
|
+
request = Request(url, None, headers)
|
|
17
22
|
try:
|
|
18
|
-
res = urlopen(
|
|
23
|
+
res = urlopen(request)
|
|
19
24
|
except HTTPError as e:
|
|
20
25
|
error_msg = "Failed to GET with status code: {}".format(e.code)
|
|
21
26
|
raise AdvisoryParserGetContentException(error_msg)
|
|
@@ -42,3 +47,15 @@ def get_text_from_url(url):
|
|
|
42
47
|
text = "\n".join(line.strip() for line in text.splitlines() if line)
|
|
43
48
|
|
|
44
49
|
return text
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def find_tag_by_text(url, tag, text):
|
|
53
|
+
html = get_request(url)
|
|
54
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
55
|
+
return soup.find(tag, text=text)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def find_tag_by_id(url, tag, tag_id):
|
|
59
|
+
html = get_request(url)
|
|
60
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
61
|
+
return soup.findAll(tag, id=tag_id)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: advisory-parser
|
|
3
|
+
Version: 1.13
|
|
4
|
+
Summary: Security flaw parser for upstream security advisories
|
|
5
|
+
Home-page: https://github.com/RedHatProductSecurity/advisory-parser
|
|
6
|
+
Author: Martin Prpič, Red Hat Product Security
|
|
7
|
+
Author-email: secalert@redhat.com
|
|
8
|
+
License: LGPLv3+
|
|
9
|
+
Keywords: security advisory parser scraper
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Security
|
|
13
|
+
Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.0.0
|
|
24
|
+
Requires-Dist: cvss>=2.0
|
|
25
|
+
|
|
26
|
+
Advisory Parser
|
|
27
|
+
===============
|
|
28
|
+
|
|
29
|
+
This library allows you to parse data from security advisories of certain
|
|
30
|
+
projects to extract information about security issues. The parsed
|
|
31
|
+
information includes metadata such as impact, CVSS score, summary,
|
|
32
|
+
description, and others; for a full list, see the
|
|
33
|
+
``advisory_parser/flaw.py`` file.
|
|
34
|
+
|
|
35
|
+
**DISCLAIMER**: Much of the advisory parsing is fairly fragile. Because web
|
|
36
|
+
pages change all the time, it is not uncommon for parsers to break when a
|
|
37
|
+
page is changed in some way. Also, the advisory parsers only work with the
|
|
38
|
+
latest version of the advisory pages.
|
|
39
|
+
|
|
40
|
+
The need for parsing raw security advisories in this way could be avoided
|
|
41
|
+
if vendors provided their security pages in a machine readable (and
|
|
42
|
+
preferably standardized) format. An example of this would be Red Hat's
|
|
43
|
+
security advisories that can be pulled in from a separate Security Data API
|
|
44
|
+
(`RHSA-2016:1883.json <https://access.redhat.com/labs/securitydataapi/cvrf/RHSA-2016:1883.json>`_)
|
|
45
|
+
or downloaded as an XML file
|
|
46
|
+
(`cvrf-rhsa-2016-1883.xml <https://www.redhat.com/security/data/cvrf/2016/cvrf-rhsa-2016-1883.xml>`_),
|
|
47
|
+
or OpenSSL's list of issues available in XML
|
|
48
|
+
(`vulnerabilities.xml <https://www.openssl.org/news/vulnerabilities.xml>`_).
|
|
49
|
+
|
|
50
|
+
If you are a vendor or an upstream project owner interested in providing
|
|
51
|
+
your security advisories in a machine readable format and don't know where
|
|
52
|
+
to start, feel free to reach out to mprpic@redhat.com.
|
|
53
|
+
|
|
54
|
+
Currently available parsers include:
|
|
55
|
+
|
|
56
|
+
.. csv-table::
|
|
57
|
+
:header: "Project", "Example URL"
|
|
58
|
+
:widths: 20, 80
|
|
59
|
+
|
|
60
|
+
"Google Chrome", `<https://chromereleases.googleblog.com/2017/06/stable-channel-update-for-desktop_15.html>`_
|
|
61
|
+
"Adobe Flash", `<https://helpx.adobe.com/security/products/flash-player/apsb17-17.html>`_
|
|
62
|
+
"Jenkins", `<https://www.jenkins.io/security/advisory/2023-04-12>`_
|
|
63
|
+
"MySQL", `<http://www.oracle.com/technetwork/security-advisory/cpujul2017verbose-3236625.html>`_
|
|
64
|
+
"phpMyAdmin", ""
|
|
65
|
+
"Wireshark", ""
|
|
66
|
+
|
|
67
|
+
Installation
|
|
68
|
+
------------
|
|
69
|
+
|
|
70
|
+
::
|
|
71
|
+
|
|
72
|
+
pip install advisory-parser
|
|
73
|
+
|
|
74
|
+
Usage
|
|
75
|
+
-----
|
|
76
|
+
|
|
77
|
+
.. code-block:: python
|
|
78
|
+
|
|
79
|
+
from pprint import pprint
|
|
80
|
+
from advisory_parser import Parser
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
url = 'https://helpx.adobe.com/security/products/flash-player/apsb17-17.html'
|
|
84
|
+
flaws, warnings = Parser.parse_from_url(url)
|
|
85
|
+
|
|
86
|
+
for flaw in flaws:
|
|
87
|
+
print()
|
|
88
|
+
pprint(vars(flaw))
|
|
@@ -2,6 +2,7 @@ COPYRIGHT
|
|
|
2
2
|
LICENSE
|
|
3
3
|
MANIFEST.in
|
|
4
4
|
README.rst
|
|
5
|
+
pyproject.toml
|
|
5
6
|
setup.cfg
|
|
6
7
|
setup.py
|
|
7
8
|
tox.ini
|
|
@@ -17,14 +18,17 @@ advisory_parser.egg-info/top_level.txt
|
|
|
17
18
|
advisory_parser/parsers/__init__.py
|
|
18
19
|
advisory_parser/parsers/chrome.py
|
|
19
20
|
advisory_parser/parsers/flash.py
|
|
21
|
+
advisory_parser/parsers/jenkins.py
|
|
20
22
|
advisory_parser/parsers/mysql.py
|
|
21
23
|
advisory_parser/parsers/utils.py
|
|
22
24
|
tests/test_chrome_parser.py
|
|
23
25
|
tests/test_flash_parser.py
|
|
26
|
+
tests/test_jenkins_parser.py
|
|
24
27
|
tests/test_mysql_parser.py
|
|
25
28
|
tests/test_data/chrome_2017-06-15.txt
|
|
26
29
|
tests/test_data/chrome_2020-02-04.txt
|
|
27
30
|
tests/test_data/flash_apsb17-21.html
|
|
28
31
|
tests/test_data/flash_apsb17-23.html
|
|
29
32
|
tests/test_data/flash_apsb17-33.html
|
|
33
|
+
tests/test_data/jenkins_2023-04-12.html
|
|
30
34
|
tests/test_data/mysql_cpu-apr-2019.html
|
|
@@ -8,16 +8,16 @@ from setuptools import setup
|
|
|
8
8
|
with open("README.rst", "r") as f:
|
|
9
9
|
description = f.read()
|
|
10
10
|
|
|
11
|
-
requires = ["beautifulsoup4>=4.0.0"]
|
|
11
|
+
requires = ["beautifulsoup4>=4.0.0", "cvss>=2.0"]
|
|
12
12
|
|
|
13
13
|
setup(
|
|
14
14
|
name="advisory-parser",
|
|
15
|
-
version="1.
|
|
15
|
+
version="1.13",
|
|
16
16
|
description="Security flaw parser for upstream security advisories",
|
|
17
17
|
long_description=description,
|
|
18
|
-
url="https://github.com/
|
|
18
|
+
url="https://github.com/RedHatProductSecurity/advisory-parser",
|
|
19
19
|
author="Martin Prpič, Red Hat Product Security",
|
|
20
|
-
author_email="
|
|
20
|
+
author_email="secalert@redhat.com",
|
|
21
21
|
license="LGPLv3+",
|
|
22
22
|
classifiers=[
|
|
23
23
|
"Development Status :: 5 - Production/Stable",
|
|
@@ -29,6 +29,9 @@ setup(
|
|
|
29
29
|
"Programming Language :: Python :: 3.7",
|
|
30
30
|
"Programming Language :: Python :: 3.8",
|
|
31
31
|
"Programming Language :: Python :: 3.9",
|
|
32
|
+
"Programming Language :: Python :: 3.10",
|
|
33
|
+
"Programming Language :: Python :: 3.11",
|
|
34
|
+
"Programming Language :: Python :: 3.12",
|
|
32
35
|
],
|
|
33
36
|
keywords="security advisory parser scraper",
|
|
34
37
|
packages=["advisory_parser", "advisory_parser.parsers"],
|