civics-cdf-validator 1.67.dev0__py3-none-any.whl → 1.67.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- civics_cdf_validator/rules.py +158 -2084
- civics_cdf_validator/rulesets/__init__.py +0 -0
- civics_cdf_validator/rulesets/common_rules.py +1873 -0
- civics_cdf_validator/rulesets/rule_util.py +146 -0
- civics_cdf_validator/validator.py +4 -3
- civics_cdf_validator/version.py +1 -1
- {civics_cdf_validator-1.67.dev0.dist-info → civics_cdf_validator-1.67.dev2.dist-info}/METADATA +1 -1
- civics_cdf_validator-1.67.dev2.dist-info/RECORD +18 -0
- civics_cdf_validator-1.67.dev0.dist-info/RECORD +0 -15
- {civics_cdf_validator-1.67.dev0.dist-info → civics_cdf_validator-1.67.dev2.dist-info}/WHEEL +0 -0
- {civics_cdf_validator-1.67.dev0.dist-info → civics_cdf_validator-1.67.dev2.dist-info}/entry_points.txt +0 -0
- {civics_cdf_validator-1.67.dev0.dist-info → civics_cdf_validator-1.67.dev2.dist-info}/licenses/LICENSE-2.0.txt +0 -0
- {civics_cdf_validator-1.67.dev0.dist-info → civics_cdf_validator-1.67.dev2.dist-info}/top_level.txt +0 -0
civics_cdf_validator/rules.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
"""Validation rules for the
|
|
14
|
+
"""Validation rules for the CDF XML validator."""
|
|
15
15
|
|
|
16
16
|
from __future__ import print_function
|
|
17
17
|
|
|
@@ -25,27 +25,13 @@ from botocore import utils as botocore_utils
|
|
|
25
25
|
from civics_cdf_validator import base
|
|
26
26
|
from civics_cdf_validator import gpunit_rules
|
|
27
27
|
from civics_cdf_validator import loggers
|
|
28
|
-
from civics_cdf_validator import
|
|
28
|
+
from civics_cdf_validator.rulesets import common_rules
|
|
29
|
+
from civics_cdf_validator.rulesets import rule_util
|
|
29
30
|
import dateutil
|
|
30
31
|
from frozendict import frozendict
|
|
31
|
-
import language_tags
|
|
32
|
-
from lxml import etree
|
|
33
32
|
import networkx
|
|
34
|
-
import pycountry
|
|
35
|
-
from six.moves.urllib.parse import urlparse
|
|
36
33
|
|
|
37
34
|
|
|
38
|
-
_XML_TRUE_VALUES = frozenset(["true", "1"])
|
|
39
|
-
_PARTY_LEADERSHIP_TYPES = ["party-leader-id", "party-chair-id"]
|
|
40
|
-
_INDEPENDENT_PARTY_NAMES = frozenset(["independent", "nonpartisan"])
|
|
41
|
-
_IDREF_TYPES = frozenset(["xs:IDREF", "xs:IDREFS"])
|
|
42
|
-
# The set of external identifiers that contain references to other entities.
|
|
43
|
-
_IDREF_EXTERNAL_IDENTIFIERS = frozenset(
|
|
44
|
-
["jurisdiction-id"] + _PARTY_LEADERSHIP_TYPES
|
|
45
|
-
)
|
|
46
|
-
_IDENTIFIER_TYPES = frozenset(
|
|
47
|
-
["local-level", "national-level", "ocd-id", "state-level"]
|
|
48
|
-
)
|
|
49
35
|
_CONTEST_STAGE_TYPES = frozenset([
|
|
50
36
|
"exit-polls",
|
|
51
37
|
"estimates",
|
|
@@ -78,26 +64,7 @@ _INTERNATIONALIZED_TEXT_ELEMENTS_WITH_ONLY_ONE_TEXT_PER_LANGUAGE = [
|
|
|
78
64
|
"Title",
|
|
79
65
|
# go/keep-sorted end
|
|
80
66
|
]
|
|
81
|
-
_GPUNIT_TYPES_WITHOUT_OCD_IDS = frozenset([
|
|
82
|
-
"ballot-batch",
|
|
83
|
-
"ballot-style-area",
|
|
84
|
-
"combined-precinct",
|
|
85
|
-
"drop-box",
|
|
86
|
-
"polling-place",
|
|
87
|
-
"split-precinct",
|
|
88
|
-
"vote-center",
|
|
89
|
-
])
|
|
90
67
|
|
|
91
|
-
_EXECUTIVE_OFFICE_ROLES = frozenset([
|
|
92
|
-
"head of state",
|
|
93
|
-
"head of government",
|
|
94
|
-
"president",
|
|
95
|
-
"vice president",
|
|
96
|
-
"state executive",
|
|
97
|
-
"deputy state executive",
|
|
98
|
-
"deputy head of government",
|
|
99
|
-
"mayor",
|
|
100
|
-
])
|
|
101
68
|
|
|
102
69
|
_VALID_FEED_LONGEVITY_BY_FEED_TYPE = frozendict({
|
|
103
70
|
"committee": ["evergreen"],
|
|
@@ -107,35 +74,6 @@ _VALID_FEED_LONGEVITY_BY_FEED_TYPE = frozendict({
|
|
|
107
74
|
"pre-election": ["limited", "yearly"],
|
|
108
75
|
})
|
|
109
76
|
|
|
110
|
-
_VALID_OFFICE_ROLE_COMBINATIONS = frozenset([
|
|
111
|
-
frozenset(["head of government", "head of state"]),
|
|
112
|
-
frozenset(["cabinet member", "general purpose officer"]),
|
|
113
|
-
])
|
|
114
|
-
|
|
115
|
-
_URI_USAGE_TYPES = frozenset([
|
|
116
|
-
"campaign",
|
|
117
|
-
"official",
|
|
118
|
-
"personal",
|
|
119
|
-
])
|
|
120
|
-
_TYPED_URI_PLATFORMS = frozenset([
|
|
121
|
-
"facebook",
|
|
122
|
-
"instagram",
|
|
123
|
-
"line",
|
|
124
|
-
"linkedin",
|
|
125
|
-
"tiktok",
|
|
126
|
-
"twitter",
|
|
127
|
-
"website",
|
|
128
|
-
"whatsapp",
|
|
129
|
-
"youtube",
|
|
130
|
-
])
|
|
131
|
-
_UNTYPED_URI_PLATFORMS = frozenset([
|
|
132
|
-
"ballotpedia",
|
|
133
|
-
"fec",
|
|
134
|
-
"followthemoney",
|
|
135
|
-
"opensecrets",
|
|
136
|
-
"wikipedia",
|
|
137
|
-
])
|
|
138
|
-
_ALL_URI_PLATFORMS = _TYPED_URI_PLATFORMS | _UNTYPED_URI_PLATFORMS
|
|
139
77
|
|
|
140
78
|
_WINNER_POST_ELECTION_STATUSES = frozenset([
|
|
141
79
|
"winner",
|
|
@@ -143,248 +81,23 @@ _WINNER_POST_ELECTION_STATUSES = frozenset([
|
|
|
143
81
|
])
|
|
144
82
|
|
|
145
83
|
|
|
146
|
-
def _is_xml_true(element):
|
|
147
|
-
return (
|
|
148
|
-
element_has_text(element)
|
|
149
|
-
and element.text.strip().lower() in _XML_TRUE_VALUES
|
|
150
|
-
)
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
def _get_office_roles(element, is_post_office_split_feed=False):
|
|
154
|
-
if is_post_office_split_feed:
|
|
155
|
-
return [element.text for element in element.findall("Role")]
|
|
156
|
-
return get_entity_info_for_value_type(element, "office-role")
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
def _is_executive_office(element, is_post_office_split_feed=False):
|
|
160
|
-
office_roles = _get_office_roles(element, is_post_office_split_feed)
|
|
161
|
-
return not _EXECUTIVE_OFFICE_ROLES.isdisjoint(office_roles)
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
def _has_government_body(element):
|
|
165
|
-
return element_has_text(element.find("GovernmentBodyIds"))
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
def get_external_id_values(
|
|
169
|
-
element, value_type, return_elements=False, search_subelements=True
|
|
170
|
-
):
|
|
171
|
-
"""Helper to gather all Values of external ids for a given type.
|
|
172
|
-
|
|
173
|
-
If search_subelements is True, will search all subelements, if False, will
|
|
174
|
-
only search child elements.
|
|
175
|
-
|
|
176
|
-
Args:
|
|
177
|
-
element: element to be processed
|
|
178
|
-
value_type: external id value type
|
|
179
|
-
return_elements: Boolean if True return elements, if False return values.
|
|
180
|
-
search_subelements: Boolean to instruct the function to recursively search
|
|
181
|
-
subelements.
|
|
182
|
-
|
|
183
|
-
Returns:
|
|
184
|
-
A list of external id values or external identifier elements
|
|
185
|
-
"""
|
|
186
|
-
external_ids = []
|
|
187
|
-
if search_subelements:
|
|
188
|
-
external_ids = element.findall(".//ExternalIdentifier")
|
|
189
|
-
else:
|
|
190
|
-
external_ids = element.findall("ExternalIdentifiers/ExternalIdentifier")
|
|
191
|
-
values = []
|
|
192
|
-
for extern_id in external_ids:
|
|
193
|
-
id_type = extern_id.find("Type")
|
|
194
|
-
if id_type is None or not id_type.text:
|
|
195
|
-
continue
|
|
196
|
-
matches_type = False
|
|
197
|
-
id_text = id_type.text.strip()
|
|
198
|
-
if id_text in _IDENTIFIER_TYPES and id_text == value_type:
|
|
199
|
-
matches_type = True
|
|
200
|
-
elif id_text == "other":
|
|
201
|
-
other_type = extern_id.find("OtherType")
|
|
202
|
-
if (
|
|
203
|
-
other_type is not None
|
|
204
|
-
and other_type.text
|
|
205
|
-
and other_type.text.strip() == value_type
|
|
206
|
-
and value_type not in _IDENTIFIER_TYPES
|
|
207
|
-
):
|
|
208
|
-
matches_type = True
|
|
209
|
-
if matches_type:
|
|
210
|
-
value = extern_id.find("Value")
|
|
211
|
-
# Could include empty text; check in calling function.
|
|
212
|
-
# Not checked here because errors should be raised in some cases.
|
|
213
|
-
if value is not None and value.text:
|
|
214
|
-
if return_elements:
|
|
215
|
-
values.append(value)
|
|
216
|
-
else:
|
|
217
|
-
values.append(value.text)
|
|
218
|
-
return values
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
def get_additional_type_values(element, value_type, return_elements=False):
|
|
222
|
-
"""Helper to gather all nested additional type values for a given type."""
|
|
223
|
-
elements = element.findall(".//AdditionalData[@type='{}']".format(value_type))
|
|
224
|
-
if not return_elements:
|
|
225
|
-
return [
|
|
226
|
-
val.text
|
|
227
|
-
for val in elements
|
|
228
|
-
if val is not None and val.text and val.text.strip()
|
|
229
|
-
]
|
|
230
|
-
return elements
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
def extract_person_fullname(person):
|
|
234
|
-
"""Extracts the person's fullname or builds it if needed."""
|
|
235
|
-
full_name_elt = person.find("FullName")
|
|
236
|
-
if full_name_elt is None:
|
|
237
|
-
return []
|
|
238
|
-
full_name_list = set()
|
|
239
|
-
for name in full_name_elt.findall("Text"):
|
|
240
|
-
if name.text:
|
|
241
|
-
full_name_list.add(name.text)
|
|
242
|
-
return full_name_list
|
|
243
|
-
return []
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
def get_entity_info_for_value_type(element, info_type, return_elements=False):
|
|
247
|
-
info_collection = get_additional_type_values(
|
|
248
|
-
element, info_type, return_elements
|
|
249
|
-
)
|
|
250
|
-
info_collection.extend(
|
|
251
|
-
list(get_external_id_values(element, info_type, return_elements))
|
|
252
|
-
)
|
|
253
|
-
return info_collection
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
def get_language_to_text_map(element):
|
|
257
|
-
"""Return a map of languages to text in an InternationalizedText element."""
|
|
258
|
-
language_map = {}
|
|
259
|
-
if element is None:
|
|
260
|
-
return language_map
|
|
261
|
-
intl_strings = element.findall("Text")
|
|
262
|
-
|
|
263
|
-
for intl_string in intl_strings:
|
|
264
|
-
text = intl_string.text
|
|
265
|
-
if text is None or not text:
|
|
266
|
-
continue
|
|
267
|
-
language = intl_string.get("language")
|
|
268
|
-
if language is None or not language:
|
|
269
|
-
continue
|
|
270
|
-
if language not in language_map:
|
|
271
|
-
language_map[language] = [text]
|
|
272
|
-
else:
|
|
273
|
-
language_map[language].append(text)
|
|
274
|
-
return language_map
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
def element_has_text(element):
|
|
278
|
-
return (
|
|
279
|
-
element is not None
|
|
280
|
-
and element.text is not None
|
|
281
|
-
and not element.text.isspace()
|
|
282
|
-
)
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
def country_code_is_valid(country_code):
|
|
286
|
-
# EU is part of ISO 3166/MA
|
|
287
|
-
return (
|
|
288
|
-
country_code.lower() == "eu"
|
|
289
|
-
or pycountry.countries.get(alpha_2=country_code.upper()) is not None
|
|
290
|
-
)
|
|
291
|
-
|
|
292
|
-
|
|
293
84
|
def _get_type_or_other_type(element):
|
|
85
|
+
"""Returns the OtherType text if Type is "other", else the Type text."""
|
|
294
86
|
type_element = element.find("Type")
|
|
295
87
|
other_type_element = element.find("OtherType")
|
|
296
88
|
type_text = (
|
|
297
|
-
type_element.text.strip()
|
|
89
|
+
type_element.text.strip()
|
|
90
|
+
if rule_util.element_has_text(type_element)
|
|
91
|
+
else ""
|
|
298
92
|
)
|
|
299
93
|
other_type_text = (
|
|
300
94
|
other_type_element.text.strip()
|
|
301
|
-
if element_has_text(other_type_element)
|
|
95
|
+
if rule_util.element_has_text(other_type_element)
|
|
302
96
|
else ""
|
|
303
97
|
)
|
|
304
98
|
return other_type_text if type_text == "other" else type_text
|
|
305
99
|
|
|
306
100
|
|
|
307
|
-
class Schema(base.TreeRule):
|
|
308
|
-
"""Checks if election file validates against the provided schema."""
|
|
309
|
-
|
|
310
|
-
def check(self):
|
|
311
|
-
try:
|
|
312
|
-
schema = etree.XMLSchema(etree=self.schema_tree)
|
|
313
|
-
except etree.XMLSchemaParseError as e:
|
|
314
|
-
raise loggers.ElectionError.from_message(
|
|
315
|
-
"The schema file could not be parsed correctly %s" % str(e)
|
|
316
|
-
)
|
|
317
|
-
valid_xml = True
|
|
318
|
-
try:
|
|
319
|
-
schema.assertValid(self.election_tree)
|
|
320
|
-
except etree.DocumentInvalid as e:
|
|
321
|
-
valid_xml = False
|
|
322
|
-
if not valid_xml:
|
|
323
|
-
errors = []
|
|
324
|
-
for error in schema.error_log:
|
|
325
|
-
errors.append(
|
|
326
|
-
loggers.LogEntry(
|
|
327
|
-
lines=[error.line],
|
|
328
|
-
message=(
|
|
329
|
-
"The election file didn't validate "
|
|
330
|
-
"against schema : {0}".format(error.message.encode("utf-8"))
|
|
331
|
-
),
|
|
332
|
-
)
|
|
333
|
-
)
|
|
334
|
-
raise loggers.ElectionError(errors)
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
class OptionalAndEmpty(base.BaseRule):
|
|
338
|
-
"""Checks for optional and empty fields."""
|
|
339
|
-
|
|
340
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
341
|
-
super(OptionalAndEmpty, self).__init__(election_tree, schema_tree, **kwargs)
|
|
342
|
-
self.previous = None
|
|
343
|
-
|
|
344
|
-
def elements(self):
|
|
345
|
-
eligible_elements = []
|
|
346
|
-
for _, element in etree.iterwalk(self.schema_tree):
|
|
347
|
-
tag = self.strip_schema_ns(element)
|
|
348
|
-
if tag and tag == "element" and element.get("minOccurs") == "0":
|
|
349
|
-
eligible_elements.append(element.get("name"))
|
|
350
|
-
return eligible_elements
|
|
351
|
-
|
|
352
|
-
# pylint: disable=g-explicit-length-test
|
|
353
|
-
def check(self, element):
|
|
354
|
-
if element == self.previous:
|
|
355
|
-
return
|
|
356
|
-
self.previous = element
|
|
357
|
-
if (element.text is None or not element.text.strip()) and not len(element):
|
|
358
|
-
raise loggers.ElectionWarning.from_message(
|
|
359
|
-
"This optional element included although it is empty.", [element]
|
|
360
|
-
)
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
class Encoding(base.TreeRule):
|
|
364
|
-
"""Checks that the file provided uses UTF-8 encoding."""
|
|
365
|
-
|
|
366
|
-
def check(self):
|
|
367
|
-
docinfo = self.election_tree.docinfo
|
|
368
|
-
if docinfo.encoding != "UTF-8":
|
|
369
|
-
raise loggers.ElectionError.from_message("Encoding on file is not UTF-8")
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
class LanguageCode(base.BaseRule):
|
|
373
|
-
"""Check that Text and Uri elements have a valid language code."""
|
|
374
|
-
|
|
375
|
-
def elements(self):
|
|
376
|
-
return ["Text", "Uri"]
|
|
377
|
-
|
|
378
|
-
def check(self, element):
|
|
379
|
-
if "language" not in element.attrib:
|
|
380
|
-
return
|
|
381
|
-
elem_lang = element.get("language")
|
|
382
|
-
if not elem_lang.strip() or not language_tags.tags.check(elem_lang):
|
|
383
|
-
raise loggers.ElectionError.from_message(
|
|
384
|
-
"%s is not a valid language code" % elem_lang, [element]
|
|
385
|
-
)
|
|
386
|
-
|
|
387
|
-
|
|
388
101
|
class PercentSum(base.BaseRule):
|
|
389
102
|
"""Check that Contest elements have percents summing to 0 or 100."""
|
|
390
103
|
|
|
@@ -413,274 +126,6 @@ class PercentSum(base.BaseRule):
|
|
|
413
126
|
)
|
|
414
127
|
|
|
415
128
|
|
|
416
|
-
class EmptyText(base.BaseRule):
|
|
417
|
-
"""Check that Text elements are not empty or strictly whitespace."""
|
|
418
|
-
|
|
419
|
-
def elements(self):
|
|
420
|
-
return ["Text"]
|
|
421
|
-
|
|
422
|
-
def check(self, element):
|
|
423
|
-
if element.text is None or not element.text.strip():
|
|
424
|
-
raise loggers.ElectionError.from_message("Text is empty", element)
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
class EmptyString(base.BaseRule):
|
|
428
|
-
"""Check that string fields are not empty or strictly whitespace."""
|
|
429
|
-
|
|
430
|
-
def elements(self):
|
|
431
|
-
return list(self.get_element_names_for_type("xs:string"))
|
|
432
|
-
|
|
433
|
-
# pylint: disable=g-explicit-length-test
|
|
434
|
-
def check(self, element):
|
|
435
|
-
if (element.text is None or not element.text.strip()) and not len(element):
|
|
436
|
-
raise loggers.ElectionError.from_message(
|
|
437
|
-
"String field is empty", [element]
|
|
438
|
-
)
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
class DuplicateID(base.TreeRule):
|
|
442
|
-
"""Check that the file does not contain duplicate object IDs."""
|
|
443
|
-
|
|
444
|
-
def check(self):
|
|
445
|
-
all_object_ids = set()
|
|
446
|
-
error_log = []
|
|
447
|
-
for _, element in etree.iterwalk(self.election_tree, events=("end",)):
|
|
448
|
-
if "objectId" not in element.attrib:
|
|
449
|
-
continue
|
|
450
|
-
else:
|
|
451
|
-
obj_id = element.get("objectId")
|
|
452
|
-
if not obj_id:
|
|
453
|
-
continue
|
|
454
|
-
if obj_id in all_object_ids:
|
|
455
|
-
error_log.append(loggers.LogEntry("duplicate object ID", element))
|
|
456
|
-
else:
|
|
457
|
-
all_object_ids.add(obj_id)
|
|
458
|
-
if error_log:
|
|
459
|
-
raise loggers.ElectionError(error_log)
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
class ValidIDREF(base.BaseRule):
|
|
463
|
-
"""Check that IDREFs are valid.
|
|
464
|
-
|
|
465
|
-
Every field of type IDREF should actually reference a value that exists in a
|
|
466
|
-
field of type ID. Additionaly the referenced value should be an objectId
|
|
467
|
-
of the proper reference type for the given field.
|
|
468
|
-
"""
|
|
469
|
-
|
|
470
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
471
|
-
super(ValidIDREF, self).__init__(election_tree, schema_tree, **kwargs)
|
|
472
|
-
self.object_id_mapping = {}
|
|
473
|
-
self.element_reference_mapping = {}
|
|
474
|
-
|
|
475
|
-
_REFERENCE_TYPE_OVERRIDES = {
|
|
476
|
-
# ID refs to GpUnits
|
|
477
|
-
"DiplomatReceivingStateId": "GpUnit",
|
|
478
|
-
"DiplomatSendingStateId": "GpUnit",
|
|
479
|
-
"ElectoralDistrictId": "GpUnit",
|
|
480
|
-
"ElectionScopeId": "GpUnit",
|
|
481
|
-
"ScopeLevel": "GpUnit",
|
|
482
|
-
"JurisdictionId": "GpUnit",
|
|
483
|
-
# ID refs to Persons
|
|
484
|
-
"AuthorityId": "Person",
|
|
485
|
-
"AuthorityIds": "Person",
|
|
486
|
-
"PartyLeaderId": "Person",
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
def setup(self):
|
|
490
|
-
object_id_map = self._gather_object_ids_by_type()
|
|
491
|
-
self.object_id_mapping = object_id_map
|
|
492
|
-
|
|
493
|
-
element_reference_map = self._gather_reference_mapping()
|
|
494
|
-
self.element_reference_mapping = element_reference_map
|
|
495
|
-
|
|
496
|
-
def _gather_object_ids_by_type(self):
|
|
497
|
-
"""Create a mapping of element types to set of objectIds of same type."""
|
|
498
|
-
|
|
499
|
-
type_obj_id_mapping = dict()
|
|
500
|
-
for _, element in etree.iterwalk(self.election_tree, events=("end",)):
|
|
501
|
-
if "objectId" in element.attrib:
|
|
502
|
-
obj_type = element.tag
|
|
503
|
-
obj_id = element.get("objectId")
|
|
504
|
-
if obj_id:
|
|
505
|
-
type_obj_id_mapping.setdefault(obj_type, set([])).add(obj_id)
|
|
506
|
-
return type_obj_id_mapping
|
|
507
|
-
|
|
508
|
-
def _gather_reference_mapping(self):
|
|
509
|
-
"""Create a mapping of each IDREF(S) element to their reference type."""
|
|
510
|
-
|
|
511
|
-
reference_mapping = dict()
|
|
512
|
-
for _, element in etree.iterwalk(self.schema_tree):
|
|
513
|
-
tag = self.strip_schema_ns(element)
|
|
514
|
-
if (
|
|
515
|
-
tag
|
|
516
|
-
and tag == "element"
|
|
517
|
-
and element.get("type") in ("xs:IDREF", "xs:IDREFS")
|
|
518
|
-
):
|
|
519
|
-
elem_name = element.get("name")
|
|
520
|
-
reference_type = self._determine_reference_type(elem_name)
|
|
521
|
-
reference_mapping[elem_name] = reference_type
|
|
522
|
-
return reference_mapping
|
|
523
|
-
|
|
524
|
-
def _determine_reference_type(self, name):
|
|
525
|
-
"""Determines the XML type being referenced by an IDREF(S) element."""
|
|
526
|
-
|
|
527
|
-
for elem_type in self.object_id_mapping.keys():
|
|
528
|
-
type_id = elem_type + "Id"
|
|
529
|
-
if name.endswith(type_id) or name.endswith(type_id + "s"):
|
|
530
|
-
return elem_type
|
|
531
|
-
if name in self._REFERENCE_TYPE_OVERRIDES:
|
|
532
|
-
return self._REFERENCE_TYPE_OVERRIDES[name]
|
|
533
|
-
return None
|
|
534
|
-
|
|
535
|
-
def elements(self):
|
|
536
|
-
return list(self.element_reference_mapping.keys())
|
|
537
|
-
|
|
538
|
-
def check(self, element):
|
|
539
|
-
error_log = []
|
|
540
|
-
|
|
541
|
-
element_name = element.tag
|
|
542
|
-
element_reference_type = self.element_reference_mapping[element_name]
|
|
543
|
-
reference_object_ids = self.object_id_mapping.get(
|
|
544
|
-
element_reference_type, []
|
|
545
|
-
)
|
|
546
|
-
if element.text:
|
|
547
|
-
id_references = element.text.split()
|
|
548
|
-
for id_ref in id_references:
|
|
549
|
-
if id_ref not in reference_object_ids:
|
|
550
|
-
error_log.append(
|
|
551
|
-
loggers.LogEntry(
|
|
552
|
-
(
|
|
553
|
-
"{} is not a valid IDREF. {} should contain an "
|
|
554
|
-
"objectId from a {} element."
|
|
555
|
-
).format(id_ref, element_name, element_reference_type),
|
|
556
|
-
element,
|
|
557
|
-
)
|
|
558
|
-
)
|
|
559
|
-
if error_log:
|
|
560
|
-
raise loggers.ElectionError(error_log)
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
class ValidStableID(base.BaseRule):
|
|
564
|
-
"""Ensure stable-ids are in the correct format."""
|
|
565
|
-
|
|
566
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
567
|
-
super(ValidStableID, self).__init__(election_tree, schema_tree, **kwargs)
|
|
568
|
-
regex = r"^[a-zA-Z0-9_-]+$"
|
|
569
|
-
self.stable_id_matcher = re.compile(regex, flags=re.U)
|
|
570
|
-
|
|
571
|
-
def elements(self):
|
|
572
|
-
return ["ExternalIdentifiers"]
|
|
573
|
-
|
|
574
|
-
def check(self, element):
|
|
575
|
-
stable_ids = get_external_id_values(element, "stable")
|
|
576
|
-
error_log = []
|
|
577
|
-
for s_id in stable_ids:
|
|
578
|
-
if not self.stable_id_matcher.match(s_id):
|
|
579
|
-
error_log.append(
|
|
580
|
-
loggers.LogEntry(
|
|
581
|
-
"Stable id '{}' is not in the correct format.".format(s_id),
|
|
582
|
-
[element],
|
|
583
|
-
)
|
|
584
|
-
)
|
|
585
|
-
if error_log:
|
|
586
|
-
raise loggers.ElectionError(error_log)
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
class GpUnitOcdId(base.BaseRule):
|
|
590
|
-
"""All GpUnits MUST have a valid OCD ID."""
|
|
591
|
-
|
|
592
|
-
def elements(self):
|
|
593
|
-
return ["GpUnit", "ReportingUnit"]
|
|
594
|
-
|
|
595
|
-
def check(self, element):
|
|
596
|
-
# ReportingDevices are not expected to have OCD IDs.
|
|
597
|
-
if self.get_element_class(element) == "ReportingDevice":
|
|
598
|
-
return
|
|
599
|
-
type_element = element.find("Type")
|
|
600
|
-
if (
|
|
601
|
-
element_has_text(type_element)
|
|
602
|
-
and type_element.text in _GPUNIT_TYPES_WITHOUT_OCD_IDS
|
|
603
|
-
):
|
|
604
|
-
return
|
|
605
|
-
ocd_ids = get_external_id_values(element, "ocd-id")
|
|
606
|
-
if not ocd_ids:
|
|
607
|
-
raise loggers.ElectionError.from_message(
|
|
608
|
-
f"The GpUnit {element.get('objectId')} does not have an ocd-id.",
|
|
609
|
-
[element],
|
|
610
|
-
[element.sourceline],
|
|
611
|
-
)
|
|
612
|
-
for ocd_id in ocd_ids:
|
|
613
|
-
if not self.ocd_id_validator.is_valid_ocd_id(ocd_id):
|
|
614
|
-
raise loggers.ElectionError.from_message(
|
|
615
|
-
f"The GpUnit {element.get('objectId')} does not have a valid "
|
|
616
|
-
f"ocd-id: '{ocd_id}'.",
|
|
617
|
-
[element],
|
|
618
|
-
[element.sourceline],
|
|
619
|
-
)
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
class DuplicatedGpUnitOcdId(base.BaseRule):
|
|
623
|
-
"""2 GpUnits should not have same OCD ID."""
|
|
624
|
-
|
|
625
|
-
def elements(self):
|
|
626
|
-
return ["GpUnitCollection"]
|
|
627
|
-
|
|
628
|
-
def check(self, element):
|
|
629
|
-
error_log = []
|
|
630
|
-
gp_ocdid = dict()
|
|
631
|
-
gpunits = element.findall("GpUnit")
|
|
632
|
-
for gpunit in gpunits:
|
|
633
|
-
ocd_ids = get_external_id_values(gpunit, "ocd-id")
|
|
634
|
-
for ocd_id in ocd_ids:
|
|
635
|
-
if ocd_id not in gp_ocdid.keys():
|
|
636
|
-
gp_ocdid[ocd_id] = gpunit.get("objectId")
|
|
637
|
-
else:
|
|
638
|
-
msg = "GpUnits %s and %s have the same ocd-id %s" % (
|
|
639
|
-
gp_ocdid[ocd_id],
|
|
640
|
-
gpunit.get("objectId"),
|
|
641
|
-
ocd_id,
|
|
642
|
-
)
|
|
643
|
-
error_log.append(loggers.LogEntry(msg, [gpunit]))
|
|
644
|
-
if error_log:
|
|
645
|
-
raise loggers.ElectionError(error_log)
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
class DuplicateGpUnits(base.BaseRule):
|
|
649
|
-
"""Detect GpUnits which are effectively duplicates of each other."""
|
|
650
|
-
|
|
651
|
-
def elements(self):
|
|
652
|
-
return ["GpUnitCollection"]
|
|
653
|
-
|
|
654
|
-
def check(self, element):
|
|
655
|
-
children = {}
|
|
656
|
-
object_ids = set()
|
|
657
|
-
error_log = []
|
|
658
|
-
for gpunit in element.findall("GpUnit"):
|
|
659
|
-
object_id = gpunit.get("objectId")
|
|
660
|
-
if not object_id:
|
|
661
|
-
continue
|
|
662
|
-
elif object_id in object_ids:
|
|
663
|
-
error_log.append(loggers.LogEntry("GpUnit is duplicated", [gpunit]))
|
|
664
|
-
continue
|
|
665
|
-
object_ids.add(object_id)
|
|
666
|
-
composing_gpunits = gpunit.find("ComposingGpUnitIds")
|
|
667
|
-
if composing_gpunits is None or not composing_gpunits.text:
|
|
668
|
-
continue
|
|
669
|
-
composing_ids = frozenset(composing_gpunits.text.split())
|
|
670
|
-
if children.get(composing_ids):
|
|
671
|
-
error_log.append(
|
|
672
|
-
loggers.LogEntry(
|
|
673
|
-
"GpUnits {} are duplicates".format(
|
|
674
|
-
str((children[composing_ids], object_id))
|
|
675
|
-
)
|
|
676
|
-
)
|
|
677
|
-
)
|
|
678
|
-
continue
|
|
679
|
-
children[composing_ids] = object_id
|
|
680
|
-
if error_log:
|
|
681
|
-
raise loggers.ElectionError(error_log)
|
|
682
|
-
|
|
683
|
-
|
|
684
129
|
class GpUnitsHaveSingleRoot(base.TreeRule):
|
|
685
130
|
"""Ensure that GpUnits form a single-rooted tree."""
|
|
686
131
|
|
|
@@ -717,7 +162,7 @@ class GpUnitsHaveSingleRoot(base.TreeRule):
|
|
|
717
162
|
else:
|
|
718
163
|
for object_id in roots:
|
|
719
164
|
element = gpunit_ids.get(object_id)
|
|
720
|
-
ocd_ids = get_external_id_values(element, "ocd-id")
|
|
165
|
+
ocd_ids = rule_util.get_external_id_values(element, "ocd-id")
|
|
721
166
|
for ocd_id in ocd_ids:
|
|
722
167
|
if not gpunit_rules.GpUnitOcdIdValidator.is_country_or_region_ocd_id(
|
|
723
168
|
ocd_id
|
|
@@ -732,99 +177,6 @@ class GpUnitsHaveSingleRoot(base.TreeRule):
|
|
|
732
177
|
raise loggers.ElectionError(self.error_log)
|
|
733
178
|
|
|
734
179
|
|
|
735
|
-
class GpUnitsCyclesRefsValidation(base.TreeRule):
|
|
736
|
-
"""Ensure that GpUnits form a valid tree and no cycles are present."""
|
|
737
|
-
|
|
738
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
739
|
-
super(GpUnitsCyclesRefsValidation, self).__init__(
|
|
740
|
-
election_tree, schema_tree, **kwargs
|
|
741
|
-
)
|
|
742
|
-
self.edges = dict() # Used to maintain the record of connected edges
|
|
743
|
-
self.visited = {} # Used to store status of the nodes as visited or not.
|
|
744
|
-
self.error_log = []
|
|
745
|
-
self.bad_nodes = []
|
|
746
|
-
|
|
747
|
-
def build_tree(self, gpunit):
|
|
748
|
-
# Check if the node is already visited
|
|
749
|
-
if gpunit in self.visited:
|
|
750
|
-
if gpunit not in self.bad_nodes:
|
|
751
|
-
self.error_log.append(
|
|
752
|
-
loggers.LogEntry("Cycle detected at node {0}".format(gpunit))
|
|
753
|
-
)
|
|
754
|
-
self.bad_nodes.append(gpunit)
|
|
755
|
-
return
|
|
756
|
-
self.visited[gpunit] = 1
|
|
757
|
-
# Check each composing_gpunit and its edges if any.
|
|
758
|
-
for child_unit in self.edges[gpunit]:
|
|
759
|
-
if child_unit in self.edges:
|
|
760
|
-
self.build_tree(child_unit)
|
|
761
|
-
else:
|
|
762
|
-
self.error_log.append(
|
|
763
|
-
loggers.LogEntry(
|
|
764
|
-
"Node {0} is not present in the file as a GpUnit element."
|
|
765
|
-
.format(child_unit)
|
|
766
|
-
)
|
|
767
|
-
)
|
|
768
|
-
|
|
769
|
-
def check(self):
|
|
770
|
-
for element in self.get_elements_by_class(self.election_tree, "GpUnit"):
|
|
771
|
-
object_id = element.get("objectId")
|
|
772
|
-
if object_id is None:
|
|
773
|
-
continue
|
|
774
|
-
self.edges[object_id] = []
|
|
775
|
-
composing_gp_unit = element.find("ComposingGpUnitIds")
|
|
776
|
-
if composing_gp_unit is None or composing_gp_unit.text is None:
|
|
777
|
-
continue
|
|
778
|
-
composing_gp_unit_ids = composing_gp_unit.text.split()
|
|
779
|
-
self.edges[object_id] = composing_gp_unit_ids
|
|
780
|
-
for gpunit in self.edges:
|
|
781
|
-
self.build_tree(gpunit)
|
|
782
|
-
self.visited.clear()
|
|
783
|
-
|
|
784
|
-
if self.error_log:
|
|
785
|
-
raise loggers.ElectionError(self.error_log)
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
class OtherType(base.BaseRule):
|
|
789
|
-
"""Elements with an "other" enum should set OtherType.
|
|
790
|
-
|
|
791
|
-
Elements that have enumerations which include a value named other should
|
|
792
|
-
-- when that enumeration value is other -- set the corresponding field
|
|
793
|
-
OtherType within the containing element.
|
|
794
|
-
"""
|
|
795
|
-
|
|
796
|
-
def elements(self):
|
|
797
|
-
eligible_elements = []
|
|
798
|
-
for element in self.schema_tree.iterfind(
|
|
799
|
-
"{%s}complexType" % self._XSCHEMA_NAMESPACE
|
|
800
|
-
):
|
|
801
|
-
for elem in element.iter():
|
|
802
|
-
tag = self.strip_schema_ns(elem)
|
|
803
|
-
if tag == "element":
|
|
804
|
-
elem_name = elem.get("name")
|
|
805
|
-
if elem_name and elem_name == "OtherType":
|
|
806
|
-
eligible_elements.append(element.get("name"))
|
|
807
|
-
return eligible_elements
|
|
808
|
-
|
|
809
|
-
def check(self, element):
|
|
810
|
-
type_element = element.find("Type")
|
|
811
|
-
other_type_element = element.find("OtherType")
|
|
812
|
-
if type_element is not None and type_element.text == "other":
|
|
813
|
-
if other_type_element is None:
|
|
814
|
-
msg = (
|
|
815
|
-
"Type on this element is set to 'other' but OtherType element "
|
|
816
|
-
"is not defined"
|
|
817
|
-
)
|
|
818
|
-
raise loggers.ElectionError.from_message(msg, [element])
|
|
819
|
-
if type_element is not None and type_element.text != "other":
|
|
820
|
-
if other_type_element is not None:
|
|
821
|
-
msg = (
|
|
822
|
-
"Type on this element is not set to 'other' but OtherType "
|
|
823
|
-
"element is defined"
|
|
824
|
-
)
|
|
825
|
-
raise loggers.ElectionError.from_message(msg, [element])
|
|
826
|
-
|
|
827
|
-
|
|
828
180
|
class PartisanPrimary(base.BaseRule):
|
|
829
181
|
"""Partisan elections should link to the correct political party.
|
|
830
182
|
|
|
@@ -850,7 +202,7 @@ class PartisanPrimary(base.BaseRule):
|
|
|
850
202
|
def check(self, election_elem):
|
|
851
203
|
election_type_elem = election_elem.find("Type")
|
|
852
204
|
election_type = None
|
|
853
|
-
if element_has_text(election_type_elem):
|
|
205
|
+
if rule_util.element_has_text(election_type_elem):
|
|
854
206
|
election_type = election_type_elem.text.strip()
|
|
855
207
|
|
|
856
208
|
if not election_type or election_type not in (
|
|
@@ -862,7 +214,7 @@ class PartisanPrimary(base.BaseRule):
|
|
|
862
214
|
contests = self.get_elements_by_class(election_elem, "CandidateContest")
|
|
863
215
|
for contest_elem in contests:
|
|
864
216
|
primary_party_ids = contest_elem.find("PrimaryPartyIds")
|
|
865
|
-
if not element_has_text(primary_party_ids):
|
|
217
|
+
if not rule_util.element_has_text(primary_party_ids):
|
|
866
218
|
msg = (
|
|
867
219
|
"Election is of ElectionType %s but PrimaryPartyIds is not present"
|
|
868
220
|
" or is empty" % (self.election_type)
|
|
@@ -882,7 +234,7 @@ class PartisanPrimaryHeuristic(base.BaseRule):
|
|
|
882
234
|
def check(self, election_elem):
|
|
883
235
|
election_type_elem = election_elem.find("Type")
|
|
884
236
|
election_type = None
|
|
885
|
-
if element_has_text(election_type_elem):
|
|
237
|
+
if rule_util.element_has_text(election_type_elem):
|
|
886
238
|
election_type = election_type_elem.text.strip()
|
|
887
239
|
|
|
888
240
|
if election_type is not None and election_type in (
|
|
@@ -895,7 +247,7 @@ class PartisanPrimaryHeuristic(base.BaseRule):
|
|
|
895
247
|
contests = self.get_elements_by_class(election_elem, "CandidateContest")
|
|
896
248
|
for contest_elem in contests:
|
|
897
249
|
contest_name = contest_elem.find("Name")
|
|
898
|
-
if element_has_text(contest_name):
|
|
250
|
+
if rule_util.element_has_text(contest_name):
|
|
899
251
|
c_name = contest_name.text.replace(" ", "").lower()
|
|
900
252
|
for p_text in self.party_text:
|
|
901
253
|
if p_text in c_name:
|
|
@@ -921,26 +273,6 @@ class CoalitionParties(base.BaseRule):
|
|
|
921
273
|
)
|
|
922
274
|
|
|
923
275
|
|
|
924
|
-
class UniqueLabel(base.BaseRule):
|
|
925
|
-
"""Labels should be unique within a file."""
|
|
926
|
-
|
|
927
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
928
|
-
super(UniqueLabel, self).__init__(election_tree, schema_tree, **kwargs)
|
|
929
|
-
self.labels = set()
|
|
930
|
-
|
|
931
|
-
def elements(self):
|
|
932
|
-
return list(self.get_element_names_for_type("InternationalizedText"))
|
|
933
|
-
|
|
934
|
-
def check(self, element):
|
|
935
|
-
element_label = element.get("label")
|
|
936
|
-
if element_label:
|
|
937
|
-
if element_label in self.labels:
|
|
938
|
-
msg = "Duplicate label '%s'. Label already defined" % element_label
|
|
939
|
-
raise loggers.ElectionError.from_message(msg, [element])
|
|
940
|
-
else:
|
|
941
|
-
self.labels.add(element_label)
|
|
942
|
-
|
|
943
|
-
|
|
944
276
|
class CandidatesReferencedInRelatedContests(base.BaseRule):
|
|
945
277
|
"""Candidate should not be referred to by multiple unrelated contests.
|
|
946
278
|
|
|
@@ -974,7 +306,7 @@ class CandidatesReferencedInRelatedContests(base.BaseRule):
|
|
|
974
306
|
candidate_id_elements = self.get_elements_by_class(contest, "CandidateId")
|
|
975
307
|
id_elements = candidate_ids_elements + candidate_id_elements
|
|
976
308
|
for id_element in id_elements:
|
|
977
|
-
if element_has_text(id_element):
|
|
309
|
+
if rule_util.element_has_text(id_element):
|
|
978
310
|
for candidate_id in id_element.text.split():
|
|
979
311
|
candidate_to_contest_mapping.setdefault(candidate_id, []).append(
|
|
980
312
|
contest_id
|
|
@@ -984,7 +316,7 @@ class CandidatesReferencedInRelatedContests(base.BaseRule):
|
|
|
984
316
|
for candidate in candidates:
|
|
985
317
|
candidate_id = candidate.get("objectId", None)
|
|
986
318
|
person_id = candidate.find("PersonId")
|
|
987
|
-
if element_has_text(person_id):
|
|
319
|
+
if rule_util.element_has_text(person_id):
|
|
988
320
|
if candidate_id not in candidate_to_contest_mapping.keys():
|
|
989
321
|
raise loggers.ElectionError.from_message(
|
|
990
322
|
(
|
|
@@ -1007,7 +339,7 @@ class CandidatesReferencedInRelatedContests(base.BaseRule):
|
|
|
1007
339
|
|
|
1008
340
|
for contest in contests:
|
|
1009
341
|
subsequent_contest = contest.find("SubsequentContestId")
|
|
1010
|
-
if element_has_text(subsequent_contest):
|
|
342
|
+
if rule_util.element_has_text(subsequent_contest):
|
|
1011
343
|
subsequent_contest_id = subsequent_contest.text
|
|
1012
344
|
# subsequent contest id is not valid if it isn't in the graph
|
|
1013
345
|
if not self.contest_graph.has_node(subsequent_contest_id):
|
|
@@ -1023,7 +355,7 @@ class CandidatesReferencedInRelatedContests(base.BaseRule):
|
|
|
1023
355
|
)
|
|
1024
356
|
# Add the composing contest if it exists
|
|
1025
357
|
composing_contests = contest.find("ComposingContestIds")
|
|
1026
|
-
if element_has_text(composing_contests):
|
|
358
|
+
if rule_util.element_has_text(composing_contests):
|
|
1027
359
|
children = composing_contests.text.split()
|
|
1028
360
|
for child in children:
|
|
1029
361
|
# composing contest id is not valid if it isn't in the graph
|
|
@@ -1186,97 +518,15 @@ class SingularPartySelection(base.BaseRule):
|
|
|
1186
518
|
)
|
|
1187
519
|
|
|
1188
520
|
|
|
1189
|
-
class
|
|
1190
|
-
"""
|
|
521
|
+
class ValidatePartyCollection(base.BaseRule):
|
|
522
|
+
"""Generic party collection validation rule.
|
|
1191
523
|
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
be raised. So, we can check if the feed is coherent.
|
|
524
|
+
All partyCollection validation rules can inherit from this class since it
|
|
525
|
+
contains basic checks.
|
|
1195
526
|
"""
|
|
1196
527
|
|
|
1197
528
|
def elements(self):
|
|
1198
|
-
return ["
|
|
1199
|
-
|
|
1200
|
-
def check_specific(self, people):
|
|
1201
|
-
person_def = collections.namedtuple(
|
|
1202
|
-
"PersonDefinition", ["fullname", "birthday"]
|
|
1203
|
-
)
|
|
1204
|
-
person_id_to_object_id = {}
|
|
1205
|
-
|
|
1206
|
-
info_log = []
|
|
1207
|
-
for person in people:
|
|
1208
|
-
person_object_id = person.get("objectId")
|
|
1209
|
-
full_name_list = extract_person_fullname(person)
|
|
1210
|
-
date_of_birthday = person.find("DateOfBirth")
|
|
1211
|
-
birthday_val = "Undefined"
|
|
1212
|
-
if date_of_birthday is not None and date_of_birthday.text:
|
|
1213
|
-
birthday_val = date_of_birthday.text
|
|
1214
|
-
|
|
1215
|
-
for full_name_val in full_name_list:
|
|
1216
|
-
person_id = person_def(full_name_val, birthday_val)
|
|
1217
|
-
if (
|
|
1218
|
-
person_id in person_id_to_object_id
|
|
1219
|
-
and person_id_to_object_id[person_id] != person_object_id
|
|
1220
|
-
):
|
|
1221
|
-
info_message = (
|
|
1222
|
-
"Person has same full name '%s' and birthday %s as Person %s."
|
|
1223
|
-
% (full_name_val, birthday_val, person_id_to_object_id[person_id])
|
|
1224
|
-
)
|
|
1225
|
-
info_log.append(loggers.LogEntry(info_message, [person]))
|
|
1226
|
-
else:
|
|
1227
|
-
person_id_to_object_id[person_id] = person_object_id
|
|
1228
|
-
return info_log
|
|
1229
|
-
|
|
1230
|
-
def check(self, element):
|
|
1231
|
-
info_log = []
|
|
1232
|
-
people = element.findall("Person")
|
|
1233
|
-
if len(people) < 1:
|
|
1234
|
-
info_log.append(
|
|
1235
|
-
loggers.LogEntry(
|
|
1236
|
-
"<PersonCollection> does not have <Person> objects", [element]
|
|
1237
|
-
)
|
|
1238
|
-
)
|
|
1239
|
-
info_log.extend(self.check_specific(people))
|
|
1240
|
-
if info_log:
|
|
1241
|
-
raise loggers.ElectionInfo(info_log)
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
class BadCharactersInPersonFullName(base.BaseRule):
|
|
1245
|
-
"""A person Fullname should not include bad characters."""
|
|
1246
|
-
|
|
1247
|
-
regex = r"([()@$%*/]|\balias\b)"
|
|
1248
|
-
|
|
1249
|
-
def elements(self):
|
|
1250
|
-
return ["Person"]
|
|
1251
|
-
|
|
1252
|
-
def check(self, element):
|
|
1253
|
-
warning_message = (
|
|
1254
|
-
"Person has known bad characters in FullName field."
|
|
1255
|
-
" Aliases should be included in Nickname field."
|
|
1256
|
-
)
|
|
1257
|
-
fullname = extract_person_fullname(element)
|
|
1258
|
-
person_fullname = re.compile(self.regex, flags=re.U)
|
|
1259
|
-
bad_characters_match = None
|
|
1260
|
-
for name in fullname:
|
|
1261
|
-
bad_characters_match = re.search(person_fullname, name.lower())
|
|
1262
|
-
if bad_characters_match:
|
|
1263
|
-
if "alias" in bad_characters_match.group():
|
|
1264
|
-
raise loggers.ElectionWarning.from_message(warning_message, [element])
|
|
1265
|
-
else:
|
|
1266
|
-
raise loggers.ElectionWarning.from_message(
|
|
1267
|
-
"Person has known bad characters in FullName field.", [element]
|
|
1268
|
-
)
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
class ValidatePartyCollection(base.BaseRule):
|
|
1272
|
-
"""Generic party collection validation rule.
|
|
1273
|
-
|
|
1274
|
-
All partyCollection validation rules can inherit from this class since it
|
|
1275
|
-
contains basic checks.
|
|
1276
|
-
"""
|
|
1277
|
-
|
|
1278
|
-
def elements(self):
|
|
1279
|
-
return ["PartyCollection"]
|
|
529
|
+
return ["PartyCollection"]
|
|
1280
530
|
|
|
1281
531
|
def check_specific(self, parties):
|
|
1282
532
|
"""Return a list of info log to be raised."""
|
|
@@ -1309,13 +559,13 @@ class ValidateDuplicateColors(base.TreeRule):
|
|
|
1309
559
|
party_objects[party_id] = party
|
|
1310
560
|
party_colors_by_id[party_id] = {}
|
|
1311
561
|
color_element = party.find("Color")
|
|
1312
|
-
if element_has_text(color_element):
|
|
562
|
+
if rule_util.element_has_text(color_element):
|
|
1313
563
|
party_colors_by_id[party_id]["Color"] = color_element.text.lower()
|
|
1314
564
|
colors_element = party.find("Colors")
|
|
1315
565
|
if colors_element is not None:
|
|
1316
566
|
for color_field in ("DarkThemeColor", "LightThemeColor"):
|
|
1317
567
|
sub_color_element = colors_element.find(color_field)
|
|
1318
|
-
if element_has_text(sub_color_element):
|
|
568
|
+
if rule_util.element_has_text(sub_color_element):
|
|
1319
569
|
party_colors_by_id[party_id][
|
|
1320
570
|
color_field
|
|
1321
571
|
] = sub_color_element.text.lower()
|
|
@@ -1527,32 +777,6 @@ class MissingPartyAbbreviationTranslation(ValidatePartyCollection):
|
|
|
1527
777
|
return info_log
|
|
1528
778
|
|
|
1529
779
|
|
|
1530
|
-
class IndependentPartyName(base.BaseRule):
|
|
1531
|
-
"""Warns on parties that contain common names indicating they are an independent party.
|
|
1532
|
-
|
|
1533
|
-
These should instead supply the IsIndependent attribute.
|
|
1534
|
-
"""
|
|
1535
|
-
|
|
1536
|
-
def elements(self):
|
|
1537
|
-
return ["Party"]
|
|
1538
|
-
|
|
1539
|
-
def check(self, party):
|
|
1540
|
-
is_independent_element = party.find("IsIndependent")
|
|
1541
|
-
if is_independent_element is not None:
|
|
1542
|
-
return
|
|
1543
|
-
name_element = party.find("Name")
|
|
1544
|
-
if name_element is None:
|
|
1545
|
-
return
|
|
1546
|
-
party_names = name_element.findall("Text")
|
|
1547
|
-
for party_name in party_names:
|
|
1548
|
-
if party_name.text.lower() in _INDEPENDENT_PARTY_NAMES:
|
|
1549
|
-
raise loggers.ElectionWarning.from_message(
|
|
1550
|
-
f"Party name '{party_name.text}' indicates an independent party."
|
|
1551
|
-
" Please use the IsIndependent attribute on the party element to"
|
|
1552
|
-
" specify this."
|
|
1553
|
-
)
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
780
|
class DuplicateContestNames(base.BaseRule):
|
|
1557
781
|
"""Check that an election contains unique ContestNames.
|
|
1558
782
|
|
|
@@ -1603,14 +827,18 @@ class DuplicateBallotTitleSeatPair(base.BaseRule):
|
|
|
1603
827
|
contests_by_title_and_seat = collections.defaultdict(list)
|
|
1604
828
|
|
|
1605
829
|
for contest in contest_elements:
|
|
1606
|
-
ballot_title_element = get_language_to_text_map(
|
|
830
|
+
ballot_title_element = rule_util.get_language_to_text_map(
|
|
1607
831
|
contest.find("BallotTitle")
|
|
1608
832
|
)
|
|
1609
833
|
english_title = ballot_title_element.get("en")
|
|
1610
834
|
seat_element = contest.find("Seat")
|
|
1611
835
|
|
|
1612
836
|
ballot_title = english_title[0].strip() if english_title else ""
|
|
1613
|
-
seat =
|
|
837
|
+
seat = (
|
|
838
|
+
seat_element.text.strip()
|
|
839
|
+
if rule_util.element_has_text(seat_element)
|
|
840
|
+
else ""
|
|
841
|
+
)
|
|
1614
842
|
|
|
1615
843
|
if not ballot_title:
|
|
1616
844
|
continue
|
|
@@ -1631,294 +859,6 @@ class DuplicateBallotTitleSeatPair(base.BaseRule):
|
|
|
1631
859
|
raise loggers.ElectionError(error_log)
|
|
1632
860
|
|
|
1633
861
|
|
|
1634
|
-
class UniqueStableID(base.TreeRule):
|
|
1635
|
-
"""Check that every stableID is unique.
|
|
1636
|
-
|
|
1637
|
-
Add an error message if stable id is not unique
|
|
1638
|
-
"""
|
|
1639
|
-
|
|
1640
|
-
def check(self):
|
|
1641
|
-
error_log = []
|
|
1642
|
-
stable_obj_dict = dict()
|
|
1643
|
-
for _, element in etree.iterwalk(self.election_tree, events=("end",)):
|
|
1644
|
-
stable_ids = get_external_id_values(
|
|
1645
|
-
element, "stable", search_subelements=False
|
|
1646
|
-
)
|
|
1647
|
-
if not stable_ids:
|
|
1648
|
-
continue
|
|
1649
|
-
elem_id = self._get_object_id_or_name(element)
|
|
1650
|
-
for stable_id in stable_ids:
|
|
1651
|
-
if stable_id in stable_obj_dict.keys():
|
|
1652
|
-
stable_obj_dict.get(stable_id).append(elem_id)
|
|
1653
|
-
else:
|
|
1654
|
-
elem_id_list = []
|
|
1655
|
-
elem_id_list.append(elem_id)
|
|
1656
|
-
stable_obj_dict[stable_id] = elem_id_list
|
|
1657
|
-
for k, v in stable_obj_dict.items():
|
|
1658
|
-
if len(v) > 1:
|
|
1659
|
-
error_message = (
|
|
1660
|
-
"Stable ID {} is not unique as it is mapped in {}".format(k, v)
|
|
1661
|
-
)
|
|
1662
|
-
error_log.append(loggers.LogEntry(error_message))
|
|
1663
|
-
if error_log:
|
|
1664
|
-
raise loggers.ElectionError(error_log)
|
|
1665
|
-
|
|
1666
|
-
def _get_object_id_or_name(self, element):
|
|
1667
|
-
if "objectId" in element.attrib:
|
|
1668
|
-
return element.get("objectId")
|
|
1669
|
-
names_by_language = get_language_to_text_map(element.find("Name"))
|
|
1670
|
-
if "en" in names_by_language:
|
|
1671
|
-
return names_by_language["en"][0]
|
|
1672
|
-
for names in names_by_language.values():
|
|
1673
|
-
return names[0]
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
class MissingStableIds(base.BaseRule):
|
|
1677
|
-
"""Check that each NIST object in the feed have a stable Id.
|
|
1678
|
-
|
|
1679
|
-
Add an error message if stable id is missing from the object.
|
|
1680
|
-
"""
|
|
1681
|
-
|
|
1682
|
-
_ELEMENTS_WITH_STABLE_IDS = [
|
|
1683
|
-
"BallotMeasureContest",
|
|
1684
|
-
"BallotMeasureSelection",
|
|
1685
|
-
"Candidate",
|
|
1686
|
-
"CandidateContest",
|
|
1687
|
-
"Coalition",
|
|
1688
|
-
"Committee",
|
|
1689
|
-
"Election",
|
|
1690
|
-
"Leadership",
|
|
1691
|
-
"Office",
|
|
1692
|
-
"OfficeHolderTenure",
|
|
1693
|
-
"Party",
|
|
1694
|
-
"PartyContest",
|
|
1695
|
-
"Person",
|
|
1696
|
-
"ReportingUnit",
|
|
1697
|
-
]
|
|
1698
|
-
|
|
1699
|
-
def elements(self):
|
|
1700
|
-
return self._ELEMENTS_WITH_STABLE_IDS
|
|
1701
|
-
|
|
1702
|
-
def check(self, element):
|
|
1703
|
-
stable_ids = []
|
|
1704
|
-
external_identifiers = element.find("ExternalIdentifiers")
|
|
1705
|
-
if external_identifiers is not None:
|
|
1706
|
-
stable_ids = get_external_id_values(external_identifiers, "stable")
|
|
1707
|
-
if not stable_ids:
|
|
1708
|
-
raise loggers.ElectionError.from_message(
|
|
1709
|
-
"The element is missing a stable id", [element]
|
|
1710
|
-
)
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
class PersonsMissingPartyData(base.BaseRule):
|
|
1714
|
-
"""Each Officeholder Person should have a Party associated with it.
|
|
1715
|
-
|
|
1716
|
-
A Person object must contain a PartyId, and if not, it should be picked
|
|
1717
|
-
up within this class and returned to the user as a warning.
|
|
1718
|
-
"""
|
|
1719
|
-
|
|
1720
|
-
def elements(self):
|
|
1721
|
-
return ["Person"]
|
|
1722
|
-
|
|
1723
|
-
def check(self, element):
|
|
1724
|
-
party_id = element.find("PartyId")
|
|
1725
|
-
if party_id is None or not party_id.text or party_id.text.isspace():
|
|
1726
|
-
raise loggers.ElectionWarning.from_message(
|
|
1727
|
-
"The person is missing party data", [element]
|
|
1728
|
-
)
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
class CandidateImageInContactInformation(base.BaseRule):
|
|
1732
|
-
"""Warn if candidate-image is set in ContactInformation.Uri."""
|
|
1733
|
-
|
|
1734
|
-
def elements(self):
|
|
1735
|
-
return ["ContactInformation"]
|
|
1736
|
-
|
|
1737
|
-
def check(self, element):
|
|
1738
|
-
candidate_image_uris = []
|
|
1739
|
-
for uri in element.findall("Uri"):
|
|
1740
|
-
annotation = uri.get("Annotation", "").strip()
|
|
1741
|
-
if annotation == "candidate-image":
|
|
1742
|
-
candidate_image_uris.append(uri)
|
|
1743
|
-
|
|
1744
|
-
if candidate_image_uris:
|
|
1745
|
-
raise loggers.ElectionWarning.from_message(
|
|
1746
|
-
"Annotation 'candidate-image' in ContactInformation.Uri is"
|
|
1747
|
-
" deprecated. Please use Person.ImageUri instead.",
|
|
1748
|
-
candidate_image_uris,
|
|
1749
|
-
)
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
class OnlyOneCandidateImagePerPerson(base.BaseRule):
|
|
1753
|
-
"""Ensure only one candidate-image is provided per Person."""
|
|
1754
|
-
|
|
1755
|
-
def elements(self):
|
|
1756
|
-
return ["Person"]
|
|
1757
|
-
|
|
1758
|
-
def check(self, element):
|
|
1759
|
-
image_uris = element.findall("ImageUri")
|
|
1760
|
-
candidate_images = []
|
|
1761
|
-
for image_uri in image_uris:
|
|
1762
|
-
annotation = image_uri.get("Annotation", "").strip()
|
|
1763
|
-
if annotation == "candidate-image":
|
|
1764
|
-
candidate_images.append(image_uri)
|
|
1765
|
-
|
|
1766
|
-
if len(candidate_images) > 1:
|
|
1767
|
-
raise loggers.ElectionError.from_message(
|
|
1768
|
-
"Person has {} ImageUri fields annotated as 'candidate-image'."
|
|
1769
|
-
" Must have at most one.".format(len(candidate_images)),
|
|
1770
|
-
candidate_images,
|
|
1771
|
-
)
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
class UniqueCandidateImageUris(base.TreeRule):
|
|
1775
|
-
"""Check that candidate-image URIs are unique to a person."""
|
|
1776
|
-
|
|
1777
|
-
def check(self):
|
|
1778
|
-
root = self.election_tree.getroot()
|
|
1779
|
-
if root is None:
|
|
1780
|
-
return
|
|
1781
|
-
|
|
1782
|
-
persons_by_candidate_image_uri = collections.defaultdict(list)
|
|
1783
|
-
person_collection = root.find("PersonCollection")
|
|
1784
|
-
if person_collection is None:
|
|
1785
|
-
return
|
|
1786
|
-
for person in person_collection.findall("Person"):
|
|
1787
|
-
for image_uri_element in person.findall("ImageUri"):
|
|
1788
|
-
if not element_has_text(image_uri_element):
|
|
1789
|
-
continue
|
|
1790
|
-
annotation = image_uri_element.get("Annotation", "").strip()
|
|
1791
|
-
if annotation == "candidate-image":
|
|
1792
|
-
image_uri = image_uri_element.text.strip()
|
|
1793
|
-
if image_uri:
|
|
1794
|
-
persons_by_candidate_image_uri[image_uri].append(person)
|
|
1795
|
-
|
|
1796
|
-
error_log = []
|
|
1797
|
-
for image_uri, persons in persons_by_candidate_image_uri.items():
|
|
1798
|
-
if len(persons) > 1:
|
|
1799
|
-
person_ids = sorted(person.get("objectId") for person in persons)
|
|
1800
|
-
error_log.append(
|
|
1801
|
-
loggers.LogEntry(
|
|
1802
|
-
f"Candidate image URI '{image_uri}' is shared by multiple"
|
|
1803
|
-
f" people: [{', '.join(person_ids)}].",
|
|
1804
|
-
persons,
|
|
1805
|
-
)
|
|
1806
|
-
)
|
|
1807
|
-
if error_log:
|
|
1808
|
-
raise loggers.ElectionError(error_log)
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
class AllCaps(base.BaseRule):
|
|
1812
|
-
"""Name elements should not be in all uppercase.
|
|
1813
|
-
|
|
1814
|
-
If the name elements in Candidates, Contests and Person elements are in
|
|
1815
|
-
uppercase, the list of objectIds of those elements will be returned to
|
|
1816
|
-
the user as a warning.
|
|
1817
|
-
"""
|
|
1818
|
-
|
|
1819
|
-
_element_field_mapping = {
|
|
1820
|
-
"Candidate": ["BallotName//Text"],
|
|
1821
|
-
"CandidateContest": ["Name"],
|
|
1822
|
-
"PartyContest": ["Name"],
|
|
1823
|
-
"Person": ["FullName//Text"],
|
|
1824
|
-
}
|
|
1825
|
-
|
|
1826
|
-
def elements(self):
|
|
1827
|
-
return list(self._element_field_mapping.keys())
|
|
1828
|
-
|
|
1829
|
-
def check(self, element):
|
|
1830
|
-
error_log = []
|
|
1831
|
-
|
|
1832
|
-
element_tag = self.get_element_class(element)
|
|
1833
|
-
to_check_field_tags = self._element_field_mapping[element_tag]
|
|
1834
|
-
for field_tag in to_check_field_tags:
|
|
1835
|
-
to_check_field = element.find(field_tag)
|
|
1836
|
-
if to_check_field is not None:
|
|
1837
|
-
text_value = to_check_field.text
|
|
1838
|
-
if text_value and text_value.isupper():
|
|
1839
|
-
raise loggers.ElectionWarning.from_message(
|
|
1840
|
-
"{0} has {1} in all upper case letters.".format(
|
|
1841
|
-
element_tag, field_tag
|
|
1842
|
-
),
|
|
1843
|
-
[element],
|
|
1844
|
-
)
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
class AllLanguages(base.BaseRule):
|
|
1848
|
-
"""Verify that required languages are present in Text fields.
|
|
1849
|
-
|
|
1850
|
-
The Text elements in all entities with those fields should cover all
|
|
1851
|
-
required languages for this schema.
|
|
1852
|
-
"""
|
|
1853
|
-
|
|
1854
|
-
required_languages = []
|
|
1855
|
-
|
|
1856
|
-
def elements(self):
|
|
1857
|
-
return ["BallotName", "BallotTitle", "FullName", "Name"]
|
|
1858
|
-
|
|
1859
|
-
def check(self, element):
|
|
1860
|
-
text_elements = element.findall("Text")
|
|
1861
|
-
if not text_elements:
|
|
1862
|
-
return
|
|
1863
|
-
languages = set()
|
|
1864
|
-
for text in text_elements:
|
|
1865
|
-
languages.add(text.attrib["language"])
|
|
1866
|
-
required_language_set = frozenset(self.required_languages)
|
|
1867
|
-
if not required_language_set.issubset(languages):
|
|
1868
|
-
msg = (
|
|
1869
|
-
"Element does not contain text in all required languages, missing"
|
|
1870
|
-
+ " : %s" % str(required_language_set - languages)
|
|
1871
|
-
)
|
|
1872
|
-
raise loggers.ElectionError.from_message(msg, [element])
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
class ValidEnumerations(base.BaseRule):
|
|
1876
|
-
"""Valid enumerations should not be encoded as 'OtherType'.
|
|
1877
|
-
|
|
1878
|
-
Elements that have valid enumerations should not be included
|
|
1879
|
-
as 'OtherType'. Instead, the corresponding <Type> field
|
|
1880
|
-
should include the actual valid enumeration value.
|
|
1881
|
-
"""
|
|
1882
|
-
|
|
1883
|
-
valid_enumerations = []
|
|
1884
|
-
|
|
1885
|
-
def elements(self):
|
|
1886
|
-
eligible_elements = []
|
|
1887
|
-
for element in self.schema_tree.iter():
|
|
1888
|
-
tag = self.strip_schema_ns(element)
|
|
1889
|
-
if tag == "enumeration":
|
|
1890
|
-
elem_val = element.get("value")
|
|
1891
|
-
if elem_val and elem_val != "other":
|
|
1892
|
-
self.valid_enumerations.append(elem_val)
|
|
1893
|
-
elif tag == "complexType":
|
|
1894
|
-
for elem in element.iter():
|
|
1895
|
-
tag = self.strip_schema_ns(elem)
|
|
1896
|
-
if tag == "element":
|
|
1897
|
-
elem_name = elem.get("name")
|
|
1898
|
-
if elem_name and element.get("name") and elem_name == "OtherType":
|
|
1899
|
-
if element.get("name") == "ExternalIdentifiers":
|
|
1900
|
-
eligible_elements.append("ExternalIdentifier")
|
|
1901
|
-
continue
|
|
1902
|
-
eligible_elements.append(element.get("name"))
|
|
1903
|
-
return eligible_elements
|
|
1904
|
-
|
|
1905
|
-
def check(self, element):
|
|
1906
|
-
type_element = element.find("Type")
|
|
1907
|
-
if type_element is not None and type_element.text == "other":
|
|
1908
|
-
other_type_element = element.find("OtherType")
|
|
1909
|
-
if (
|
|
1910
|
-
other_type_element is not None
|
|
1911
|
-
and other_type_element.text in self.valid_enumerations
|
|
1912
|
-
):
|
|
1913
|
-
raise loggers.ElectionError.from_message(
|
|
1914
|
-
(
|
|
1915
|
-
"Type is set to 'other' even though '%s' is a valid enumeration"
|
|
1916
|
-
% other_type_element.text
|
|
1917
|
-
),
|
|
1918
|
-
[element],
|
|
1919
|
-
)
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
862
|
class MultipleCandidatesPointToTheSamePersonInTheSameContest(base.TreeRule):
|
|
1923
863
|
"""Raise an error when multiple candidates point to the same person in the same contest."""
|
|
1924
864
|
|
|
@@ -1989,7 +929,9 @@ class SelfDeclaredCandidateMethod(base.BaseRule):
|
|
|
1989
929
|
def check(self, element):
|
|
1990
930
|
status = element.find("PreElectionStatus")
|
|
1991
931
|
if status is not None and status.text == "self-declared":
|
|
1992
|
-
externalidvalues = get_external_id_values(
|
|
932
|
+
externalidvalues = rule_util.get_external_id_values(
|
|
933
|
+
element, "electoral-commission"
|
|
934
|
+
)
|
|
1993
935
|
length = len(externalidvalues)
|
|
1994
936
|
if length > 0:
|
|
1995
937
|
msg = (
|
|
@@ -1999,25 +941,6 @@ class SelfDeclaredCandidateMethod(base.BaseRule):
|
|
|
1999
941
|
raise loggers.ElectionWarning.from_message(msg, [element])
|
|
2000
942
|
|
|
2001
943
|
|
|
2002
|
-
class ValidateOcdidLowerCase(base.BaseRule):
|
|
2003
|
-
"""Validate that the ocd-ids are all lower case.
|
|
2004
|
-
|
|
2005
|
-
Throw a warning if the ocd-ids are not all in lowercase.
|
|
2006
|
-
"""
|
|
2007
|
-
|
|
2008
|
-
def elements(self):
|
|
2009
|
-
return ["ExternalIdentifiers"]
|
|
2010
|
-
|
|
2011
|
-
def check(self, element):
|
|
2012
|
-
for ocd_id in get_external_id_values(element, "ocd-id"):
|
|
2013
|
-
if not ocd_id.islower():
|
|
2014
|
-
msg = (
|
|
2015
|
-
"OCD-ID %s is not in all lower case letters. "
|
|
2016
|
-
"Valid OCD-IDs should be all lowercase." % (ocd_id)
|
|
2017
|
-
)
|
|
2018
|
-
raise loggers.ElectionWarning.from_message(msg, [element])
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
944
|
class ContestHasMultipleOffices(base.BaseRule):
|
|
2022
945
|
"""Ensure that each contest has exactly one Office."""
|
|
2023
946
|
|
|
@@ -2058,7 +981,10 @@ class PersonHasOffice(base.ValidReferenceRule):
|
|
|
2058
981
|
# Add party leaders provided in the External Identifier
|
|
2059
982
|
for external_id in root.findall(".//Party//ExternalIdentifier"):
|
|
2060
983
|
other_type = external_id.find("OtherType")
|
|
2061
|
-
if
|
|
984
|
+
if (
|
|
985
|
+
other_type is not None
|
|
986
|
+
and other_type.text in rule_util.PARTY_LEADERSHIP_TYPES
|
|
987
|
+
):
|
|
2062
988
|
person_reference_ids.add(external_id.find("Value").text)
|
|
2063
989
|
# Add party leaders provided in the Leadership entity
|
|
2064
990
|
for leader_id in root.findall(".//Party//PartyLeaderId"):
|
|
@@ -2093,79 +1019,6 @@ class PersonHasOffice(base.ValidReferenceRule):
|
|
|
2093
1019
|
return person_reference_ids
|
|
2094
1020
|
|
|
2095
1021
|
|
|
2096
|
-
class PartyLeadershipMustExist(base.ValidReferenceRule):
|
|
2097
|
-
"""Each party leader or party chair should refer to a person in the feed."""
|
|
2098
|
-
|
|
2099
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
2100
|
-
super(PartyLeadershipMustExist, self).__init__(
|
|
2101
|
-
election_tree, schema_tree, "Person", **kwargs
|
|
2102
|
-
)
|
|
2103
|
-
|
|
2104
|
-
def _gather_reference_values(self):
|
|
2105
|
-
root = self.election_tree.getroot()
|
|
2106
|
-
if root is None:
|
|
2107
|
-
return
|
|
2108
|
-
|
|
2109
|
-
party_leader_ids = set()
|
|
2110
|
-
for external_id in root.findall(".//Party//ExternalIdentifier"):
|
|
2111
|
-
other_type = external_id.find("OtherType")
|
|
2112
|
-
if other_type is not None and other_type.text in _PARTY_LEADERSHIP_TYPES:
|
|
2113
|
-
party_leader_ids.add(external_id.find("Value").text)
|
|
2114
|
-
return party_leader_ids
|
|
2115
|
-
|
|
2116
|
-
def _gather_defined_values(self):
|
|
2117
|
-
root = self.election_tree.getroot()
|
|
2118
|
-
if root is None:
|
|
2119
|
-
return
|
|
2120
|
-
|
|
2121
|
-
persons = root.find("PersonCollection")
|
|
2122
|
-
all_person_ids = set()
|
|
2123
|
-
if persons is not None:
|
|
2124
|
-
all_person_ids = {person.attrib["objectId"] for person in persons}
|
|
2125
|
-
return all_person_ids
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
class ProhibitElectionData(base.TreeRule):
|
|
2129
|
-
"""Ensure that election data is not provided for officeholder feeds."""
|
|
2130
|
-
|
|
2131
|
-
def check(self):
|
|
2132
|
-
root = self.election_tree.getroot()
|
|
2133
|
-
if root is not None and root.find("Election") is not None:
|
|
2134
|
-
raise loggers.ElectionError.from_message(
|
|
2135
|
-
"Election data is prohibited in officeholder feeds."
|
|
2136
|
-
)
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
class PersonsHaveValidGender(base.BaseRule):
|
|
2140
|
-
"""Ensure that all Person objects have a valid gender identification."""
|
|
2141
|
-
|
|
2142
|
-
_VALID_GENDERS = {
|
|
2143
|
-
"male",
|
|
2144
|
-
"m",
|
|
2145
|
-
"man",
|
|
2146
|
-
"female",
|
|
2147
|
-
"f",
|
|
2148
|
-
"woman",
|
|
2149
|
-
"o",
|
|
2150
|
-
"x",
|
|
2151
|
-
"other",
|
|
2152
|
-
"nonbinary",
|
|
2153
|
-
}
|
|
2154
|
-
|
|
2155
|
-
def elements(self):
|
|
2156
|
-
return ["Gender"]
|
|
2157
|
-
|
|
2158
|
-
def check(self, element):
|
|
2159
|
-
if (
|
|
2160
|
-
element.text is not None
|
|
2161
|
-
and element.text.lower() not in self._VALID_GENDERS
|
|
2162
|
-
):
|
|
2163
|
-
raise loggers.ElectionError.from_message(
|
|
2164
|
-
"Person object has invalid gender value: {0}".format(element.text),
|
|
2165
|
-
[element],
|
|
2166
|
-
)
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
1022
|
class VoteCountTypesCoherency(base.BaseRule):
|
|
2170
1023
|
"""Ensure VoteCount types describe the appropriate votable."""
|
|
2171
1024
|
|
|
@@ -2205,325 +1058,35 @@ class VoteCountTypesCoherency(base.BaseRule):
|
|
|
2205
1058
|
errors.append(vc_type)
|
|
2206
1059
|
if errors:
|
|
2207
1060
|
msg = "VoteCount types {0} should not be nested in {1} Contest".format(
|
|
2208
|
-
", ".join(errors), contest_type
|
|
2209
|
-
)
|
|
2210
|
-
raise loggers.ElectionError.from_message(msg, [element])
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
class VoteCountValidSeatsDeltaTypes(base.BaseRule):
|
|
2214
|
-
"""Ensure VoteCount seats delta types are valid."""
|
|
2215
|
-
|
|
2216
|
-
def elements(self):
|
|
2217
|
-
return ["BallotSelection"] + list(
|
|
2218
|
-
self.get_element_names_for_type("SpecialBallotSelection")
|
|
2219
|
-
)
|
|
2220
|
-
|
|
2221
|
-
def check(self, element):
|
|
2222
|
-
vote_count_types = set()
|
|
2223
|
-
vote_counts_collection = element.find("VoteCountsCollection")
|
|
2224
|
-
if vote_counts_collection is None:
|
|
2225
|
-
return
|
|
2226
|
-
for vote_counts in vote_counts_collection.findall("VoteCounts"):
|
|
2227
|
-
vote_count_types.add(_get_type_or_other_type(vote_counts))
|
|
2228
|
-
if "seats-delta" in vote_count_types:
|
|
2229
|
-
raise loggers.ElectionError.from_message(
|
|
2230
|
-
"Legacy VoteCounts type seats-delta is no longer supported as of"
|
|
2231
|
-
" July 1, 2026. Please update your implementation to use"
|
|
2232
|
-
" seats-delta-mandate and/or seats-delta-institutional.",
|
|
2233
|
-
[element],
|
|
2234
|
-
)
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
class URIValidator(base.BaseRule):
|
|
2238
|
-
"""Basic URL validations.
|
|
2239
|
-
|
|
2240
|
-
Ensure each URL has valid protocol, domain, and query.
|
|
2241
|
-
"""
|
|
2242
|
-
|
|
2243
|
-
def elements(self):
|
|
2244
|
-
return ["Uri"]
|
|
2245
|
-
|
|
2246
|
-
def check(self, element):
|
|
2247
|
-
url = element.text
|
|
2248
|
-
if url is None:
|
|
2249
|
-
raise loggers.ElectionError.from_message("Missing URI value.", [element])
|
|
2250
|
-
|
|
2251
|
-
parsed_url = urlparse(url)
|
|
2252
|
-
discrepancies = []
|
|
2253
|
-
|
|
2254
|
-
try:
|
|
2255
|
-
url.encode("ascii")
|
|
2256
|
-
except UnicodeEncodeError:
|
|
2257
|
-
discrepancies.append("not ascii encoded")
|
|
2258
|
-
|
|
2259
|
-
if parsed_url.scheme not in {"http", "https"}:
|
|
2260
|
-
discrepancies.append("protocol - invalid")
|
|
2261
|
-
if not parsed_url.netloc:
|
|
2262
|
-
discrepancies.append("domain - missing")
|
|
2263
|
-
if discrepancies:
|
|
2264
|
-
msg = (
|
|
2265
|
-
"The provided URI, {}, is invalid for the following reasons: {}."
|
|
2266
|
-
.format(url.encode("ascii", "ignore"), ", ".join(discrepancies))
|
|
2267
|
-
)
|
|
2268
|
-
raise loggers.ElectionError.from_message(msg, [element])
|
|
2269
|
-
|
|
2270
|
-
for platform in _ALL_URI_PLATFORMS:
|
|
2271
|
-
if (
|
|
2272
|
-
re.search(platform, parsed_url.netloc)
|
|
2273
|
-
and parsed_url.scheme != "https"
|
|
2274
|
-
):
|
|
2275
|
-
msg = (
|
|
2276
|
-
"It is recommended to use https instead of http. The provided "
|
|
2277
|
-
"URI, '{}'."
|
|
2278
|
-
).format(url)
|
|
2279
|
-
raise loggers.ElectionInfo.from_message(msg, [element])
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
class UniqueURIPerAnnotationCategory(base.TreeRule):
|
|
2283
|
-
"""Check that annotated URIs for each category are unique.
|
|
2284
|
-
|
|
2285
|
-
Ex: No ballotpedia URIs can be the same.
|
|
2286
|
-
"""
|
|
2287
|
-
|
|
2288
|
-
def _extract_uris_by_category(self, uri_elements):
|
|
2289
|
-
"""For given list of Uri elements return nested paths of Uri values.
|
|
2290
|
-
|
|
2291
|
-
Args:
|
|
2292
|
-
uri_elements: List of Uri elements
|
|
2293
|
-
|
|
2294
|
-
Returns:
|
|
2295
|
-
Top level dict contains Annotation values as keys with uri/paths mapping
|
|
2296
|
-
as value.
|
|
2297
|
-
"""
|
|
2298
|
-
uri_mapping = {}
|
|
2299
|
-
for uri in uri_elements:
|
|
2300
|
-
annotation = uri.get("Annotation", "").strip()
|
|
2301
|
-
annotation_elements = annotation.split("-")
|
|
2302
|
-
annotation_platform = ""
|
|
2303
|
-
if annotation_elements:
|
|
2304
|
-
annotation_platform = annotation_elements[-1]
|
|
2305
|
-
|
|
2306
|
-
uri_value = uri.text
|
|
2307
|
-
|
|
2308
|
-
if annotation_platform not in uri_mapping.keys():
|
|
2309
|
-
uri_mapping[annotation_platform] = {}
|
|
2310
|
-
|
|
2311
|
-
if uri_mapping[annotation_platform].get(uri_value):
|
|
2312
|
-
uri_mapping[annotation_platform][uri_value].append(uri)
|
|
2313
|
-
else:
|
|
2314
|
-
uri_mapping[annotation_platform][uri_value] = [uri]
|
|
2315
|
-
return uri_mapping
|
|
2316
|
-
|
|
2317
|
-
def check(self):
|
|
2318
|
-
all_uri_elements = self.get_elements_by_class(self.election_tree, "Uri")
|
|
2319
|
-
office_uri_elements = self.get_elements_by_class(
|
|
2320
|
-
self.election_tree, "Office//ContactInformation//Uri"
|
|
2321
|
-
)
|
|
2322
|
-
uri_elements = set(all_uri_elements) - set(office_uri_elements)
|
|
2323
|
-
annotation_mapper = self._extract_uris_by_category(uri_elements)
|
|
2324
|
-
|
|
2325
|
-
error_log = []
|
|
2326
|
-
for annotation, value_counter in annotation_mapper.items():
|
|
2327
|
-
for uri, uri_elements in value_counter.items():
|
|
2328
|
-
if len(uri_elements) > 1:
|
|
2329
|
-
error_message = (
|
|
2330
|
-
"The Uris contain the annotation type '{}' with the "
|
|
2331
|
-
"same value '{}'."
|
|
2332
|
-
).format(annotation, uri)
|
|
2333
|
-
error_log.append(loggers.LogEntry(error_message, uri_elements))
|
|
2334
|
-
|
|
2335
|
-
if error_log:
|
|
2336
|
-
raise loggers.ElectionWarning(error_log)
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
class ValidYoutubeURL(base.BaseRule):
|
|
2340
|
-
"""Validate Youtube URL.
|
|
2341
|
-
|
|
2342
|
-
Ensure the provided URL is not a generic youtube url or direct link to a
|
|
2343
|
-
playlist as one of the invalid youtube URL types.
|
|
2344
|
-
"""
|
|
2345
|
-
|
|
2346
|
-
def elements(self):
|
|
2347
|
-
return ["Uri"]
|
|
2348
|
-
|
|
2349
|
-
def check(self, element):
|
|
2350
|
-
url = element.text.strip()
|
|
2351
|
-
parsed_url = urlparse(url)
|
|
2352
|
-
if "youtube" in parsed_url.netloc and (
|
|
2353
|
-
parsed_url.path in ["", "/"]
|
|
2354
|
-
or "watch" in parsed_url.path
|
|
2355
|
-
or "playlist" in parsed_url.path
|
|
2356
|
-
or "hashtag" in parsed_url.path
|
|
2357
|
-
):
|
|
2358
|
-
raise loggers.ElectionError.from_message(
|
|
2359
|
-
"'{}' is not an expected value for a youtube channel.".format(url),
|
|
2360
|
-
[element],
|
|
2361
|
-
)
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
class ValidTiktokURL(base.BaseRule):
|
|
2365
|
-
"""Validate Tiktok URL.
|
|
2366
|
-
|
|
2367
|
-
Ensure the provided URL is a valid Tiktok URL types.
|
|
2368
|
-
"""
|
|
2369
|
-
|
|
2370
|
-
def elements(self):
|
|
2371
|
-
return ["Uri"]
|
|
2372
|
-
|
|
2373
|
-
def check(self, element):
|
|
2374
|
-
url = element.text.strip()
|
|
2375
|
-
parsed_url = urlparse(url)
|
|
2376
|
-
if "tiktok" in parsed_url.netloc and (
|
|
2377
|
-
not re.match(r"^\/@[^\/@]+$", parsed_url.path)
|
|
2378
|
-
or parsed_url.query
|
|
2379
|
-
or parsed_url.fragment
|
|
2380
|
-
):
|
|
2381
|
-
raise loggers.ElectionError.from_message(
|
|
2382
|
-
"'{}' is not an expected value for a tiktok account.".format(url),
|
|
2383
|
-
[element],
|
|
2384
|
-
)
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
class ValidURIAnnotation(base.BaseRule):
|
|
2388
|
-
"""Validate annotations on candidate/officeholder URLs.
|
|
2389
|
-
|
|
2390
|
-
Ensure they describe the type of URL presented.
|
|
2391
|
-
Throws Warnings and Errors depending on type of invalidity.
|
|
2392
|
-
"""
|
|
2393
|
-
|
|
2394
|
-
def elements(self):
|
|
2395
|
-
return ["ContactInformation"]
|
|
2396
|
-
|
|
2397
|
-
def check_url(self, uri, annotation, platform):
|
|
2398
|
-
url = uri.text.strip()
|
|
2399
|
-
parsed_url = urlparse(url)
|
|
2400
|
-
# Ensure media platform name is in URL.
|
|
2401
|
-
if (
|
|
2402
|
-
platform != "website"
|
|
2403
|
-
and platform not in parsed_url.netloc
|
|
2404
|
-
and not (platform == "facebook" and "fb.com" in parsed_url.netloc)
|
|
2405
|
-
and not (platform == "twitter" and "x.com" in parsed_url.netloc)
|
|
2406
|
-
):
|
|
2407
|
-
# Note that the URL is encoded for printing purposes
|
|
2408
|
-
raise loggers.ElectionError.from_message(
|
|
2409
|
-
"Annotation '{}' is incorrect for URI {}.".format(
|
|
2410
|
-
annotation, url.encode("ascii", "ignore")
|
|
2411
|
-
),
|
|
2412
|
-
[uri],
|
|
2413
|
-
)
|
|
2414
|
-
|
|
2415
|
-
def check(self, element):
|
|
2416
|
-
uris = element.findall("Uri")
|
|
2417
|
-
|
|
2418
|
-
for uri in uris:
|
|
2419
|
-
annotation = uri.get("Annotation", "").strip()
|
|
2420
|
-
url = uri.text.strip()
|
|
2421
|
-
ascii_url = url.encode("ascii", "ignore")
|
|
2422
|
-
|
|
2423
|
-
if not annotation:
|
|
2424
|
-
raise loggers.ElectionWarning.from_message(
|
|
2425
|
-
"URI {} is missing annotation.".format(ascii_url), [uri]
|
|
2426
|
-
)
|
|
2427
|
-
|
|
2428
|
-
# Skip platform checks for office contact form and candidate-image
|
|
2429
|
-
# annotations.
|
|
2430
|
-
if annotation in ("office-contact_form", "candidate-image"):
|
|
2431
|
-
continue
|
|
2432
|
-
|
|
2433
|
-
ann_elements = annotation.split("-")
|
|
2434
|
-
if len(ann_elements) == 1:
|
|
2435
|
-
platform = ann_elements[0]
|
|
2436
|
-
# One element would imply the annotation could be a platform
|
|
2437
|
-
# without a usage type, which is checked here.
|
|
2438
|
-
if platform in _TYPED_URI_PLATFORMS:
|
|
2439
|
-
raise loggers.ElectionWarning.from_message(
|
|
2440
|
-
"Annotation '{}' missing usage type.".format(annotation), [uri]
|
|
2441
|
-
)
|
|
2442
|
-
elif platform in _URI_USAGE_TYPES:
|
|
2443
|
-
raise loggers.ElectionError.from_message(
|
|
2444
|
-
"Annotation '{}' has usage type, missing platform.".format(
|
|
2445
|
-
annotation
|
|
2446
|
-
),
|
|
2447
|
-
[uri],
|
|
2448
|
-
)
|
|
2449
|
-
elif platform not in _UNTYPED_URI_PLATFORMS:
|
|
2450
|
-
raise loggers.ElectionError.from_message(
|
|
2451
|
-
"Annotation '{}' is not a valid annotation for URI {}.".format(
|
|
2452
|
-
annotation, ascii_url
|
|
2453
|
-
),
|
|
2454
|
-
[uri],
|
|
2455
|
-
)
|
|
2456
|
-
elif len(ann_elements) == 2:
|
|
2457
|
-
# Two elements at this stage would mean the annotation
|
|
2458
|
-
# must be a platform with a usage type.
|
|
2459
|
-
usage_type, platform = ann_elements
|
|
2460
|
-
if (
|
|
2461
|
-
usage_type not in _URI_USAGE_TYPES
|
|
2462
|
-
or platform not in _TYPED_URI_PLATFORMS
|
|
2463
|
-
):
|
|
2464
|
-
raise loggers.ElectionWarning.from_message(
|
|
2465
|
-
"'{}' is not a valid annotation.".format(annotation), [uri]
|
|
2466
|
-
)
|
|
2467
|
-
else:
|
|
2468
|
-
# More than two implies an invalid annotation.
|
|
2469
|
-
raise loggers.ElectionError.from_message(
|
|
2470
|
-
"Annotation '{}' is invalid for URI {}.".format(
|
|
2471
|
-
annotation, ascii_url
|
|
2472
|
-
),
|
|
2473
|
-
[uri],
|
|
1061
|
+
", ".join(errors), contest_type
|
|
2474
1062
|
)
|
|
2475
|
-
|
|
2476
|
-
self.check_url(uri, annotation, platform)
|
|
1063
|
+
raise loggers.ElectionError.from_message(msg, [element])
|
|
2477
1064
|
|
|
2478
1065
|
|
|
2479
|
-
class
|
|
2480
|
-
"""
|
|
1066
|
+
class VoteCountValidSeatsDeltaTypes(base.BaseRule):
|
|
1067
|
+
"""Ensure VoteCount seats delta types are valid."""
|
|
2481
1068
|
|
|
2482
1069
|
def elements(self):
|
|
2483
|
-
return ["
|
|
1070
|
+
return ["BallotSelection"] + list(
|
|
1071
|
+
self.get_element_names_for_type("SpecialBallotSelection")
|
|
1072
|
+
)
|
|
2484
1073
|
|
|
2485
1074
|
def check(self, element):
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
if
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
)
|
|
2494
|
-
|
|
2495
|
-
jurisdiction_values = [j_id for j_id in jurisdiction_values if j_id.strip()]
|
|
2496
|
-
if not jurisdiction_values:
|
|
2497
|
-
raise loggers.ElectionError.from_message(
|
|
2498
|
-
"Office is missing a jurisdiction ID.", [element]
|
|
2499
|
-
)
|
|
2500
|
-
if len(jurisdiction_values) > 1:
|
|
1075
|
+
vote_count_types = set()
|
|
1076
|
+
vote_counts_collection = element.find("VoteCountsCollection")
|
|
1077
|
+
if vote_counts_collection is None:
|
|
1078
|
+
return
|
|
1079
|
+
for vote_counts in vote_counts_collection.findall("VoteCounts"):
|
|
1080
|
+
vote_count_types.add(_get_type_or_other_type(vote_counts))
|
|
1081
|
+
if "seats-delta" in vote_count_types:
|
|
2501
1082
|
raise loggers.ElectionError.from_message(
|
|
2502
|
-
"
|
|
1083
|
+
"Legacy VoteCounts type seats-delta is no longer supported as of"
|
|
1084
|
+
" July 1, 2026. Please update your implementation to use"
|
|
1085
|
+
" seats-delta-mandate and/or seats-delta-institutional.",
|
|
2503
1086
|
[element],
|
|
2504
1087
|
)
|
|
2505
1088
|
|
|
2506
1089
|
|
|
2507
|
-
class ValidJurisdictionID(base.ValidReferenceRule):
|
|
2508
|
-
"""Each jurisdiction id should refer to a valid GpUnit."""
|
|
2509
|
-
|
|
2510
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
2511
|
-
super(ValidJurisdictionID, self).__init__(
|
|
2512
|
-
election_tree, schema_tree, "GpUnit", **kwargs
|
|
2513
|
-
)
|
|
2514
|
-
|
|
2515
|
-
def _gather_reference_values(self):
|
|
2516
|
-
root = self.election_tree.getroot()
|
|
2517
|
-
jurisdiction_values = get_entity_info_for_value_type(
|
|
2518
|
-
root, "jurisdiction-id"
|
|
2519
|
-
)
|
|
2520
|
-
return set(jurisdiction_values)
|
|
2521
|
-
|
|
2522
|
-
def _gather_defined_values(self):
|
|
2523
|
-
gp_unit_elements = self.election_tree.getroot().findall(".//GpUnit")
|
|
2524
|
-
return {elem.get("objectId") for elem in gp_unit_elements}
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
1090
|
class OfficeHasjurisdictionSameAsElectoralDistrict(base.BaseRule):
|
|
2528
1091
|
"""In election feeds, office has the electoral district same as jurisdiction."""
|
|
2529
1092
|
|
|
@@ -2531,7 +1094,7 @@ class OfficeHasjurisdictionSameAsElectoralDistrict(base.BaseRule):
|
|
|
2531
1094
|
return ["Office"]
|
|
2532
1095
|
|
|
2533
1096
|
def check(self, element):
|
|
2534
|
-
jurisdiction_values = get_entity_info_for_value_type(
|
|
1097
|
+
jurisdiction_values = rule_util.get_entity_info_for_value_type(
|
|
2535
1098
|
element, "jurisdiction-id"
|
|
2536
1099
|
)
|
|
2537
1100
|
jurisdiction_values = [
|
|
@@ -2550,108 +1113,6 @@ class OfficeHasjurisdictionSameAsElectoralDistrict(base.BaseRule):
|
|
|
2550
1113
|
)
|
|
2551
1114
|
|
|
2552
1115
|
|
|
2553
|
-
class OfficesHaveValidOfficeLevel(base.BaseRule):
|
|
2554
|
-
"""Each office must have a valid office-level."""
|
|
2555
|
-
|
|
2556
|
-
def elements(self):
|
|
2557
|
-
return ["Office"]
|
|
2558
|
-
|
|
2559
|
-
def check(self, element):
|
|
2560
|
-
office_levels = []
|
|
2561
|
-
post_office_split_office_level_element = element.find("Level")
|
|
2562
|
-
if element_has_text(post_office_split_office_level_element):
|
|
2563
|
-
office_levels.append(post_office_split_office_level_element.text)
|
|
2564
|
-
else:
|
|
2565
|
-
office_levels = [
|
|
2566
|
-
ol_id.strip()
|
|
2567
|
-
for ol_id in get_external_id_values(element, "office-level")
|
|
2568
|
-
if ol_id.strip()
|
|
2569
|
-
]
|
|
2570
|
-
|
|
2571
|
-
if not office_levels:
|
|
2572
|
-
raise loggers.ElectionError.from_message(
|
|
2573
|
-
"Office is missing an office level.",
|
|
2574
|
-
[element],
|
|
2575
|
-
)
|
|
2576
|
-
if len(office_levels) > 1:
|
|
2577
|
-
raise loggers.ElectionError.from_message(
|
|
2578
|
-
"Office has more than one office level.",
|
|
2579
|
-
[element],
|
|
2580
|
-
)
|
|
2581
|
-
office_level = office_levels[0]
|
|
2582
|
-
if office_level not in office_utils.VALID_OFFICE_LEVELS:
|
|
2583
|
-
raise loggers.ElectionError.from_message(
|
|
2584
|
-
f"Office has an invalid office level: '{office_level}'.",
|
|
2585
|
-
[element],
|
|
2586
|
-
)
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
class OfficesHaveValidOfficeRole(base.BaseRule):
|
|
2590
|
-
"""Each office must have valid office role(s)."""
|
|
2591
|
-
|
|
2592
|
-
def elements(self):
|
|
2593
|
-
return ["Office"]
|
|
2594
|
-
|
|
2595
|
-
def check(self, element):
|
|
2596
|
-
office_roles = [role.text for role in element.findall("Role")]
|
|
2597
|
-
if not office_roles:
|
|
2598
|
-
office_roles = [
|
|
2599
|
-
office_role.strip()
|
|
2600
|
-
for office_role in get_external_id_values(element, "office-role")
|
|
2601
|
-
]
|
|
2602
|
-
|
|
2603
|
-
if not office_roles:
|
|
2604
|
-
raise loggers.ElectionError.from_message(
|
|
2605
|
-
"Office is missing an office role.",
|
|
2606
|
-
[element],
|
|
2607
|
-
)
|
|
2608
|
-
if len(office_roles) == 1:
|
|
2609
|
-
office_role = office_roles[0]
|
|
2610
|
-
if office_role not in office_utils.VALID_OFFICE_ROLES:
|
|
2611
|
-
raise loggers.ElectionError.from_message(
|
|
2612
|
-
f"Office has an invalid office role: '{office_role}'.",
|
|
2613
|
-
[element],
|
|
2614
|
-
)
|
|
2615
|
-
elif len(office_roles) == 2:
|
|
2616
|
-
if set(office_roles) not in _VALID_OFFICE_ROLE_COMBINATIONS:
|
|
2617
|
-
raise loggers.ElectionError.from_message(
|
|
2618
|
-
"Office has an invalid combination of office roles: "
|
|
2619
|
-
f"{office_roles}. Valid combinations are "
|
|
2620
|
-
f"{_VALID_OFFICE_ROLE_COMBINATIONS}.",
|
|
2621
|
-
[element],
|
|
2622
|
-
)
|
|
2623
|
-
else:
|
|
2624
|
-
raise loggers.ElectionError.from_message(
|
|
2625
|
-
"Office has more than two office roles.",
|
|
2626
|
-
[element],
|
|
2627
|
-
)
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
class DiplomaticOfficesHaveCorrespondingGpUnitsSet(base.BaseRule):
|
|
2631
|
-
"""Offices with the 'diplomat' role must have a sending or receiving state."""
|
|
2632
|
-
|
|
2633
|
-
def elements(self):
|
|
2634
|
-
return ["Office"]
|
|
2635
|
-
|
|
2636
|
-
def check(self, element):
|
|
2637
|
-
office_roles = (
|
|
2638
|
-
role.text.strip()
|
|
2639
|
-
for role in element.findall("Role")
|
|
2640
|
-
if element_has_text(role)
|
|
2641
|
-
)
|
|
2642
|
-
if "diplomat" not in office_roles:
|
|
2643
|
-
return
|
|
2644
|
-
|
|
2645
|
-
if not element_has_text(
|
|
2646
|
-
element.find("DiplomatSendingStateId")
|
|
2647
|
-
) and not element_has_text(element.find("DiplomatReceivingStateId")):
|
|
2648
|
-
raise loggers.ElectionError.from_message(
|
|
2649
|
-
"Office with role 'diplomat' must have at least one of"
|
|
2650
|
-
" DiplomatSendingStateId or DiplomatReceivingStateId set.",
|
|
2651
|
-
[element],
|
|
2652
|
-
)
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
1116
|
class ContestHasValidContestStage(base.BaseRule):
|
|
2656
1117
|
"""Each Contest must have a valid contest-stage."""
|
|
2657
1118
|
|
|
@@ -2661,7 +1122,7 @@ class ContestHasValidContestStage(base.BaseRule):
|
|
|
2661
1122
|
def check(self, element):
|
|
2662
1123
|
contest_stage_values = [
|
|
2663
1124
|
contest_stage_value.strip()
|
|
2664
|
-
for contest_stage_value in get_external_id_values(
|
|
1125
|
+
for contest_stage_value in rule_util.get_external_id_values(
|
|
2665
1126
|
element, "contest-stage"
|
|
2666
1127
|
)
|
|
2667
1128
|
]
|
|
@@ -2808,23 +1269,6 @@ class ElectionEndDatesOccurAfterStartDates(base.DateRule):
|
|
|
2808
1269
|
raise loggers.ElectionError(self.error_log)
|
|
2809
1270
|
|
|
2810
1271
|
|
|
2811
|
-
class ValidPartyLeadershipDates(base.DateRule):
|
|
2812
|
-
"""Party Leadership start/end dates should be valid if specified.
|
|
2813
|
-
|
|
2814
|
-
End dates should not occur before the start date.
|
|
2815
|
-
"""
|
|
2816
|
-
|
|
2817
|
-
def elements(self):
|
|
2818
|
-
return ["PartyLeadership"]
|
|
2819
|
-
|
|
2820
|
-
def check(self, element):
|
|
2821
|
-
self.reset_instance_vars()
|
|
2822
|
-
self.gather_dates(element)
|
|
2823
|
-
self.check_end_after_start()
|
|
2824
|
-
if self.error_log:
|
|
2825
|
-
raise loggers.ElectionError(self.error_log)
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
1272
|
class ElectionDatesSpanContestDates(base.DateRule):
|
|
2829
1273
|
"""Election start/end dates should span the Contest start/end dates.
|
|
2830
1274
|
|
|
@@ -2931,7 +1375,7 @@ class ElectionTypesAndCandidateContestTypesAreCompatible(base.BaseRule):
|
|
|
2931
1375
|
return {
|
|
2932
1376
|
element.text.strip().lower()
|
|
2933
1377
|
for element in elements
|
|
2934
|
-
if element_has_text(element)
|
|
1378
|
+
if rule_util.element_has_text(element)
|
|
2935
1379
|
}
|
|
2936
1380
|
|
|
2937
1381
|
def elements(self):
|
|
@@ -2992,13 +1436,13 @@ class DateStatusMatches(base.DateRule):
|
|
|
2992
1436
|
election_date_status = "confirmed" # default value
|
|
2993
1437
|
election_status_elem = election_elem.find("ElectionDateStatus")
|
|
2994
1438
|
|
|
2995
|
-
if element_has_text(election_status_elem):
|
|
1439
|
+
if rule_util.element_has_text(election_status_elem):
|
|
2996
1440
|
election_date_status = election_status_elem.text.strip()
|
|
2997
1441
|
|
|
2998
1442
|
contest_statuses = set()
|
|
2999
1443
|
for contest_elem in self.get_elements_by_class(election_elem, "Contest"):
|
|
3000
1444
|
contest_status_elem = contest_elem.find("ContestDateStatus")
|
|
3001
|
-
if element_has_text(contest_status_elem):
|
|
1445
|
+
if rule_util.element_has_text(contest_status_elem):
|
|
3002
1446
|
contest_statuses.add(contest_status_elem.text.strip())
|
|
3003
1447
|
else:
|
|
3004
1448
|
contest_statuses.add("confirmed")
|
|
@@ -3074,7 +1518,9 @@ class OfficeHolderTenureTermDates(base.DateRule):
|
|
|
3074
1518
|
def check(self, element):
|
|
3075
1519
|
start_date = element.find("StartDate")
|
|
3076
1520
|
end_date = element.find("EndDate")
|
|
3077
|
-
if element_has_text(start_date) and element_has_text(
|
|
1521
|
+
if rule_util.element_has_text(start_date) and rule_util.element_has_text(
|
|
1522
|
+
end_date
|
|
1523
|
+
):
|
|
3078
1524
|
start_date_obj = base.PartialDate.init_partial_date(start_date.text)
|
|
3079
1525
|
end_date_obj = base.PartialDate.init_partial_date(end_date.text)
|
|
3080
1526
|
if end_date_obj < start_date_obj:
|
|
@@ -3118,7 +1564,7 @@ class OfficeTermDates(base.DateRule):
|
|
|
3118
1564
|
raise loggers.ElectionError(self.error_log)
|
|
3119
1565
|
else:
|
|
3120
1566
|
off_per_id = element.find("OfficeHolderPersonIds")
|
|
3121
|
-
if element_has_text(off_per_id):
|
|
1567
|
+
if rule_util.element_has_text(off_per_id):
|
|
3122
1568
|
term = element.find("Term")
|
|
3123
1569
|
if term is None:
|
|
3124
1570
|
raise loggers.ElectionWarning.from_message(
|
|
@@ -3301,15 +1747,17 @@ class UniqueStartDatesForOfficeRoleAndJurisdiction(base.BaseRule):
|
|
|
3301
1747
|
start_date = ""
|
|
3302
1748
|
|
|
3303
1749
|
start_date_elem = office.find(".//Term//StartDate")
|
|
3304
|
-
if not element_has_text(start_date_elem):
|
|
1750
|
+
if not rule_util.element_has_text(start_date_elem):
|
|
3305
1751
|
continue
|
|
3306
1752
|
start_date = start_date_elem.text
|
|
3307
1753
|
|
|
3308
|
-
office_roles = get_entity_info_for_value_type(
|
|
1754
|
+
office_roles = rule_util.get_entity_info_for_value_type(
|
|
1755
|
+
office, "office-role"
|
|
1756
|
+
)
|
|
3309
1757
|
if office_roles:
|
|
3310
1758
|
office_role = office_roles[0]
|
|
3311
1759
|
|
|
3312
|
-
jurisdiction_ids = get_entity_info_for_value_type(
|
|
1760
|
+
jurisdiction_ids = rule_util.get_entity_info_for_value_type(
|
|
3313
1761
|
office, "jurisdiction-id"
|
|
3314
1762
|
)
|
|
3315
1763
|
if jurisdiction_ids:
|
|
@@ -3363,45 +1811,6 @@ class UniqueStartDatesForOfficeRoleAndJurisdiction(base.BaseRule):
|
|
|
3363
1811
|
raise loggers.ElectionWarning(warning_log)
|
|
3364
1812
|
|
|
3365
1813
|
|
|
3366
|
-
class GpUnitsHaveInternationalizedName(base.BaseRule):
|
|
3367
|
-
"""GpUnits must have at least one non-empty InternationlizedName element."""
|
|
3368
|
-
|
|
3369
|
-
def elements(self):
|
|
3370
|
-
return ["GpUnit"]
|
|
3371
|
-
|
|
3372
|
-
def check(self, element):
|
|
3373
|
-
intl_names = element.findall("InternationalizedName")
|
|
3374
|
-
object_id = element.get("objectId", "")
|
|
3375
|
-
if intl_names is None or not intl_names or len(intl_names) > 1:
|
|
3376
|
-
raise loggers.ElectionError.from_message(
|
|
3377
|
-
"GpUnit is required to have exactly one InterationalizedName"
|
|
3378
|
-
" element.",
|
|
3379
|
-
[element],
|
|
3380
|
-
)
|
|
3381
|
-
intl_name = intl_names[0]
|
|
3382
|
-
name_texts = intl_name.findall("Text")
|
|
3383
|
-
if name_texts is None or not name_texts:
|
|
3384
|
-
raise loggers.ElectionError.from_message(
|
|
3385
|
-
(
|
|
3386
|
-
"GpUnit InternationalizedName is required to have one or more"
|
|
3387
|
-
" Text elements."
|
|
3388
|
-
),
|
|
3389
|
-
[intl_name],
|
|
3390
|
-
)
|
|
3391
|
-
error_log = []
|
|
3392
|
-
for name_text in name_texts:
|
|
3393
|
-
if name_text is None or not (name_text.text and name_text.text.strip()):
|
|
3394
|
-
error_log.append(
|
|
3395
|
-
loggers.LogEntry(
|
|
3396
|
-
"GpUnit InternationalizedName does not have a text value.",
|
|
3397
|
-
[name_text],
|
|
3398
|
-
)
|
|
3399
|
-
)
|
|
3400
|
-
|
|
3401
|
-
if error_log:
|
|
3402
|
-
raise loggers.ElectionError(error_log)
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
1814
|
class ValidateInfoUriAnnotation(base.BaseRule):
|
|
3406
1815
|
"""InfoUri is an annotated Uri that accepts the following annotations.
|
|
3407
1816
|
|
|
@@ -3460,11 +1869,13 @@ class FullTextOrBallotText(base.BaseRule):
|
|
|
3460
1869
|
return ["BallotMeasureContest"]
|
|
3461
1870
|
|
|
3462
1871
|
def check(self, element):
|
|
3463
|
-
full_text_map = get_language_to_text_map(element.find("FullText"))
|
|
1872
|
+
full_text_map = rule_util.get_language_to_text_map(element.find("FullText"))
|
|
3464
1873
|
if not full_text_map:
|
|
3465
1874
|
return
|
|
3466
1875
|
|
|
3467
|
-
ballot_text_map = get_language_to_text_map(
|
|
1876
|
+
ballot_text_map = rule_util.get_language_to_text_map(
|
|
1877
|
+
element.find("BallotText")
|
|
1878
|
+
)
|
|
3468
1879
|
for language, full_text_strings in full_text_map.items():
|
|
3469
1880
|
full_text_string = full_text_strings[0]
|
|
3470
1881
|
if (
|
|
@@ -3487,13 +1898,17 @@ class BallotTitle(base.BaseRule):
|
|
|
3487
1898
|
return ["BallotMeasureContest"]
|
|
3488
1899
|
|
|
3489
1900
|
def check(self, element):
|
|
3490
|
-
ballot_title_map = get_language_to_text_map(
|
|
1901
|
+
ballot_title_map = rule_util.get_language_to_text_map(
|
|
1902
|
+
element.find("BallotTitle")
|
|
1903
|
+
)
|
|
3491
1904
|
if not ballot_title_map:
|
|
3492
1905
|
raise loggers.ElectionError.from_message(
|
|
3493
1906
|
"BallotMeasureContest is missing BallotTitle.", [element]
|
|
3494
1907
|
)
|
|
3495
1908
|
|
|
3496
|
-
ballot_text_map = get_language_to_text_map(
|
|
1909
|
+
ballot_text_map = rule_util.get_language_to_text_map(
|
|
1910
|
+
element.find("BallotText")
|
|
1911
|
+
)
|
|
3497
1912
|
if not ballot_text_map:
|
|
3498
1913
|
msg = (
|
|
3499
1914
|
"BallotText is missing. Please confirm that the ballot "
|
|
@@ -3592,7 +2007,7 @@ class WinnerCountLimit(base.BaseRule):
|
|
|
3592
2007
|
for candidate in candidates:
|
|
3593
2008
|
object_id = candidate.get("objectId")
|
|
3594
2009
|
post_election_status = candidate.find("PostElectionStatus")
|
|
3595
|
-
if object_id and element_has_text(post_election_status):
|
|
2010
|
+
if object_id and rule_util.element_has_text(post_election_status):
|
|
3596
2011
|
candidate_post_election_status_by_object_id[object_id] = (
|
|
3597
2012
|
post_election_status.text.strip()
|
|
3598
2013
|
)
|
|
@@ -3605,14 +2020,14 @@ class WinnerCountLimit(base.BaseRule):
|
|
|
3605
2020
|
number_elected_element = element.find("NumberElected")
|
|
3606
2021
|
number_elected = (
|
|
3607
2022
|
int(number_elected_element.text)
|
|
3608
|
-
if element_has_text(number_elected_element)
|
|
2023
|
+
if rule_util.element_has_text(number_elected_element)
|
|
3609
2024
|
else 1
|
|
3610
2025
|
)
|
|
3611
2026
|
|
|
3612
2027
|
candidate_ids_elements = element.findall("BallotSelection//CandidateIds")
|
|
3613
2028
|
candidate_ids = set()
|
|
3614
2029
|
for candidate_ids_element in candidate_ids_elements:
|
|
3615
|
-
if element_has_text(candidate_ids_element):
|
|
2030
|
+
if rule_util.element_has_text(candidate_ids_element):
|
|
3616
2031
|
candidate_ids.update(candidate_ids_element.text.split())
|
|
3617
2032
|
|
|
3618
2033
|
winner_count = 0
|
|
@@ -3632,193 +2047,6 @@ class WinnerCountLimit(base.BaseRule):
|
|
|
3632
2047
|
)
|
|
3633
2048
|
|
|
3634
2049
|
|
|
3635
|
-
class MissingFieldsError(base.MissingFieldRule):
|
|
3636
|
-
"""Check for missing fields for given entity types and field names.
|
|
3637
|
-
|
|
3638
|
-
Raise error for missing fields. To add a field, include the entity
|
|
3639
|
-
and field in element_field_mapping.
|
|
3640
|
-
"""
|
|
3641
|
-
|
|
3642
|
-
def get_severity(self):
|
|
3643
|
-
return 2
|
|
3644
|
-
|
|
3645
|
-
def element_field_mapping(self):
|
|
3646
|
-
return {
|
|
3647
|
-
"Person": [
|
|
3648
|
-
"FullName//Text",
|
|
3649
|
-
],
|
|
3650
|
-
"Candidate": [
|
|
3651
|
-
"PersonId",
|
|
3652
|
-
],
|
|
3653
|
-
"Election": [
|
|
3654
|
-
"StartDate",
|
|
3655
|
-
"EndDate",
|
|
3656
|
-
],
|
|
3657
|
-
"Party": [
|
|
3658
|
-
"PartyScopeGpUnitIds",
|
|
3659
|
-
],
|
|
3660
|
-
}
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
class MissingFieldsWarning(base.MissingFieldRule):
|
|
3664
|
-
"""Check for missing fields for given entity types and field names.
|
|
3665
|
-
|
|
3666
|
-
Raise warning for missing fields. To add a field, include the entity
|
|
3667
|
-
and field in element_field_mapping.
|
|
3668
|
-
"""
|
|
3669
|
-
|
|
3670
|
-
def get_severity(self):
|
|
3671
|
-
return 1
|
|
3672
|
-
|
|
3673
|
-
def element_field_mapping(self):
|
|
3674
|
-
return {
|
|
3675
|
-
"Candidate": [
|
|
3676
|
-
"PartyId",
|
|
3677
|
-
],
|
|
3678
|
-
}
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
class MissingFieldsInfo(base.MissingFieldRule):
|
|
3682
|
-
"""Check for missing fields for given entity types and field names.
|
|
3683
|
-
|
|
3684
|
-
Raise info for missing fields. To add a field, include the entity
|
|
3685
|
-
and field in element_field_mapping.
|
|
3686
|
-
"""
|
|
3687
|
-
|
|
3688
|
-
def get_severity(self):
|
|
3689
|
-
return 0
|
|
3690
|
-
|
|
3691
|
-
def element_field_mapping(self):
|
|
3692
|
-
return {}
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
class PartySpanMultipleCountries(base.BaseRule):
|
|
3696
|
-
"""Check if a party operate on multiple countries.
|
|
3697
|
-
|
|
3698
|
-
Parties can have PartyScopeGpUnitIds which span multiple countries. This is
|
|
3699
|
-
sometimes correct, but we should flag it to double check.
|
|
3700
|
-
"""
|
|
3701
|
-
|
|
3702
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
3703
|
-
super(PartySpanMultipleCountries, self).__init__(
|
|
3704
|
-
election_tree, schema_tree, **kwargs
|
|
3705
|
-
)
|
|
3706
|
-
self.existing_gpunits = dict()
|
|
3707
|
-
country_pattern = re.compile(r"^ocd-division\/country:[a-z]{2}")
|
|
3708
|
-
for gpunit in self.get_elements_by_class(election_tree, "GpUnit"):
|
|
3709
|
-
ocd_ids = get_external_id_values(gpunit, "ocd-id")
|
|
3710
|
-
if ocd_ids:
|
|
3711
|
-
country_match = country_pattern.search(ocd_ids[0])
|
|
3712
|
-
if country_match:
|
|
3713
|
-
country = country_match[0].split("/")[1]
|
|
3714
|
-
self.existing_gpunits[gpunit.get("objectId")] = country
|
|
3715
|
-
|
|
3716
|
-
def elements(self):
|
|
3717
|
-
return ["PartyScopeGpUnitIds"]
|
|
3718
|
-
|
|
3719
|
-
def check(self, element):
|
|
3720
|
-
if element.text is None:
|
|
3721
|
-
return
|
|
3722
|
-
referenced_country = dict()
|
|
3723
|
-
for gpunit_id in element.text.split():
|
|
3724
|
-
country = self.existing_gpunits.get(gpunit_id)
|
|
3725
|
-
if country is not None:
|
|
3726
|
-
if referenced_country.get(country) is None:
|
|
3727
|
-
referenced_country[country] = []
|
|
3728
|
-
referenced_country[country].append(gpunit_id)
|
|
3729
|
-
|
|
3730
|
-
if len(referenced_country) > 1:
|
|
3731
|
-
gpunit_country_mapping = " / ".join([
|
|
3732
|
-
"%s -> %s" % (key, str(value))
|
|
3733
|
-
for (key, value) in referenced_country.items()
|
|
3734
|
-
])
|
|
3735
|
-
|
|
3736
|
-
raise loggers.ElectionWarning.from_message(
|
|
3737
|
-
(
|
|
3738
|
-
"PartyScopeGpUnitIds refer to GpUnit from different countries:"
|
|
3739
|
-
" {}. Please double check.".format(gpunit_country_mapping)
|
|
3740
|
-
),
|
|
3741
|
-
[element],
|
|
3742
|
-
)
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
class NonExecutiveOfficeShouldHaveGovernmentBody(base.BaseRule):
|
|
3746
|
-
"""Ensure non-Head of Government/State Office elements have a government body defined."""
|
|
3747
|
-
|
|
3748
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
3749
|
-
self.is_post_office_split_feed = False
|
|
3750
|
-
officeholder_tenure_collection_element = self.get_elements_by_class(
|
|
3751
|
-
election_tree, "OfficeHolderTenureCollection"
|
|
3752
|
-
)
|
|
3753
|
-
role_element = self.get_elements_by_class(election_tree, "Role")
|
|
3754
|
-
if officeholder_tenure_collection_element or role_element:
|
|
3755
|
-
self.is_post_office_split_feed = True
|
|
3756
|
-
|
|
3757
|
-
def elements(self):
|
|
3758
|
-
return ["Office"]
|
|
3759
|
-
|
|
3760
|
-
def check(self, element):
|
|
3761
|
-
if not _is_executive_office(
|
|
3762
|
-
element, self.is_post_office_split_feed
|
|
3763
|
-
) and not _has_government_body(element):
|
|
3764
|
-
raise loggers.ElectionInfo.from_message(
|
|
3765
|
-
"Non-Head of Government/State Office element is missing a government"
|
|
3766
|
-
" body.",
|
|
3767
|
-
[element],
|
|
3768
|
-
)
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
class ExecutiveOfficeShouldNotHaveGovernmentBody(base.BaseRule):
|
|
3772
|
-
"""Ensure Head of Government/State Office elements do not have a government body defined."""
|
|
3773
|
-
|
|
3774
|
-
def __init__(self, election_tree, schema_tree, **kwargs):
|
|
3775
|
-
self.is_post_office_split_feed = False
|
|
3776
|
-
officeholder_tenure_collection_element = self.get_elements_by_class(
|
|
3777
|
-
election_tree, "OfficeHolderTenureCollection"
|
|
3778
|
-
)
|
|
3779
|
-
role_element = self.get_elements_by_class(election_tree, "Role")
|
|
3780
|
-
if officeholder_tenure_collection_element or role_element:
|
|
3781
|
-
self.is_post_office_split_feed = True
|
|
3782
|
-
|
|
3783
|
-
def elements(self):
|
|
3784
|
-
return ["Office"]
|
|
3785
|
-
|
|
3786
|
-
def check(self, element):
|
|
3787
|
-
if _is_executive_office(
|
|
3788
|
-
element, self.is_post_office_split_feed
|
|
3789
|
-
) and _has_government_body(element):
|
|
3790
|
-
office_roles = _get_office_roles(element, self.is_post_office_split_feed)
|
|
3791
|
-
raise loggers.ElectionError.from_message(
|
|
3792
|
-
"Head of Government/State Office element (roles:"
|
|
3793
|
-
f" {','.join(office_roles)}) has a government body. Head of"
|
|
3794
|
-
" Government/State offices should not have government bodies.",
|
|
3795
|
-
[element],
|
|
3796
|
-
)
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
class MissingOfficeSelectionMethod(base.BaseRule):
|
|
3800
|
-
"""Checks that SelectionMethod is present and .
|
|
3801
|
-
|
|
3802
|
-
Values supported for SelectionMethod:
|
|
3803
|
-
-appointed
|
|
3804
|
-
-directly-elected
|
|
3805
|
-
-hereditary
|
|
3806
|
-
-indirectly-elected
|
|
3807
|
-
-succession
|
|
3808
|
-
-ex-officio
|
|
3809
|
-
"""
|
|
3810
|
-
|
|
3811
|
-
def elements(self):
|
|
3812
|
-
return ["Office"]
|
|
3813
|
-
|
|
3814
|
-
def check(self, element):
|
|
3815
|
-
selection = element.find("SelectionMethod")
|
|
3816
|
-
if selection is None:
|
|
3817
|
-
raise loggers.ElectionWarning.from_message(
|
|
3818
|
-
"Office element is missing its SelectionMethod.", [element]
|
|
3819
|
-
)
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
2050
|
class SubsequentContestIdIsValidRelatedContest(base.DateRule):
|
|
3823
2051
|
"""Check that SubsequentContests are valid.
|
|
3824
2052
|
|
|
@@ -3852,7 +2080,7 @@ class SubsequentContestIdIsValidRelatedContest(base.DateRule):
|
|
|
3852
2080
|
|
|
3853
2081
|
for contest_id, contest in contest_by_id.items():
|
|
3854
2082
|
element = contest.find("SubsequentContestId")
|
|
3855
|
-
if not element_has_text(element):
|
|
2083
|
+
if not rule_util.element_has_text(element):
|
|
3856
2084
|
continue
|
|
3857
2085
|
subsequent_contest_id = element.text.strip()
|
|
3858
2086
|
|
|
@@ -3887,10 +2115,10 @@ class SubsequentContestIdIsValidRelatedContest(base.DateRule):
|
|
|
3887
2115
|
contest_office_id = None
|
|
3888
2116
|
subsequent_contest_office_id = None
|
|
3889
2117
|
element = contest.find("OfficeIds")
|
|
3890
|
-
if element_has_text(element):
|
|
2118
|
+
if rule_util.element_has_text(element):
|
|
3891
2119
|
contest_office_id = element.text
|
|
3892
2120
|
element = subsequent_contest.find("OfficeIds")
|
|
3893
|
-
if element_has_text(element):
|
|
2121
|
+
if rule_util.element_has_text(element):
|
|
3894
2122
|
subsequent_contest_office_id = element.text
|
|
3895
2123
|
|
|
3896
2124
|
if contest_office_id != subsequent_contest_office_id:
|
|
@@ -3908,10 +2136,10 @@ class SubsequentContestIdIsValidRelatedContest(base.DateRule):
|
|
|
3908
2136
|
contest_primary_party_ids = None
|
|
3909
2137
|
subsequent_contest_primary_party_ids = None
|
|
3910
2138
|
element = contest.find("PrimaryPartyIds")
|
|
3911
|
-
if element_has_text(element):
|
|
2139
|
+
if rule_util.element_has_text(element):
|
|
3912
2140
|
contest_primary_party_ids = set(element.text.split())
|
|
3913
2141
|
element = subsequent_contest.find("PrimaryPartyIds")
|
|
3914
|
-
if element_has_text(element):
|
|
2142
|
+
if rule_util.element_has_text(element):
|
|
3915
2143
|
subsequent_contest_primary_party_ids = set(element.text.split())
|
|
3916
2144
|
|
|
3917
2145
|
if (
|
|
@@ -3929,7 +2157,7 @@ class SubsequentContestIdIsValidRelatedContest(base.DateRule):
|
|
|
3929
2157
|
|
|
3930
2158
|
# Check that there is not a subsequent contest <-> composing contest loop
|
|
3931
2159
|
element = subsequent_contest.find("ComposingContestIds")
|
|
3932
|
-
if element_has_text(element):
|
|
2160
|
+
if rule_util.element_has_text(element):
|
|
3933
2161
|
subsequent_composing_ids = element.text.split()
|
|
3934
2162
|
if contest_id in subsequent_composing_ids:
|
|
3935
2163
|
error_log.append(
|
|
@@ -3967,7 +2195,7 @@ class ComposingContestIdsAreValidRelatedContests(base.BaseRule):
|
|
|
3967
2195
|
composing_contests = {}
|
|
3968
2196
|
for contest_id, contest in contest_ids.items():
|
|
3969
2197
|
element = contest.find("ComposingContestIds")
|
|
3970
|
-
if not element_has_text(element):
|
|
2198
|
+
if not rule_util.element_has_text(element):
|
|
3971
2199
|
continue
|
|
3972
2200
|
composing_contests[contest_id] = element.text.split()
|
|
3973
2201
|
|
|
@@ -4004,10 +2232,10 @@ class ComposingContestIdsAreValidRelatedContests(base.BaseRule):
|
|
|
4004
2232
|
c_office_id = None
|
|
4005
2233
|
cc_office_id = None
|
|
4006
2234
|
element = contest.find("OfficeIds")
|
|
4007
|
-
if element_has_text(element):
|
|
2235
|
+
if rule_util.element_has_text(element):
|
|
4008
2236
|
c_office_id = element.text
|
|
4009
2237
|
element = composing_contest.find("OfficeIds")
|
|
4010
|
-
if element_has_text(element):
|
|
2238
|
+
if rule_util.element_has_text(element):
|
|
4011
2239
|
cc_office_id = element.text
|
|
4012
2240
|
|
|
4013
2241
|
if c_office_id != cc_office_id:
|
|
@@ -4022,10 +2250,10 @@ class ComposingContestIdsAreValidRelatedContests(base.BaseRule):
|
|
|
4022
2250
|
c_primary_party_ids = None
|
|
4023
2251
|
cc_primary_party_ids = None
|
|
4024
2252
|
element = contest.find("PrimaryPartyIds")
|
|
4025
|
-
if element_has_text(element):
|
|
2253
|
+
if rule_util.element_has_text(element):
|
|
4026
2254
|
c_primary_party_ids = set(element.text.split())
|
|
4027
2255
|
element = composing_contest.find("PrimaryPartyIds")
|
|
4028
|
-
if element_has_text(element):
|
|
2256
|
+
if rule_util.element_has_text(element):
|
|
4029
2257
|
cc_primary_party_ids = set(element.text.split())
|
|
4030
2258
|
|
|
4031
2259
|
if c_primary_party_ids != cc_primary_party_ids:
|
|
@@ -4059,7 +2287,7 @@ class MultipleInternationalizedTextWithSameLanguageCode(base.BaseRule):
|
|
|
4059
2287
|
return _INTERNATIONALIZED_TEXT_ELEMENTS_WITH_ONLY_ONE_TEXT_PER_LANGUAGE
|
|
4060
2288
|
|
|
4061
2289
|
def check(self, element):
|
|
4062
|
-
language_map = get_language_to_text_map(element)
|
|
2290
|
+
language_map = rule_util.get_language_to_text_map(element)
|
|
4063
2291
|
|
|
4064
2292
|
for language, texts in language_map.items():
|
|
4065
2293
|
if len(texts) > 1:
|
|
@@ -4068,21 +2296,6 @@ class MultipleInternationalizedTextWithSameLanguageCode(base.BaseRule):
|
|
|
4068
2296
|
)
|
|
4069
2297
|
|
|
4070
2298
|
|
|
4071
|
-
class AllInternationalizedTextHaveEnVersion(base.BaseRule):
|
|
4072
|
-
"""Checks for InternationalizedText elements missing an English version."""
|
|
4073
|
-
|
|
4074
|
-
def elements(self):
|
|
4075
|
-
return list(self.get_element_names_for_type("InternationalizedText"))
|
|
4076
|
-
|
|
4077
|
-
def check(self, element):
|
|
4078
|
-
language_map = get_language_to_text_map(element)
|
|
4079
|
-
if "en" not in language_map:
|
|
4080
|
-
raise loggers.ElectionInfo.from_message(
|
|
4081
|
-
message='No "english" version found for the InternationalizedText.',
|
|
4082
|
-
elements=[element],
|
|
4083
|
-
)
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
2299
|
class ContestContainsValidStartDate(base.DateRule):
|
|
4087
2300
|
"""Contest elements should contain valid start dates.
|
|
4088
2301
|
|
|
@@ -4168,7 +2381,7 @@ class ContestEndDateOccursBeforeSubsequentContestStartDate(base.DateRule):
|
|
|
4168
2381
|
for contest_id, contest in contest_by_contest_id.items():
|
|
4169
2382
|
# ignore contests that don't have a subsequent contest
|
|
4170
2383
|
subsequent_element = contest.find("SubsequentContestId")
|
|
4171
|
-
if not element_has_text(subsequent_element):
|
|
2384
|
+
if not rule_util.element_has_text(subsequent_element):
|
|
4172
2385
|
continue
|
|
4173
2386
|
# ignore invalid subsequent contest ids
|
|
4174
2387
|
subsequent_contest_id = subsequent_element.text.strip()
|
|
@@ -4234,9 +2447,9 @@ class ValidatePollsCloseDatetimes(base.BaseRule):
|
|
|
4234
2447
|
earliest_polls_close_element = element.find("EarliestPollsClose")
|
|
4235
2448
|
latest_polls_close_element = element.find("LatestPollsClose")
|
|
4236
2449
|
|
|
4237
|
-
if not element_has_text(
|
|
2450
|
+
if not rule_util.element_has_text(
|
|
4238
2451
|
earliest_polls_close_element
|
|
4239
|
-
) or not element_has_text(latest_polls_close_element):
|
|
2452
|
+
) or not rule_util.element_has_text(latest_polls_close_element):
|
|
4240
2453
|
return
|
|
4241
2454
|
|
|
4242
2455
|
earliest_polls_close_text = earliest_polls_close_element.text.strip()
|
|
@@ -4277,7 +2490,7 @@ class ValidateResultsExpected(base.BaseRule):
|
|
|
4277
2490
|
|
|
4278
2491
|
def check(self, element):
|
|
4279
2492
|
results_expected_element = element.find("ResultsExpected")
|
|
4280
|
-
if not element_has_text(results_expected_element):
|
|
2493
|
+
if not rule_util.element_has_text(results_expected_element):
|
|
4281
2494
|
return
|
|
4282
2495
|
|
|
4283
2496
|
results_expected_text = results_expected_element.text.strip()
|
|
@@ -4300,13 +2513,13 @@ class ValidateResultsExpected(base.BaseRule):
|
|
|
4300
2513
|
for stage in stage_collection.findall("ResultsReportingStage"):
|
|
4301
2514
|
stage_type_element = stage.find("StageType")
|
|
4302
2515
|
if (
|
|
4303
|
-
element_has_text(stage_type_element)
|
|
2516
|
+
rule_util.element_has_text(stage_type_element)
|
|
4304
2517
|
and stage_type_element.text.strip() == "no-results"
|
|
4305
2518
|
):
|
|
4306
2519
|
continue
|
|
4307
2520
|
|
|
4308
2521
|
start_element = stage.find("ExpectedStartDateTime")
|
|
4309
|
-
if not element_has_text(start_element):
|
|
2522
|
+
if not rule_util.element_has_text(start_element):
|
|
4310
2523
|
continue
|
|
4311
2524
|
|
|
4312
2525
|
start_text = start_element.text.strip()
|
|
@@ -4340,7 +2553,7 @@ class ValidateResultsEmbargoEnd(base.BaseRule):
|
|
|
4340
2553
|
|
|
4341
2554
|
def check(self, element):
|
|
4342
2555
|
results_embargo_end_element = element.find("ResultsEmbargoEnd")
|
|
4343
|
-
if not element_has_text(results_embargo_end_element):
|
|
2556
|
+
if not rule_util.element_has_text(results_embargo_end_element):
|
|
4344
2557
|
return
|
|
4345
2558
|
|
|
4346
2559
|
results_embargo_end_text = results_embargo_end_element.text.strip()
|
|
@@ -4365,11 +2578,11 @@ class ValidateResultsEmbargoEnd(base.BaseRule):
|
|
|
4365
2578
|
for stage in stage_collection.findall("ResultsReportingStage"):
|
|
4366
2579
|
stage_type_element = stage.find("StageType")
|
|
4367
2580
|
if (
|
|
4368
|
-
element_has_text(stage_type_element)
|
|
2581
|
+
rule_util.element_has_text(stage_type_element)
|
|
4369
2582
|
and stage_type_element.text.strip() == "official"
|
|
4370
2583
|
):
|
|
4371
2584
|
start_element = stage.find("ExpectedStartDateTime")
|
|
4372
|
-
if not element_has_text(start_element):
|
|
2585
|
+
if not rule_util.element_has_text(start_element):
|
|
4373
2586
|
continue
|
|
4374
2587
|
|
|
4375
2588
|
start_text = start_element.text.strip()
|
|
@@ -4404,7 +2617,7 @@ class ResultsReportingStagesMustHaveUniqueType(base.BaseRule):
|
|
|
4404
2617
|
stages_by_type = collections.defaultdict(list)
|
|
4405
2618
|
for stage in element.findall("ResultsReportingStage"):
|
|
4406
2619
|
stage_type_element = stage.find("StageType")
|
|
4407
|
-
if element_has_text(stage_type_element):
|
|
2620
|
+
if rule_util.element_has_text(stage_type_element):
|
|
4408
2621
|
stage_type = stage_type_element.text.strip()
|
|
4409
2622
|
stages_by_type[stage_type].append(stage)
|
|
4410
2623
|
|
|
@@ -4432,7 +2645,7 @@ class ValidateResultReportingStages(base.BaseRule):
|
|
|
4432
2645
|
composing_contest_ids = set()
|
|
4433
2646
|
for contest in self.get_elements_by_class(self.election_tree, "Contest"):
|
|
4434
2647
|
composing_contest_ids_element = contest.find("ComposingContestIds")
|
|
4435
|
-
if element_has_text(composing_contest_ids_element):
|
|
2648
|
+
if rule_util.element_has_text(composing_contest_ids_element):
|
|
4436
2649
|
composing_contest_ids.update(composing_contest_ids_element.text.split())
|
|
4437
2650
|
return composing_contest_ids
|
|
4438
2651
|
|
|
@@ -4440,7 +2653,7 @@ class ValidateResultReportingStages(base.BaseRule):
|
|
|
4440
2653
|
return ["ResultsReportingStageCollection"]
|
|
4441
2654
|
|
|
4442
2655
|
def check(self, element):
|
|
4443
|
-
inherit_from_composed =
|
|
2656
|
+
inherit_from_composed = rule_util.is_xml_true(
|
|
4444
2657
|
element.find("InheritFromComposedContest")
|
|
4445
2658
|
)
|
|
4446
2659
|
stages = element.findall("ResultsReportingStage")
|
|
@@ -4482,7 +2695,7 @@ class ValidateResultReportingStages(base.BaseRule):
|
|
|
4482
2695
|
stage_types = set()
|
|
4483
2696
|
for stage in stages:
|
|
4484
2697
|
stage_type_element = stage.find("StageType")
|
|
4485
|
-
if element_has_text(stage_type_element):
|
|
2698
|
+
if rule_util.element_has_text(stage_type_element):
|
|
4486
2699
|
stage_types.add(stage_type_element.text.strip())
|
|
4487
2700
|
|
|
4488
2701
|
if (
|
|
@@ -4516,7 +2729,7 @@ class CandidateContestTypesAreCompatible(base.BaseRule):
|
|
|
4516
2729
|
contest_types = {
|
|
4517
2730
|
type_element.text.strip().lower()
|
|
4518
2731
|
for type_element in contest_type_elements
|
|
4519
|
-
if element_has_text(type_element)
|
|
2732
|
+
if rule_util.element_has_text(type_element)
|
|
4520
2733
|
}
|
|
4521
2734
|
if "general" in contest_types and primary_types.intersection(
|
|
4522
2735
|
contest_types
|
|
@@ -4576,7 +2789,7 @@ class EinMatchesFormat(base.BaseRule):
|
|
|
4576
2789
|
def check(self, element):
|
|
4577
2790
|
external_identifiers = element.find("ExternalIdentifiers")
|
|
4578
2791
|
if external_identifiers is not None:
|
|
4579
|
-
ein_ids = get_external_id_values(external_identifiers, "ein")
|
|
2792
|
+
ein_ids = rule_util.get_external_id_values(external_identifiers, "ein")
|
|
4580
2793
|
if ein_ids:
|
|
4581
2794
|
e_id = ein_ids.pop()
|
|
4582
2795
|
if not self.ein_id_matcher.match(e_id):
|
|
@@ -4612,9 +2825,9 @@ class FeedTypeHasValidFeedLongevity(base.BaseRule):
|
|
|
4612
2825
|
def check(self, element):
|
|
4613
2826
|
feed_type_element = element.find("FeedType")
|
|
4614
2827
|
feed_longevity_element = element.find("FeedLongevity")
|
|
4615
|
-
if element_has_text(
|
|
4616
|
-
|
|
4617
|
-
):
|
|
2828
|
+
if rule_util.element_has_text(
|
|
2829
|
+
feed_type_element
|
|
2830
|
+
) and rule_util.element_has_text(feed_longevity_element):
|
|
4618
2831
|
feed_type = feed_type_element.text.lower().replace("_", "-")
|
|
4619
2832
|
feed_longevity = feed_longevity_element.text.lower().replace("_", "-")
|
|
4620
2833
|
if (
|
|
@@ -4643,7 +2856,7 @@ class FeedIdsAreUnique(base.BaseRule):
|
|
|
4643
2856
|
feed_ids = set()
|
|
4644
2857
|
error_log = []
|
|
4645
2858
|
for feed_element in element.findall("Feed"):
|
|
4646
|
-
if element_has_text(feed_element.find("FeedId")):
|
|
2859
|
+
if rule_util.element_has_text(feed_element.find("FeedId")):
|
|
4647
2860
|
feed_id = feed_element.find("FeedId").text
|
|
4648
2861
|
if feed_id in feed_ids:
|
|
4649
2862
|
msg = (
|
|
@@ -4672,7 +2885,7 @@ class SourceDirPathsAreUnique(base.BaseRule):
|
|
|
4672
2885
|
source_dir_paths = set()
|
|
4673
2886
|
error_log = []
|
|
4674
2887
|
for feed_element in element.findall("Feed"):
|
|
4675
|
-
if element_has_text(feed_element.find("SourceDirPath")):
|
|
2888
|
+
if rule_util.element_has_text(feed_element.find("SourceDirPath")):
|
|
4676
2889
|
source_dir_path = feed_element.find("SourceDirPath").text
|
|
4677
2890
|
if source_dir_path in source_dir_paths:
|
|
4678
2891
|
msg = (
|
|
@@ -4705,13 +2918,13 @@ class SqsQueueNameIsFullyQualifiedArn(base.BaseRule):
|
|
|
4705
2918
|
|
|
4706
2919
|
def check(self, element):
|
|
4707
2920
|
sqs_queue_name = element.find("SqsQueueName")
|
|
4708
|
-
if not element_has_text(sqs_queue_name):
|
|
2921
|
+
if not rule_util.element_has_text(sqs_queue_name):
|
|
4709
2922
|
return
|
|
4710
2923
|
queue_name = sqs_queue_name.text.strip()
|
|
4711
2924
|
feed_id_element = element.find("FeedId")
|
|
4712
2925
|
feed_id = (
|
|
4713
2926
|
feed_id_element.text.strip()
|
|
4714
|
-
if element_has_text(feed_id_element)
|
|
2927
|
+
if rule_util.element_has_text(feed_id_element)
|
|
4715
2928
|
else ""
|
|
4716
2929
|
)
|
|
4717
2930
|
|
|
@@ -4754,10 +2967,10 @@ class SqsQueueNameRequiresS3SourceDirPath(base.BaseRule):
|
|
|
4754
2967
|
|
|
4755
2968
|
def check(self, element):
|
|
4756
2969
|
sqs_queue_name = element.find("SqsQueueName")
|
|
4757
|
-
if not element_has_text(sqs_queue_name):
|
|
2970
|
+
if not rule_util.element_has_text(sqs_queue_name):
|
|
4758
2971
|
return
|
|
4759
2972
|
source_dir_path = element.find("SourceDirPath")
|
|
4760
|
-
if not element_has_text(
|
|
2973
|
+
if not rule_util.element_has_text(
|
|
4761
2974
|
source_dir_path
|
|
4762
2975
|
) or not source_dir_path.text.strip().lower().startswith("s3://"):
|
|
4763
2976
|
raise loggers.ElectionError.from_message(
|
|
@@ -4777,7 +2990,10 @@ class ElectionEventDatesAreSequential(base.DateRule):
|
|
|
4777
2990
|
self.reset_instance_vars()
|
|
4778
2991
|
self.gather_dates(element)
|
|
4779
2992
|
self.check_end_after_start()
|
|
4780
|
-
if
|
|
2993
|
+
if (
|
|
2994
|
+
rule_util.element_has_text(element.find("FullDeliveryDate"))
|
|
2995
|
+
and self.start_date
|
|
2996
|
+
):
|
|
4781
2997
|
full_delivery_date = base.PartialDate.init_partial_date(
|
|
4782
2998
|
element.find("FullDeliveryDate").text
|
|
4783
2999
|
)
|
|
@@ -4788,9 +3004,9 @@ class ElectionEventDatesAreSequential(base.DateRule):
|
|
|
4788
3004
|
[element],
|
|
4789
3005
|
)
|
|
4790
3006
|
)
|
|
4791
|
-
if element_has_text(
|
|
3007
|
+
if rule_util.element_has_text(
|
|
4792
3008
|
element.find("InitialDeliveryDate")
|
|
4793
|
-
) and element_has_text(element.find("FullDeliveryDate")):
|
|
3009
|
+
) and rule_util.element_has_text(element.find("FullDeliveryDate")):
|
|
4794
3010
|
initial_delivery_date = base.PartialDate.init_partial_date(
|
|
4795
3011
|
element.find("InitialDeliveryDate").text
|
|
4796
3012
|
)
|
|
@@ -4816,7 +3032,7 @@ class SourceDirPathMustBeSetAfterInitialDeliveryDate(base.BaseRule):
|
|
|
4816
3032
|
return ["Feed"]
|
|
4817
3033
|
|
|
4818
3034
|
def check(self, element):
|
|
4819
|
-
if element_has_text(element.find("SourceDirPath")):
|
|
3035
|
+
if rule_util.element_has_text(element.find("SourceDirPath")):
|
|
4820
3036
|
return
|
|
4821
3037
|
today = datetime.date.today()
|
|
4822
3038
|
today_partial_date = base.PartialDate(
|
|
@@ -4826,7 +3042,7 @@ class SourceDirPathMustBeSetAfterInitialDeliveryDate(base.BaseRule):
|
|
|
4826
3042
|
for initial_delivery in initial_deliveries:
|
|
4827
3043
|
initial_delivery_date = (
|
|
4828
3044
|
base.PartialDate.init_partial_date(initial_delivery.text)
|
|
4829
|
-
if element_has_text(initial_delivery)
|
|
3045
|
+
if rule_util.element_has_text(initial_delivery)
|
|
4830
3046
|
else None
|
|
4831
3047
|
)
|
|
4832
3048
|
if initial_delivery_date and initial_delivery_date < today_partial_date:
|
|
@@ -4844,9 +3060,9 @@ class OfficeholderSubFeedDatesAreSequential(base.DateRule):
|
|
|
4844
3060
|
return ["OfficeholderSubFeed"]
|
|
4845
3061
|
|
|
4846
3062
|
def check(self, element):
|
|
4847
|
-
if element_has_text(
|
|
3063
|
+
if rule_util.element_has_text(
|
|
4848
3064
|
element.find("InitialDeliveryDate")
|
|
4849
|
-
) and element_has_text(element.find("FullDeliveryDate")):
|
|
3065
|
+
) and rule_util.element_has_text(element.find("FullDeliveryDate")):
|
|
4850
3066
|
initial_delivery_date = base.PartialDate.init_partial_date(
|
|
4851
3067
|
element.find("InitialDeliveryDate").text
|
|
4852
3068
|
)
|
|
@@ -4870,10 +3086,10 @@ class FeedInactiveDateIsLatestDate(base.BaseRule):
|
|
|
4870
3086
|
|
|
4871
3087
|
def check(self, element):
|
|
4872
3088
|
# Skip this rule for test feeds.
|
|
4873
|
-
if
|
|
3089
|
+
if rule_util.is_xml_true(element.find("IsTest")):
|
|
4874
3090
|
return
|
|
4875
3091
|
|
|
4876
|
-
if element_has_text(element.find("FeedInactiveDate")):
|
|
3092
|
+
if rule_util.element_has_text(element.find("FeedInactiveDate")):
|
|
4877
3093
|
feed_inactive_date = base.PartialDate.init_partial_date(
|
|
4878
3094
|
element.find("FeedInactiveDate").text
|
|
4879
3095
|
)
|
|
@@ -4887,14 +3103,14 @@ class FeedInactiveDateIsLatestDate(base.BaseRule):
|
|
|
4887
3103
|
[element],
|
|
4888
3104
|
)
|
|
4889
3105
|
for election_event_element in element.iter("ElectionEvent"):
|
|
4890
|
-
if element_has_text(
|
|
3106
|
+
if rule_util.element_has_text(
|
|
4891
3107
|
election_event_element.find("ElectionDateStatus")
|
|
4892
3108
|
) and (
|
|
4893
3109
|
election_event_element.find("ElectionDateStatus").text
|
|
4894
3110
|
in self.ignorable_election_date_statuses
|
|
4895
3111
|
):
|
|
4896
3112
|
continue
|
|
4897
|
-
if element_has_text(election_event_element.find("EndDate")):
|
|
3113
|
+
if rule_util.element_has_text(election_event_element.find("EndDate")):
|
|
4898
3114
|
end_date_element = election_event_element.find("EndDate")
|
|
4899
3115
|
end_date = base.PartialDate.init_partial_date(end_date_element.text)
|
|
4900
3116
|
if feed_inactive_date < end_date:
|
|
@@ -4912,16 +3128,16 @@ class FeedHasValidCountryCode(base.BaseRule):
|
|
|
4912
3128
|
|
|
4913
3129
|
def check(self, element):
|
|
4914
3130
|
country_code_element = element.find("CountryCode")
|
|
4915
|
-
if element_has_text(country_code_element):
|
|
3131
|
+
if rule_util.element_has_text(country_code_element):
|
|
4916
3132
|
country_code = country_code_element.text.upper()
|
|
4917
|
-
if not country_code_is_valid(country_code):
|
|
3133
|
+
if not rule_util.country_code_is_valid(country_code):
|
|
4918
3134
|
raise loggers.ElectionError.from_message(
|
|
4919
3135
|
"Invalid country code {}.".format(country_code),
|
|
4920
3136
|
[element],
|
|
4921
3137
|
)
|
|
4922
3138
|
else:
|
|
4923
3139
|
feed_type = element.find("FeedType")
|
|
4924
|
-
if element_has_text(feed_type) and (
|
|
3140
|
+
if rule_util.element_has_text(feed_type) and (
|
|
4925
3141
|
feed_type.text.lower().replace("_", "-") == "election-dates"
|
|
4926
3142
|
or feed_type.text.lower().replace("_", "-") == "voter-information"
|
|
4927
3143
|
):
|
|
@@ -4940,7 +3156,7 @@ class FeedInactiveDateNotOlderThanOneYear(base.BaseRule):
|
|
|
4940
3156
|
|
|
4941
3157
|
def check(self, element):
|
|
4942
3158
|
feed_inactive_date_element = element.find("FeedInactiveDate")
|
|
4943
|
-
if not element_has_text(feed_inactive_date_element):
|
|
3159
|
+
if not rule_util.element_has_text(feed_inactive_date_element):
|
|
4944
3160
|
return
|
|
4945
3161
|
feed_inactive_date = datetime.date.fromisoformat(
|
|
4946
3162
|
feed_inactive_date_element.text
|
|
@@ -4966,9 +3182,9 @@ class FeedInactiveDateSetForNonEvergreenFeed(base.BaseRule):
|
|
|
4966
3182
|
def check(self, element):
|
|
4967
3183
|
feed_longevity = element.find("FeedLongevity")
|
|
4968
3184
|
if (
|
|
4969
|
-
element_has_text(feed_longevity)
|
|
3185
|
+
rule_util.element_has_text(feed_longevity)
|
|
4970
3186
|
and feed_longevity.text.lower() != "evergreen"
|
|
4971
|
-
and not element_has_text(element.find("FeedInactiveDate"))
|
|
3187
|
+
and not rule_util.element_has_text(element.find("FeedInactiveDate"))
|
|
4972
3188
|
):
|
|
4973
3189
|
raise loggers.ElectionError.from_message(
|
|
4974
3190
|
"FeedInactiveDate is not set for non-evergreen feed with FeedId {}."
|
|
@@ -4977,81 +3193,7 @@ class FeedInactiveDateSetForNonEvergreenFeed(base.BaseRule):
|
|
|
4977
3193
|
)
|
|
4978
3194
|
|
|
4979
3195
|
|
|
4980
|
-
class
|
|
4981
|
-
"""All non-top-level entities in a feed should be referenced by at least one other entity.
|
|
4982
|
-
|
|
4983
|
-
In the context of this rule, top-level means that an entity is not expected to
|
|
4984
|
-
be referenced by anything else in the feed. This base class allows derived
|
|
4985
|
-
rules to specify the set of top-level and warning-level entities since these
|
|
4986
|
-
differ by feed type. The rule is an info for gpunits (as long as they have
|
|
4987
|
-
ComposingGpunitIds) since top-level gpunits may exist solely to contain
|
|
4988
|
-
others.
|
|
4989
|
-
"""
|
|
4990
|
-
|
|
4991
|
-
def __init__(
|
|
4992
|
-
self,
|
|
4993
|
-
election_tree,
|
|
4994
|
-
schema_tree,
|
|
4995
|
-
top_level_entity_types,
|
|
4996
|
-
warned_entity_types,
|
|
4997
|
-
**kwargs,
|
|
4998
|
-
):
|
|
4999
|
-
super(UnreferencedEntitiesBase, self).__init__(
|
|
5000
|
-
election_tree, schema_tree, **kwargs
|
|
5001
|
-
)
|
|
5002
|
-
self.referenced_entities = self._gather_referenced_entities()
|
|
5003
|
-
self.top_level_entity_types = top_level_entity_types
|
|
5004
|
-
self.warned_entity_types = warned_entity_types
|
|
5005
|
-
|
|
5006
|
-
def _gather_referenced_entities(self):
|
|
5007
|
-
"""Create a set of all entities referenced by either IDREF(S) elements or external identifiers."""
|
|
5008
|
-
referenced_entities = set()
|
|
5009
|
-
for external_id_type in _IDREF_EXTERNAL_IDENTIFIERS:
|
|
5010
|
-
referenced_entities.update(
|
|
5011
|
-
get_external_id_values(self.election_tree, external_id_type)
|
|
5012
|
-
)
|
|
5013
|
-
idref_elements = self.get_element_names_for_types(_IDREF_TYPES)
|
|
5014
|
-
for _, element in etree.iterwalk(self.election_tree):
|
|
5015
|
-
tag = self.strip_schema_ns(element)
|
|
5016
|
-
if tag and tag in idref_elements:
|
|
5017
|
-
referenced_entities.update(element.text.split())
|
|
5018
|
-
return referenced_entities
|
|
5019
|
-
|
|
5020
|
-
def check(self):
|
|
5021
|
-
for _, element in etree.iterwalk(self.election_tree):
|
|
5022
|
-
element_name = element.tag
|
|
5023
|
-
# Skip anything without an object id.
|
|
5024
|
-
if "objectId" not in element.attrib:
|
|
5025
|
-
continue
|
|
5026
|
-
obj_id = element.get("objectId")
|
|
5027
|
-
if (
|
|
5028
|
-
obj_id not in self.referenced_entities
|
|
5029
|
-
and element_name not in self.top_level_entity_types
|
|
5030
|
-
):
|
|
5031
|
-
if element_name in self.warned_entity_types:
|
|
5032
|
-
raise loggers.ElectionWarning.from_message(
|
|
5033
|
-
f"Element of type {element_name} with object id {obj_id} is not"
|
|
5034
|
-
" referenced by anything else in the feed. This is only ok if"
|
|
5035
|
-
" there are explicit instructions to include this entity anyways."
|
|
5036
|
-
)
|
|
5037
|
-
elif (
|
|
5038
|
-
element_name == "GpUnit"
|
|
5039
|
-
and element.find("ComposingGpUnitIds") is not None
|
|
5040
|
-
):
|
|
5041
|
-
raise loggers.ElectionInfo.from_message(
|
|
5042
|
-
f"GpUnit with object id {obj_id} is not referenced by anything"
|
|
5043
|
-
" else in the feed. This is ok for top-level GpUnits that"
|
|
5044
|
-
" contain others; please ensure this GpUnit is still required in"
|
|
5045
|
-
" the feed."
|
|
5046
|
-
)
|
|
5047
|
-
else:
|
|
5048
|
-
raise loggers.ElectionError.from_message(
|
|
5049
|
-
f"Element of type {element_name} with object id {obj_id} is not"
|
|
5050
|
-
" referenced by anything else in the feed."
|
|
5051
|
-
)
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
class UnreferencedEntitiesElectionDates(UnreferencedEntitiesBase):
|
|
3196
|
+
class UnreferencedEntitiesElectionDates(common_rules.UnreferencedEntitiesBase):
|
|
5055
3197
|
"""CDF elections and contests are top-level in election dates feeds.
|
|
5056
3198
|
|
|
5057
3199
|
All other entity types must be referenced.
|
|
@@ -5067,7 +3209,7 @@ class UnreferencedEntitiesElectionDates(UnreferencedEntitiesBase):
|
|
|
5067
3209
|
)
|
|
5068
3210
|
|
|
5069
3211
|
|
|
5070
|
-
class UnreferencedEntitiesOfficeholders(UnreferencedEntitiesBase):
|
|
3212
|
+
class UnreferencedEntitiesOfficeholders(common_rules.UnreferencedEntitiesBase):
|
|
5071
3213
|
"""CDF offices and party leadership entities are top-level in officeholders feeds.
|
|
5072
3214
|
|
|
5073
3215
|
This rule is a warning for CDF parties since we ask for all parties for some
|
|
@@ -5094,22 +3236,6 @@ class UnreferencedEntitiesOfficeholders(UnreferencedEntitiesBase):
|
|
|
5094
3236
|
)
|
|
5095
3237
|
|
|
5096
3238
|
|
|
5097
|
-
class DeprecatedPartyLeadershipSchema(base.BaseRule):
|
|
5098
|
-
"""Errors if the deprecated party leadership schema is used."""
|
|
5099
|
-
|
|
5100
|
-
def elements(self):
|
|
5101
|
-
return ["Party"]
|
|
5102
|
-
|
|
5103
|
-
def check(self, element):
|
|
5104
|
-
if len(get_external_id_values(element, "party-leader-id")) or len(
|
|
5105
|
-
get_external_id_values(element, "party-chair-id")
|
|
5106
|
-
):
|
|
5107
|
-
raise loggers.ElectionError.from_message(
|
|
5108
|
-
"Specifying party leadership via external identifiers is deprecated."
|
|
5109
|
-
" Please use the PartyLeadership element instead."
|
|
5110
|
-
)
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
3239
|
class ElectoralCommissionCollectionExists(base.BaseRule):
|
|
5114
3240
|
"""ElectoralCommissionCollection should exist."""
|
|
5115
3241
|
|
|
@@ -5238,7 +3364,7 @@ class FeedElementsShouldHaveSubElementsBasedOnType(base.BaseRule):
|
|
|
5238
3364
|
feed_type = element.find("FeedType")
|
|
5239
3365
|
feed_id = element.find("FeedId").text
|
|
5240
3366
|
|
|
5241
|
-
if not element_has_text(feed_type):
|
|
3367
|
+
if not rule_util.element_has_text(feed_type):
|
|
5242
3368
|
return
|
|
5243
3369
|
|
|
5244
3370
|
feed_type = feed_type.text
|
|
@@ -5274,7 +3400,7 @@ class NotEmptyUniqueDataSourceUris(base.BaseRule):
|
|
|
5274
3400
|
for data_source in element.findall("DataSource"):
|
|
5275
3401
|
datasource_id = data_source.get("objectId")
|
|
5276
3402
|
for uri_element in data_source.findall("Uri"):
|
|
5277
|
-
if not element_has_text(uri_element):
|
|
3403
|
+
if not rule_util.element_has_text(uri_element):
|
|
5278
3404
|
error_log.append(
|
|
5279
3405
|
loggers.LogEntry(
|
|
5280
3406
|
"DataSource {} has an empty Uri.".format(datasource_id),
|
|
@@ -5361,7 +3487,7 @@ class UniqueDataSourceDisplayNames(base.BaseRule):
|
|
|
5361
3487
|
data_source_id = data_source.get("objectId")
|
|
5362
3488
|
display_name_element = data_source.find("DisplayName")
|
|
5363
3489
|
for text_element in display_name_element.findall("Text"):
|
|
5364
|
-
if not element_has_text(text_element):
|
|
3490
|
+
if not rule_util.element_has_text(text_element):
|
|
5365
3491
|
error_log.append(
|
|
5366
3492
|
loggers.LogEntry(
|
|
5367
3493
|
"DataSource {} has a DisplayName element without"
|
|
@@ -5404,7 +3530,9 @@ def _get_attribution_depth(element):
|
|
|
5404
3530
|
def _get_datasource_id(element):
|
|
5405
3531
|
"""Helper to safely extract and strip the text of the DataSourceId child."""
|
|
5406
3532
|
ds_element = element.find("DataSourceId")
|
|
5407
|
-
return
|
|
3533
|
+
return (
|
|
3534
|
+
ds_element.text.strip() if rule_util.element_has_text(ds_element) else ""
|
|
3535
|
+
)
|
|
5408
3536
|
|
|
5409
3537
|
|
|
5410
3538
|
class AttributionDepthLimit(base.BaseRule):
|
|
@@ -5524,7 +3652,7 @@ class ValidateSpecialBallotSelectionCountedInTotal(base.BaseRule):
|
|
|
5524
3652
|
"BlankBallotSelection",
|
|
5525
3653
|
"NullBallotSelection",
|
|
5526
3654
|
"NoneOfTheAboveBallotSelection",
|
|
5527
|
-
) and not element_has_text(counted_in_total):
|
|
3655
|
+
) and not rule_util.element_has_text(counted_in_total):
|
|
5528
3656
|
raise loggers.ElectionError.from_message(
|
|
5529
3657
|
f"{tag} must have an explicit value for CountedInTotal.",
|
|
5530
3658
|
[element],
|
|
@@ -5558,7 +3686,7 @@ class ValidateIncludeInAggregationBallotSelections(base.BaseRule):
|
|
|
5558
3686
|
|
|
5559
3687
|
for vote_counts in vote_counts_collection.findall("VoteCounts"):
|
|
5560
3688
|
count_element = vote_counts.find("Count")
|
|
5561
|
-
if not element_has_text(count_element):
|
|
3689
|
+
if not rule_util.element_has_text(count_element):
|
|
5562
3690
|
continue
|
|
5563
3691
|
count = float(count_element.text)
|
|
5564
3692
|
|
|
@@ -5566,7 +3694,7 @@ class ValidateIncludeInAggregationBallotSelections(base.BaseRule):
|
|
|
5566
3694
|
gp_unit_id_element = vote_counts.find("GpUnitId")
|
|
5567
3695
|
gp_unit_id = (
|
|
5568
3696
|
gp_unit_id_element.text.strip()
|
|
5569
|
-
if element_has_text(gp_unit_id_element)
|
|
3697
|
+
if rule_util.element_has_text(gp_unit_id_element)
|
|
5570
3698
|
else ""
|
|
5571
3699
|
)
|
|
5572
3700
|
|
|
@@ -5586,7 +3714,7 @@ class ValidateIncludeInAggregationBallotSelections(base.BaseRule):
|
|
|
5586
3714
|
|
|
5587
3715
|
included_selections = []
|
|
5588
3716
|
for selection in all_selections:
|
|
5589
|
-
if
|
|
3717
|
+
if rule_util.is_xml_true(selection.find("IncludedInAggregation")):
|
|
5590
3718
|
included_selections.append(selection)
|
|
5591
3719
|
if not included_selections:
|
|
5592
3720
|
return
|
|
@@ -5636,61 +3764,7 @@ class ValidateIncludeInAggregationBallotSelections(base.BaseRule):
|
|
|
5636
3764
|
|
|
5637
3765
|
# To add new rules, create a new class, inherit the base rule,
|
|
5638
3766
|
# and add it to the correct rule list.
|
|
5639
|
-
COMMON_RULES =
|
|
5640
|
-
# go/keep-sorted start
|
|
5641
|
-
AllCaps,
|
|
5642
|
-
AllInternationalizedTextHaveEnVersion,
|
|
5643
|
-
AllLanguages,
|
|
5644
|
-
BadCharactersInPersonFullName,
|
|
5645
|
-
CandidateImageInContactInformation,
|
|
5646
|
-
DeprecatedPartyLeadershipSchema,
|
|
5647
|
-
DiplomaticOfficesHaveCorrespondingGpUnitsSet,
|
|
5648
|
-
DuplicateGpUnits,
|
|
5649
|
-
DuplicateID,
|
|
5650
|
-
DuplicatedGpUnitOcdId,
|
|
5651
|
-
EmptyString,
|
|
5652
|
-
EmptyText,
|
|
5653
|
-
Encoding,
|
|
5654
|
-
ExecutiveOfficeShouldNotHaveGovernmentBody,
|
|
5655
|
-
GpUnitOcdId,
|
|
5656
|
-
GpUnitsCyclesRefsValidation,
|
|
5657
|
-
GpUnitsHaveInternationalizedName,
|
|
5658
|
-
IndependentPartyName,
|
|
5659
|
-
LanguageCode,
|
|
5660
|
-
MissingFieldsError,
|
|
5661
|
-
MissingFieldsInfo,
|
|
5662
|
-
MissingFieldsWarning,
|
|
5663
|
-
MissingOfficeSelectionMethod,
|
|
5664
|
-
MissingStableIds,
|
|
5665
|
-
NonExecutiveOfficeShouldHaveGovernmentBody,
|
|
5666
|
-
OfficesHaveJurisdictionID,
|
|
5667
|
-
OfficesHaveValidOfficeLevel,
|
|
5668
|
-
OfficesHaveValidOfficeRole,
|
|
5669
|
-
OnlyOneCandidateImagePerPerson,
|
|
5670
|
-
OptionalAndEmpty,
|
|
5671
|
-
OtherType,
|
|
5672
|
-
PartyLeadershipMustExist,
|
|
5673
|
-
PartySpanMultipleCountries,
|
|
5674
|
-
PersonHasUniqueFullName,
|
|
5675
|
-
PersonsHaveValidGender,
|
|
5676
|
-
PersonsMissingPartyData,
|
|
5677
|
-
Schema,
|
|
5678
|
-
URIValidator,
|
|
5679
|
-
UniqueCandidateImageUris,
|
|
5680
|
-
UniqueLabel,
|
|
5681
|
-
UniqueStableID,
|
|
5682
|
-
UniqueURIPerAnnotationCategory,
|
|
5683
|
-
ValidEnumerations,
|
|
5684
|
-
ValidIDREF,
|
|
5685
|
-
ValidJurisdictionID,
|
|
5686
|
-
ValidPartyLeadershipDates,
|
|
5687
|
-
ValidStableID,
|
|
5688
|
-
ValidTiktokURL,
|
|
5689
|
-
ValidURIAnnotation,
|
|
5690
|
-
ValidYoutubeURL,
|
|
5691
|
-
ValidateOcdidLowerCase,
|
|
5692
|
-
# go/keep-sorted end
|
|
5693
|
-
)
|
|
3767
|
+
COMMON_RULES = common_rules.COMMON_RULES
|
|
5694
3768
|
|
|
5695
3769
|
ELECTION_RULES = COMMON_RULES + (
|
|
5696
3770
|
# go/keep-sorted start
|
|
@@ -5763,10 +3837,10 @@ OFFICEHOLDER_RULES = COMMON_RULES + (
|
|
|
5763
3837
|
OfficeSelectionMethodMatch,
|
|
5764
3838
|
OfficeTermDates,
|
|
5765
3839
|
PersonHasOffice,
|
|
5766
|
-
ProhibitElectionData,
|
|
5767
3840
|
RemovePersonAndOfficeHolderId60DaysAfterEndDate,
|
|
5768
3841
|
UniqueStartDatesForOfficeRoleAndJurisdiction,
|
|
5769
3842
|
UnreferencedEntitiesOfficeholders,
|
|
3843
|
+
common_rules.ProhibitElectionData,
|
|
5770
3844
|
# go/keep-sorted end
|
|
5771
3845
|
)
|
|
5772
3846
|
|
|
@@ -5776,7 +3850,7 @@ COMMITTEE_RULES = COMMON_RULES + (
|
|
|
5776
3850
|
AffiliationHasEitherPartyOrPerson,
|
|
5777
3851
|
CommitteeClassificationEndDateOccursAfterStartDate,
|
|
5778
3852
|
EinMatchesFormat,
|
|
5779
|
-
ProhibitElectionData,
|
|
3853
|
+
common_rules.ProhibitElectionData,
|
|
5780
3854
|
# go/keep-sorted end
|
|
5781
3855
|
)
|
|
5782
3856
|
|
|
@@ -5785,9 +3859,6 @@ ELECTION_DATES_RULES = ELECTION_RULES + (UnreferencedEntitiesElectionDates,)
|
|
|
5785
3859
|
METADATA_RULES = (
|
|
5786
3860
|
# go/keep-sorted start
|
|
5787
3861
|
ElectionEventDatesAreSequential,
|
|
5788
|
-
EmptyString,
|
|
5789
|
-
EmptyText,
|
|
5790
|
-
Encoding,
|
|
5791
3862
|
FeedElementsShouldHaveSubElementsBasedOnType,
|
|
5792
3863
|
FeedHasValidCountryCode,
|
|
5793
3864
|
FeedIdsAreUnique,
|
|
@@ -5796,14 +3867,17 @@ METADATA_RULES = (
|
|
|
5796
3867
|
FeedInactiveDateSetForNonEvergreenFeed,
|
|
5797
3868
|
FeedTypeHasValidFeedLongevity,
|
|
5798
3869
|
OfficeholderSubFeedDatesAreSequential,
|
|
5799
|
-
OptionalAndEmpty,
|
|
5800
|
-
Schema,
|
|
5801
3870
|
SourceDirPathMustBeSetAfterInitialDeliveryDate,
|
|
5802
3871
|
SourceDirPathsAreUnique,
|
|
5803
3872
|
# TODO(b/555757644): Reenable this rule once we are ready to enforce it.
|
|
5804
3873
|
# SqsQueueNameIsFullyQualifiedArn,
|
|
5805
3874
|
SqsQueueNameRequiresS3SourceDirPath,
|
|
5806
|
-
|
|
3875
|
+
common_rules.EmptyString,
|
|
3876
|
+
common_rules.EmptyText,
|
|
3877
|
+
common_rules.Encoding,
|
|
3878
|
+
common_rules.OptionalAndEmpty,
|
|
3879
|
+
common_rules.Schema,
|
|
3880
|
+
common_rules.UniqueLabel,
|
|
5807
3881
|
# go/keep-sorted end
|
|
5808
3882
|
)
|
|
5809
3883
|
|