py-epc-uk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. py_epc/__init__.py +60 -0
  2. py_epc/_config.py +375 -0
  3. py_epc/_dataset/__init__.py +1 -0
  4. py_epc/_dataset/_air_tightness.py +132 -0
  5. py_epc/_dataset/_bilingual.py +130 -0
  6. py_epc/_dataset/_build.py +221 -0
  7. py_epc/_dataset/_categories.py +237 -0
  8. py_epc/_dataset/_construction_age.py +158 -0
  9. py_epc/_dataset/_definition.py +227 -0
  10. py_epc/_dataset/_definitions/epc-england-and-wales-domestic.json +126 -0
  11. py_epc/_dataset/_definitions/epc-scotland-domestic.json +206 -0
  12. py_epc/_dataset/_distinct.py +176 -0
  13. py_epc/_dataset/_fabric.py +222 -0
  14. py_epc/_dataset/_floor_level.py +112 -0
  15. py_epc/_dataset/_lighting.py +134 -0
  16. py_epc/_dataset/_main_fuel.py +105 -0
  17. py_epc/_dataset/_models.py +13 -0
  18. py_epc/_dataset/_parts.py +123 -0
  19. py_epc/_dataset/_photo_supply.py +221 -0
  20. py_epc/_dataset/_phrases.py +272 -0
  21. py_epc/_dataset/_prepare.py +86 -0
  22. py_epc/_dataset/_summary.py +127 -0
  23. py_epc/_dataset/_systems.py +249 -0
  24. py_epc/_dataset/_text.py +256 -0
  25. py_epc/_dataset/_vocabularies/alternative_improvements.json +32 -0
  26. py_epc/_dataset/_vocabularies/built_form.json +33 -0
  27. py_epc/_dataset/_vocabularies/energy_tariff.json +33 -0
  28. py_epc/_dataset/_vocabularies/floor.json +59 -0
  29. py_epc/_dataset/_vocabularies/glazed_area.json +29 -0
  30. py_epc/_dataset/_vocabularies/glazed_type.json +41 -0
  31. py_epc/_dataset/_vocabularies/heat_loss_corridor.json +24 -0
  32. py_epc/_dataset/_vocabularies/hotwater.json +269 -0
  33. py_epc/_dataset/_vocabularies/lzc_energy_sources.json +72 -0
  34. py_epc/_dataset/_vocabularies/main_fuel.json +153 -0
  35. py_epc/_dataset/_vocabularies/main_heating_category.json +53 -0
  36. py_epc/_dataset/_vocabularies/mainheat.json +667 -0
  37. py_epc/_dataset/_vocabularies/mainheatcont.json +296 -0
  38. py_epc/_dataset/_vocabularies/mechanical_ventilation.json +44 -0
  39. py_epc/_dataset/_vocabularies/property_type.json +28 -0
  40. py_epc/_dataset/_vocabularies/report_type.json +16 -0
  41. py_epc/_dataset/_vocabularies/roof.json +117 -0
  42. py_epc/_dataset/_vocabularies/secondheat.json +291 -0
  43. py_epc/_dataset/_vocabularies/tenure.json +21 -0
  44. py_epc/_dataset/_vocabularies/transaction_type.json +77 -0
  45. py_epc/_dataset/_vocabularies/type_of_assessment.json +16 -0
  46. py_epc/_dataset/_vocabularies/walls.json +132 -0
  47. py_epc/_dataset/_vocabularies/windows.json +132 -0
  48. py_epc/_dataset/_vocabulary.py +157 -0
  49. py_epc/_download.py +305 -0
  50. py_epc/_enums.py +109 -0
  51. py_epc/_files.py +60 -0
  52. py_epc/_manifest.py +121 -0
  53. py_epc/_output.py +140 -0
  54. py_epc/_pipeline.py +426 -0
  55. py_epc/_process/__init__.py +1 -0
  56. py_epc/_process/_archive.py +239 -0
  57. py_epc/_process/_build.py +348 -0
  58. py_epc/_process/_latest.py +202 -0
  59. py_epc/_process/_postcodes.py +503 -0
  60. py_epc/_process/_register.py +258 -0
  61. py_epc/_process/_registers/epc-england-and-wales-domestic.json +600 -0
  62. py_epc/_process/_registers/epc-scotland-domestic.json +690 -0
  63. py_epc/_process/_registers/ons-postcode-directory.json +157 -0
  64. py_epc/_process/_values.py +119 -0
  65. py_epc/_scan.py +286 -0
  66. py_epc/_source.py +30 -0
  67. py_epc/defaults.json +59 -0
  68. py_epc/py.typed +0 -0
  69. py_epc_uk-0.1.0.dist-info/METADATA +193 -0
  70. py_epc_uk-0.1.0.dist-info/RECORD +73 -0
  71. py_epc_uk-0.1.0.dist-info/WHEEL +4 -0
  72. py_epc_uk-0.1.0.dist-info/licenses/ATTRIBUTION.md +46 -0
  73. py_epc_uk-0.1.0.dist-info/licenses/LICENSE +373 -0
py_epc/__init__.py ADDED
@@ -0,0 +1,60 @@
1
+ """Py-EPC library to download & process Energy Performance of Building data."""
2
+
3
+ from py_epc._config import Config
4
+ from py_epc._dataset._build import (
5
+ DatasetManifest,
6
+ build_dataset,
7
+ build_epc_dataset,
8
+ )
9
+ from py_epc._download import (
10
+ DownloadError,
11
+ download,
12
+ download_epc,
13
+ download_ons_postcode_directory,
14
+ download_os_open_uprn,
15
+ )
16
+ from py_epc._enums import EPCOrigin, EPCSegment, ONSProduct, OSProduct
17
+ from py_epc._manifest import DownloadManifest
18
+ from py_epc._output import OutputType
19
+ from py_epc._pipeline import OutputManifest, run_epc_pipeline
20
+ from py_epc._process._build import (
21
+ ProcessManifest,
22
+ ProcessWarning,
23
+ process,
24
+ process_epc,
25
+ )
26
+ from py_epc._process._postcodes import (
27
+ PostcodeManifest,
28
+ process_ons_postcode_directory,
29
+ )
30
+ from py_epc._scan import scan, scan_epc
31
+ from py_epc._source import Source
32
+
33
+ __all__ = [
34
+ "Config",
35
+ "DatasetManifest",
36
+ "DownloadError",
37
+ "DownloadManifest",
38
+ "EPCOrigin",
39
+ "EPCSegment",
40
+ "ONSProduct",
41
+ "OSProduct",
42
+ "OutputManifest",
43
+ "OutputType",
44
+ "PostcodeManifest",
45
+ "ProcessManifest",
46
+ "ProcessWarning",
47
+ "Source",
48
+ "build_dataset",
49
+ "build_epc_dataset",
50
+ "download",
51
+ "download_epc",
52
+ "download_ons_postcode_directory",
53
+ "download_os_open_uprn",
54
+ "process",
55
+ "process_epc",
56
+ "process_ons_postcode_directory",
57
+ "run_epc_pipeline",
58
+ "scan",
59
+ "scan_epc",
60
+ ]
py_epc/_config.py ADDED
@@ -0,0 +1,375 @@
1
+ """Configurations for the Py-EPC library."""
2
+
3
+ import copy
4
+ import json
5
+ import os
6
+ from contextlib import ContextDecorator
7
+ from importlib.resources import files
8
+ from importlib.resources.abc import Traversable
9
+ from pathlib import Path
10
+ from typing import Any, Final, Literal, Self
11
+
12
+ from pydantic import BaseModel, ConfigDict, Field
13
+
14
+ from py_epc._enums import EPCOrigin, EPCSegment, ONSProduct, OSProduct
15
+ from py_epc._source import Source
16
+
17
+ _DEFAULT_CONFIG_PATH: Final[Traversable] = files("py_epc") / "defaults.json"
18
+
19
+ # Environment variable naming a saved config to load on first use.
20
+ _CONFIG_ENV_VAR: Final = "PY_EPC_CONFIG"
21
+
22
+
23
+ class _Geography(BaseModel):
24
+ """Settings for the areas added to each postcode."""
25
+
26
+ model_config = ConfigDict(
27
+ frozen=True,
28
+ extra="forbid",
29
+ strict=True,
30
+ hide_input_in_errors=True,
31
+ use_attribute_docstrings=True,
32
+ )
33
+
34
+ fields: tuple[str, ...] = ()
35
+ """Fields of the ONS Postcode Directory to keep, such as ``"lad"``."""
36
+
37
+ names: bool = True
38
+ """Whether to add each field's names beside its codes."""
39
+
40
+
41
+ class _ConfigContent(BaseModel):
42
+ """The checked contents of the config."""
43
+
44
+ model_config = ConfigDict(
45
+ title="Config",
46
+ frozen=True,
47
+ extra="forbid",
48
+ strict=True,
49
+ hide_input_in_errors=True,
50
+ use_attribute_docstrings=True,
51
+ )
52
+ version: Literal[1]
53
+ """Version of the config format."""
54
+
55
+ epc: dict[EPCOrigin, dict[EPCSegment, Source]] = Field(default_factory=dict)
56
+ """EPC sources, keyed by origin, then by segment."""
57
+
58
+ ons: dict[ONSProduct, Source] = Field(default_factory=dict)
59
+ """Sources of the Office for National Statistics, keyed by product."""
60
+
61
+ os: dict[OSProduct, Source] = Field(default_factory=dict)
62
+ """Sources of Ordnance Survey, keyed by product."""
63
+
64
+ geography: _Geography = Field(default_factory=_Geography)
65
+ """Areas added to each postcode from the ONS Postcode Directory."""
66
+
67
+
68
+ _current_config: _ConfigContent | None = None
69
+
70
+
71
+ class Config(ContextDecorator):
72
+ """
73
+ The configuration settings for the Py-EPC library.
74
+
75
+ The defaults are shipped with the package. Change them with the `set`,
76
+ `reset` and `load` methods. Every change is checked against the config's
77
+ schema, and a rejected change leaves the config as it was. Errors never
78
+ show the values found, so private tokens used to fetch data cannot leak.
79
+
80
+ `save` writes only the settings that differ from the defaults, so fixes to
81
+ the defaults in later releases still apply to everything you have not
82
+ changed. If the ``PY_EPC_CONFIG`` environment variable names a saved
83
+ config, it is loaded on first use.
84
+
85
+ When used as a context manager or a decorator, `Config` undoes every change
86
+ made inside it on exit. The config is shared by every thread.
87
+
88
+ Examples
89
+ --------
90
+ >>> from py_epc import Config
91
+ >>> Config.get("epc.scotland.domestic.publisher")
92
+ 'Scottish Government'
93
+
94
+ Change a setting:
95
+
96
+ >>> Config.set("epc.scotland.domestic.url", "https://example.org/a.csv")
97
+
98
+ Save the change, and load it in a later session:
99
+
100
+ >>> Config.save("my_config.json")
101
+ >>> Config.reset()
102
+ >>> Config.load("my_config.json")
103
+ >>> Config.get("epc.scotland.domestic.url")
104
+ 'https://example.org/a.csv'
105
+
106
+ Change settings only inside a block:
107
+
108
+ >>> with Config():
109
+ ... Config.set("epc.scotland.domestic.url", "https://example.org/b.csv")
110
+ >>> Config.get("epc.scotland.domestic.url")
111
+ 'https://example.org/a.csv'
112
+ """
113
+
114
+ def __init__(self) -> None:
115
+ # Configs saved on entry.
116
+ self._saved: list[_ConfigContent] = []
117
+
118
+ def __enter__(self) -> Self:
119
+ """Save the config, so changes inside the block can be undone."""
120
+ self._saved.append(_config())
121
+ return self
122
+
123
+ def __exit__(self, *exc_info: object) -> None:
124
+ """Restore the config saved on entry."""
125
+ _replace(self._saved.pop())
126
+
127
+ @classmethod
128
+ def get(cls, key: str) -> Any:
129
+ """
130
+ Return the value of a setting.
131
+
132
+ Parameters
133
+ ----------
134
+ key : str
135
+ Dotted key of the setting, such as ``"epc.scotland.domestic.url"``.
136
+
137
+ Returns
138
+ -------
139
+ Any
140
+ A copy of the value, so changing it does not change the config.
141
+
142
+ Raises
143
+ ------
144
+ KeyError
145
+ If there is no setting with that key.
146
+ """
147
+ node: Any = _config()
148
+ for part in _parts(key):
149
+ if isinstance(node, BaseModel) and part in type(node).model_fields:
150
+ node = getattr(node, part)
151
+ elif isinstance(node, dict) and part in node:
152
+ node = node[part]
153
+ else:
154
+ raise KeyError(f"No config setting named {key!r}.")
155
+
156
+ return copy.deepcopy(node)
157
+
158
+ @classmethod
159
+ def set(cls, key: str, value: object) -> None:
160
+ """
161
+ Change a setting, or add an entry, such as a source.
162
+
163
+ Parameters
164
+ ----------
165
+ key : str
166
+ Dotted key of the setting. New entries, such as a source, may be
167
+ added to parts of the config that accept them.
168
+ value : object
169
+ The new value, which must be JSON-compatible. An entry may be
170
+ given as a model, such as a `Source`, or as a dict.
171
+
172
+ Raises
173
+ ------
174
+ KeyError
175
+ If the setting the key belongs to does not exist.
176
+ TypeError
177
+ If the value cannot be written as JSON.
178
+ pydantic.ValidationError
179
+ If the config would not be valid with the new entry added.
180
+ """
181
+ *parents, last = _parts(key)
182
+ data = _config().model_dump(mode="json")
183
+ if isinstance(value, BaseModel):
184
+ value = value.model_dump(mode="json")
185
+ _node(data, parents, key)[last] = value
186
+ _replace(_validate(data))
187
+
188
+ @classmethod
189
+ def reset(cls, key: str | None = None) -> None:
190
+ """
191
+ Put a setting, or the whole config, back to the defaults.
192
+
193
+ Parameters
194
+ ----------
195
+ key : str, optional
196
+ Dotted key of the setting. An entry that is not in the defaults,
197
+ such as an added source, is removed. Without a key, the whole
198
+ config is reset.
199
+
200
+ Raises
201
+ ------
202
+ KeyError
203
+ If there is no setting with that key.
204
+ pydantic.ValidationError
205
+ If the reset would leave an added entry incomplete, such as
206
+ resetting a field of an added source.
207
+ """
208
+ defaults = _defaults()
209
+ if key is None:
210
+ _replace(defaults)
211
+ return
212
+ *parents, last = _parts(key)
213
+ data = _config().model_dump(mode="json")
214
+ node = _node(data, parents, key)
215
+ if last not in node:
216
+ raise KeyError(f"No config setting named {key!r}.")
217
+ try:
218
+ default = _node(defaults.model_dump(mode="json"), parents, key)
219
+ except KeyError:
220
+ default = {}
221
+ if last in default:
222
+ node[last] = default[last]
223
+ else:
224
+ del node[last]
225
+ _replace(_validate(data))
226
+
227
+ @classmethod
228
+ def state(cls) -> dict[str, Any]:
229
+ """Return every setting as a dict, as it would be saved in JSON."""
230
+ return _config().model_dump(mode="json", exclude={"version"})
231
+
232
+ @classmethod
233
+ def load(cls, path: str | os.PathLike[str]) -> None:
234
+ """
235
+ Use the defaults with a saved config laid over them.
236
+
237
+ This replaces the config in use, so changes made with `set` are lost.
238
+ If loading fails, the config in use is left as it was.
239
+
240
+ Parameters
241
+ ----------
242
+ path : str | os.PathLike[str]
243
+ A saved config, holding only what differs from the defaults.
244
+
245
+ Raises
246
+ ------
247
+ FileNotFoundError
248
+ If the file does not exist.
249
+ ValueError
250
+ If the file is not valid JSON, or not a JSON object with a
251
+ ``"version"``.
252
+ pydantic.ValidationError
253
+ If the file does not match the config.
254
+ """
255
+ config_file = Path(path)
256
+ try:
257
+ saved = json.loads(config_file.read_bytes())
258
+ if not isinstance(saved, dict) or "version" not in saved:
259
+ raise ValueError('a saved config is a JSON object with "version"')
260
+ config = _validate(_merge(_defaults().model_dump(mode="json"), saved))
261
+ except ValueError as error:
262
+ error.add_note(f"while loading {config_file}")
263
+ raise
264
+ _replace(config)
265
+
266
+ @classmethod
267
+ def save(cls, path: str | os.PathLike[str], *, overwrite: bool = False) -> None:
268
+ """
269
+ Save the settings that differ from the defaults, as JSON.
270
+
271
+ The file is written under a temporary name and then renamed, so a
272
+ failed write never leaves a partial file.
273
+
274
+ Parameters
275
+ ----------
276
+ path : str | os.PathLike[str]
277
+ File to write.
278
+ overwrite : bool, default False
279
+ Whether to replace the file if it already exists.
280
+
281
+ Raises
282
+ ------
283
+ FileExistsError
284
+ If the file exists and `overwrite` is false.
285
+ ValueError
286
+ If a default entry has been removed, which a saved config,
287
+ holding only changes, cannot record.
288
+ """
289
+ target = Path(path)
290
+ if target.exists() and not overwrite:
291
+ raise FileExistsError(f"{target} exists; pass overwrite=True to replace it")
292
+ config = _config()
293
+ changes = _diff(
294
+ _defaults().model_dump(mode="json"), config.model_dump(mode="json")
295
+ )
296
+ text = json.dumps({"version": config.version, **changes}, indent=2)
297
+ temporary = target.with_name(f".{target.name}.tmp")
298
+ try:
299
+ temporary.write_text(text + "\n", encoding="utf-8")
300
+ temporary.replace(target)
301
+ finally:
302
+ temporary.unlink(missing_ok=True)
303
+
304
+
305
+ def _config() -> _ConfigContent:
306
+ """Return the config in use, loading it on first use."""
307
+ if _current_config is None:
308
+ _replace(_defaults())
309
+ if path := os.environ.get(_CONFIG_ENV_VAR):
310
+ Config.load(path)
311
+ assert _current_config is not None
312
+ return _current_config
313
+
314
+
315
+ def _replace(config: _ConfigContent) -> None:
316
+ """Make `config` the config in use."""
317
+ global _current_config
318
+ _current_config = config
319
+
320
+
321
+ def _defaults() -> _ConfigContent:
322
+ """Read the defaults shipped with the package."""
323
+ return _ConfigContent.model_validate_json(_DEFAULT_CONFIG_PATH.read_bytes())
324
+
325
+
326
+ def _validate(data: object) -> _ConfigContent:
327
+ """Check JSON-shaped data against the config."""
328
+ return _ConfigContent.model_validate_json(json.dumps(data))
329
+
330
+
331
+ def _parts(key: str) -> list[str]:
332
+ """Split a dotted key, rejecting keys that name no setting."""
333
+ parts = key.split(".")
334
+ if "" in parts or parts[0] == "version":
335
+ raise KeyError(f"No config setting named {key!r}.")
336
+ return parts
337
+
338
+
339
+ def _node(data: dict[str, Any], parts: list[str], key: str) -> dict[str, Any]:
340
+ """Return the dict that `parts` lead to in `data`."""
341
+ for part in parts:
342
+ if not isinstance(data, dict) or part not in data:
343
+ raise KeyError(f"No config setting named {key!r}.")
344
+ data = data[part]
345
+ return data
346
+
347
+
348
+ def _merge(default: object, override: object) -> object:
349
+ """Lay `override` over `default`, merging nested dicts key by key."""
350
+ if not (isinstance(default, dict) and isinstance(override, dict)):
351
+ return override
352
+ merged = dict(default)
353
+ for key, value in override.items():
354
+ merged[key] = _merge(merged[key], value) if key in merged else value
355
+ return merged
356
+
357
+
358
+ def _diff(
359
+ default: dict[str, Any], current: dict[str, Any], location: str = ""
360
+ ) -> dict[str, Any]:
361
+ """Return the part of `current` that differs from `default`."""
362
+ prefix = f"{location}." if location else ""
363
+ if removed := default.keys() - current.keys():
364
+ names = ", ".join(sorted(f"{prefix}{key}" for key in removed))
365
+ raise ValueError(f"cannot save without {names}: a saved config holds changes")
366
+ changes = {}
367
+ for key, value in current.items():
368
+ if key not in default:
369
+ changes[key] = value
370
+ elif isinstance(value, dict) and isinstance(default[key], dict):
371
+ if nested := _diff(default[key], value, f"{prefix}{key}"):
372
+ changes[key] = nested
373
+ elif value != default[key]:
374
+ changes[key] = value
375
+ return changes
@@ -0,0 +1 @@
1
+ """Build datasets from processed builds: cleaned and ready to use."""
@@ -0,0 +1,132 @@
1
+ """Parse the phrases of a dwelling's air tightness."""
2
+
3
+ import re
4
+ from dataclasses import dataclass
5
+ from typing import Final
6
+
7
+ import polars as pl
8
+
9
+ from py_epc._dataset._phrases import (
10
+ ColumnParser,
11
+ OutputColumn,
12
+ ParsedPhrase,
13
+ prepare_phrase,
14
+ )
15
+
16
+ # Tests an air permeability may be measured by, as published in brackets,
17
+ # such as "[AP50]".
18
+ _AIR_TIGHTNESS_TESTS: Final = ("ap50", "ap4")
19
+
20
+ # Air permeability, its test, value and basis, in English or Welsh.
21
+ _AIR_PERMEABILITY_PATTERN: Final = re.compile(
22
+ r"(?:air permeability|athreiddedd aer)\s*"
23
+ rf"(?:\[(?P<test>{'|'.join(_AIR_TIGHTNESS_TESTS)})\]\s*=\s*)?"
24
+ r"(?P<value>-?[0-9]+(?:\.[0-9]+)?)\s*m[3?]/h\.m[2?]\s*"
25
+ r"\((?P<basis>[^)]*)\)",
26
+ re.IGNORECASE,
27
+ )
28
+
29
+ # Basis of an air permeability, keyed by the phrase in its brackets.
30
+ _AIR_TIGHTNESS_BASIS_PHRASES: Final = {
31
+ "assumed": "assumed",
32
+ "rhagdybiaeth": "assumed",
33
+ "as tested": "tested",
34
+ "assessed average": "assessed_average",
35
+ "": None,
36
+ }
37
+
38
+ # Phrases saying the air permeability was not tested, given as the basis.
39
+ _AIR_TIGHTNESS_NOT_TESTED_PHRASES: Final = frozenset({"(not tested)", "(heb ei brofi)"})
40
+ _NOT_TESTED_BASIS: Final = "not_tested"
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class ParsedAirTightness(ParsedPhrase):
45
+ """What one phrase says of a dwelling's air tightness."""
46
+
47
+ permeability: float | None = None
48
+ """Air permeability, in cubic metres an hour per square metre."""
49
+
50
+ basis: str | None = None
51
+ """How it was found, such as ``"tested"`` or ``"assumed"``."""
52
+
53
+ test: str | None = None
54
+ """Test it was measured by, such as ``"ap50"``, if published."""
55
+
56
+
57
+ def parse_air_tightness(text: str | None) -> ParsedAirTightness:
58
+ """
59
+ Return what a cleaned phrase says of a dwelling's air tightness.
60
+
61
+ A phrase such as ``"Air permeability [AP50] = 5.4 m3/h.m2 (as tested)"``
62
+ gives the permeability, the test and the basis. ``"(not tested)"`` gives
63
+ only the basis. A permeability of zero or less is invalid. Phrases are
64
+ matched in any case, in English or Welsh.
65
+
66
+ Parameters
67
+ ----------
68
+ text : str or None
69
+ A phrase, already cleaned by `clean_text`.
70
+
71
+ Returns
72
+ -------
73
+ ParsedAirTightness
74
+ What the phrase says. Its kind is ``"unknown"`` if it matches no
75
+ known phrase, or has an unknown basis or test.
76
+ """
77
+ phrase = prepare_phrase(text)
78
+ if phrase is None:
79
+ return ParsedAirTightness("not_a_description")
80
+ if phrase.casefold() in _AIR_TIGHTNESS_NOT_TESTED_PHRASES:
81
+ return ParsedAirTightness("described", basis=_NOT_TESTED_BASIS)
82
+ match = _AIR_PERMEABILITY_PATTERN.fullmatch(phrase)
83
+ if match is None:
84
+ return ParsedAirTightness("unknown")
85
+ basis_phrase = match["basis"].strip().casefold()
86
+ if basis_phrase not in _AIR_TIGHTNESS_BASIS_PHRASES:
87
+ return ParsedAirTightness("unknown")
88
+ permeability = float(match["value"])
89
+ return ParsedAirTightness(
90
+ "described" if permeability > 0 else "invalid",
91
+ permeability=permeability,
92
+ basis=_AIR_TIGHTNESS_BASIS_PHRASES[basis_phrase],
93
+ test=match["test"].casefold() if match["test"] else None,
94
+ )
95
+
96
+
97
+ def air_tightness_parser() -> ColumnParser[ParsedAirTightness]:
98
+ """
99
+ Return a parser giving a dwelling's air permeability, basis and test.
100
+
101
+ The columns given are ``air_tightness_m3_per_h_m2``, then
102
+ ``air_tightness_basis`` and ``air_tightness_test``, each a
103
+ `polars.Enum`.
104
+
105
+ Returns
106
+ -------
107
+ ColumnParser
108
+ A parser of ``air_tightness_description`` for `parse_column`.
109
+ """
110
+ bases = [
111
+ *dict.fromkeys(b for b in _AIR_TIGHTNESS_BASIS_PHRASES.values() if b),
112
+ _NOT_TESTED_BASIS,
113
+ ]
114
+ return ColumnParser(
115
+ column="air_tightness_description",
116
+ parse_phrase=parse_air_tightness,
117
+ outputs=(
118
+ OutputColumn(
119
+ "air_tightness_m3_per_h_m2",
120
+ lambda parsed: parsed.permeability,
121
+ pl.Float64(),
122
+ ),
123
+ OutputColumn(
124
+ "air_tightness_basis", lambda parsed: parsed.basis, pl.Enum(bases)
125
+ ),
126
+ OutputColumn(
127
+ "air_tightness_test",
128
+ lambda parsed: parsed.test,
129
+ pl.Enum(list(_AIR_TIGHTNESS_TESTS)),
130
+ ),
131
+ ),
132
+ )
@@ -0,0 +1,130 @@
1
+ """
2
+ Keep the English of England and Wales values published in English and Welsh.
3
+
4
+ Notes
5
+ -----
6
+ Some certificates lodged from 2009 to 2011 give each description in both
7
+ languages, alternating English and Welsh parts separated by ``"|"``. Only
8
+ England and Wales does this. In Scotland, ``"|"`` separates the parts of a
9
+ building instead, which `split_building_parts` handles.
10
+ """
11
+
12
+ import warnings
13
+ from collections.abc import Sequence
14
+ from typing import Final
15
+
16
+ import polars as pl
17
+
18
+ # Separator between the English and Welsh parts of a bilingual value.
19
+ _LANGUAGE_SEPARATOR: Final = "|"
20
+
21
+ _WELSH_PART_PATTERN: Final = r"\|[^|]*(?:\||$)"
22
+
23
+ # Most values with an odd number of parts an error or warning lists.
24
+ _SHOWN_ODD_VALUES: Final = 20
25
+
26
+
27
+ def english_text(value: pl.Expr) -> pl.Expr:
28
+ """
29
+ Return the English of each value published in English and Welsh.
30
+
31
+ The English parts are the first, third and so on, and are joined as
32
+ published. A value without ``"|"`` is returned as it is. A value of no
33
+ English gives null.
34
+
35
+ Parameters
36
+ ----------
37
+ value : polars.Expr
38
+ Text as published.
39
+
40
+ Returns
41
+ -------
42
+ polars.Expr
43
+ The English text.
44
+
45
+ Examples
46
+ --------
47
+ >>> import polars as pl
48
+ >>> value = pl.lit("Room heaters, |Gwresogyddion, |electric|trydan")
49
+ >>> pl.select(english_text(value)).item()
50
+ 'Room heaters, electric'
51
+ """
52
+ english = value.str.replace_all(_WELSH_PART_PATTERN, "").str.strip_chars()
53
+ return pl.when(~_is_bilingual(value)).then(value).when(english != "").then(english)
54
+
55
+
56
+ def keep_english_text(
57
+ frame: pl.LazyFrame, columns: Sequence[str], *, strict: bool = True
58
+ ) -> pl.LazyFrame:
59
+ """
60
+ Return `frame` with each of `columns` in English, by `english_text`.
61
+
62
+ The columns are first read once to find any bilingual value with an odd
63
+ number of parts, whose English and Welsh cannot be told apart.
64
+
65
+ Parameters
66
+ ----------
67
+ frame : polars.LazyFrame
68
+ Rows of an England and Wales register, as published.
69
+ columns : Sequence[str]
70
+ Text columns that may hold bilingual values.
71
+ strict : bool, default True
72
+ Whether a value with an odd number of parts raises an error. If
73
+ false, it gives null, with a warning listing it.
74
+
75
+ Returns
76
+ -------
77
+ polars.LazyFrame
78
+ The rows, in the same order, with the columns in English.
79
+
80
+ Raises
81
+ ------
82
+ ValueError
83
+ If `strict` is true and a value has an odd number of parts. The
84
+ message lists them, with their columns.
85
+ """
86
+ counts = (
87
+ frame.select(_has_odd_parts(pl.col(column)).sum() for column in columns)
88
+ .collect(engine="streaming")
89
+ .row(0, named=True)
90
+ )
91
+ odd = {column: count for column, count in counts.items() if count}
92
+ if odd:
93
+ listed = "; ".join(
94
+ f"{column}: {count} such as {_odd_values(frame, column)}"
95
+ for column, count in odd.items()
96
+ )
97
+ message = f"bilingual values with an odd number of parts: {listed}"
98
+ if strict:
99
+ raise ValueError(message)
100
+ warnings.warn(f"{message}, given null", stacklevel=2)
101
+ return frame.with_columns(
102
+ pl.when(~_has_odd_parts(pl.col(column)))
103
+ .then(english_text(pl.col(column)))
104
+ .alias(column)
105
+ for column in columns
106
+ )
107
+
108
+
109
+ def _is_bilingual(value: pl.Expr) -> pl.Expr:
110
+ """Return whether each value has English and Welsh parts."""
111
+ return value.str.contains(_LANGUAGE_SEPARATOR, literal=True)
112
+
113
+
114
+ def _has_odd_parts(value: pl.Expr) -> pl.Expr:
115
+ """Return whether each bilingual value has an odd number of parts."""
116
+ separators = value.str.count_matches(_LANGUAGE_SEPARATOR, literal=True)
117
+ return (_is_bilingual(value) & (separators % 2 == 0)).fill_null(False)
118
+
119
+
120
+ def _odd_values(frame: pl.LazyFrame, column: str) -> list[str]:
121
+ """Return the first values of `column` with an odd number of parts."""
122
+ return (
123
+ frame.select(column)
124
+ .filter(_has_odd_parts(pl.col(column)))
125
+ .unique()
126
+ .sort(column)
127
+ .head(_SHOWN_ODD_VALUES)
128
+ .collect(engine="streaming")[column]
129
+ .to_list()
130
+ )