py-epc-uk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_epc/__init__.py +60 -0
- py_epc/_config.py +375 -0
- py_epc/_dataset/__init__.py +1 -0
- py_epc/_dataset/_air_tightness.py +132 -0
- py_epc/_dataset/_bilingual.py +130 -0
- py_epc/_dataset/_build.py +221 -0
- py_epc/_dataset/_categories.py +237 -0
- py_epc/_dataset/_construction_age.py +158 -0
- py_epc/_dataset/_definition.py +227 -0
- py_epc/_dataset/_definitions/epc-england-and-wales-domestic.json +126 -0
- py_epc/_dataset/_definitions/epc-scotland-domestic.json +206 -0
- py_epc/_dataset/_distinct.py +176 -0
- py_epc/_dataset/_fabric.py +222 -0
- py_epc/_dataset/_floor_level.py +112 -0
- py_epc/_dataset/_lighting.py +134 -0
- py_epc/_dataset/_main_fuel.py +105 -0
- py_epc/_dataset/_models.py +13 -0
- py_epc/_dataset/_parts.py +123 -0
- py_epc/_dataset/_photo_supply.py +221 -0
- py_epc/_dataset/_phrases.py +272 -0
- py_epc/_dataset/_prepare.py +86 -0
- py_epc/_dataset/_summary.py +127 -0
- py_epc/_dataset/_systems.py +249 -0
- py_epc/_dataset/_text.py +256 -0
- py_epc/_dataset/_vocabularies/alternative_improvements.json +32 -0
- py_epc/_dataset/_vocabularies/built_form.json +33 -0
- py_epc/_dataset/_vocabularies/energy_tariff.json +33 -0
- py_epc/_dataset/_vocabularies/floor.json +59 -0
- py_epc/_dataset/_vocabularies/glazed_area.json +29 -0
- py_epc/_dataset/_vocabularies/glazed_type.json +41 -0
- py_epc/_dataset/_vocabularies/heat_loss_corridor.json +24 -0
- py_epc/_dataset/_vocabularies/hotwater.json +269 -0
- py_epc/_dataset/_vocabularies/lzc_energy_sources.json +72 -0
- py_epc/_dataset/_vocabularies/main_fuel.json +153 -0
- py_epc/_dataset/_vocabularies/main_heating_category.json +53 -0
- py_epc/_dataset/_vocabularies/mainheat.json +667 -0
- py_epc/_dataset/_vocabularies/mainheatcont.json +296 -0
- py_epc/_dataset/_vocabularies/mechanical_ventilation.json +44 -0
- py_epc/_dataset/_vocabularies/property_type.json +28 -0
- py_epc/_dataset/_vocabularies/report_type.json +16 -0
- py_epc/_dataset/_vocabularies/roof.json +117 -0
- py_epc/_dataset/_vocabularies/secondheat.json +291 -0
- py_epc/_dataset/_vocabularies/tenure.json +21 -0
- py_epc/_dataset/_vocabularies/transaction_type.json +77 -0
- py_epc/_dataset/_vocabularies/type_of_assessment.json +16 -0
- py_epc/_dataset/_vocabularies/walls.json +132 -0
- py_epc/_dataset/_vocabularies/windows.json +132 -0
- py_epc/_dataset/_vocabulary.py +157 -0
- py_epc/_download.py +305 -0
- py_epc/_enums.py +109 -0
- py_epc/_files.py +60 -0
- py_epc/_manifest.py +121 -0
- py_epc/_output.py +140 -0
- py_epc/_pipeline.py +426 -0
- py_epc/_process/__init__.py +1 -0
- py_epc/_process/_archive.py +239 -0
- py_epc/_process/_build.py +348 -0
- py_epc/_process/_latest.py +202 -0
- py_epc/_process/_postcodes.py +503 -0
- py_epc/_process/_register.py +258 -0
- py_epc/_process/_registers/epc-england-and-wales-domestic.json +600 -0
- py_epc/_process/_registers/epc-scotland-domestic.json +690 -0
- py_epc/_process/_registers/ons-postcode-directory.json +157 -0
- py_epc/_process/_values.py +119 -0
- py_epc/_scan.py +286 -0
- py_epc/_source.py +30 -0
- py_epc/defaults.json +59 -0
- py_epc/py.typed +0 -0
- py_epc_uk-0.1.0.dist-info/METADATA +193 -0
- py_epc_uk-0.1.0.dist-info/RECORD +73 -0
- py_epc_uk-0.1.0.dist-info/WHEEL +4 -0
- py_epc_uk-0.1.0.dist-info/licenses/ATTRIBUTION.md +46 -0
- py_epc_uk-0.1.0.dist-info/licenses/LICENSE +373 -0
py_epc/__init__.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Py-EPC library to download & process Energy Performance of Building data."""
|
|
2
|
+
|
|
3
|
+
from py_epc._config import Config
|
|
4
|
+
from py_epc._dataset._build import (
|
|
5
|
+
DatasetManifest,
|
|
6
|
+
build_dataset,
|
|
7
|
+
build_epc_dataset,
|
|
8
|
+
)
|
|
9
|
+
from py_epc._download import (
|
|
10
|
+
DownloadError,
|
|
11
|
+
download,
|
|
12
|
+
download_epc,
|
|
13
|
+
download_ons_postcode_directory,
|
|
14
|
+
download_os_open_uprn,
|
|
15
|
+
)
|
|
16
|
+
from py_epc._enums import EPCOrigin, EPCSegment, ONSProduct, OSProduct
|
|
17
|
+
from py_epc._manifest import DownloadManifest
|
|
18
|
+
from py_epc._output import OutputType
|
|
19
|
+
from py_epc._pipeline import OutputManifest, run_epc_pipeline
|
|
20
|
+
from py_epc._process._build import (
|
|
21
|
+
ProcessManifest,
|
|
22
|
+
ProcessWarning,
|
|
23
|
+
process,
|
|
24
|
+
process_epc,
|
|
25
|
+
)
|
|
26
|
+
from py_epc._process._postcodes import (
|
|
27
|
+
PostcodeManifest,
|
|
28
|
+
process_ons_postcode_directory,
|
|
29
|
+
)
|
|
30
|
+
from py_epc._scan import scan, scan_epc
|
|
31
|
+
from py_epc._source import Source
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"Config",
|
|
35
|
+
"DatasetManifest",
|
|
36
|
+
"DownloadError",
|
|
37
|
+
"DownloadManifest",
|
|
38
|
+
"EPCOrigin",
|
|
39
|
+
"EPCSegment",
|
|
40
|
+
"ONSProduct",
|
|
41
|
+
"OSProduct",
|
|
42
|
+
"OutputManifest",
|
|
43
|
+
"OutputType",
|
|
44
|
+
"PostcodeManifest",
|
|
45
|
+
"ProcessManifest",
|
|
46
|
+
"ProcessWarning",
|
|
47
|
+
"Source",
|
|
48
|
+
"build_dataset",
|
|
49
|
+
"build_epc_dataset",
|
|
50
|
+
"download",
|
|
51
|
+
"download_epc",
|
|
52
|
+
"download_ons_postcode_directory",
|
|
53
|
+
"download_os_open_uprn",
|
|
54
|
+
"process",
|
|
55
|
+
"process_epc",
|
|
56
|
+
"process_ons_postcode_directory",
|
|
57
|
+
"run_epc_pipeline",
|
|
58
|
+
"scan",
|
|
59
|
+
"scan_epc",
|
|
60
|
+
]
|
py_epc/_config.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""Configurations for the Py-EPC library."""
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from contextlib import ContextDecorator
|
|
7
|
+
from importlib.resources import files
|
|
8
|
+
from importlib.resources.abc import Traversable
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Final, Literal, Self
|
|
11
|
+
|
|
12
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
13
|
+
|
|
14
|
+
from py_epc._enums import EPCOrigin, EPCSegment, ONSProduct, OSProduct
|
|
15
|
+
from py_epc._source import Source
|
|
16
|
+
|
|
17
|
+
_DEFAULT_CONFIG_PATH: Final[Traversable] = files("py_epc") / "defaults.json"
|
|
18
|
+
|
|
19
|
+
# Environment variable naming a saved config to load on first use.
|
|
20
|
+
_CONFIG_ENV_VAR: Final = "PY_EPC_CONFIG"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _Geography(BaseModel):
|
|
24
|
+
"""Settings for the areas added to each postcode."""
|
|
25
|
+
|
|
26
|
+
model_config = ConfigDict(
|
|
27
|
+
frozen=True,
|
|
28
|
+
extra="forbid",
|
|
29
|
+
strict=True,
|
|
30
|
+
hide_input_in_errors=True,
|
|
31
|
+
use_attribute_docstrings=True,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
fields: tuple[str, ...] = ()
|
|
35
|
+
"""Fields of the ONS Postcode Directory to keep, such as ``"lad"``."""
|
|
36
|
+
|
|
37
|
+
names: bool = True
|
|
38
|
+
"""Whether to add each field's names beside its codes."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class _ConfigContent(BaseModel):
|
|
42
|
+
"""The checked contents of the config."""
|
|
43
|
+
|
|
44
|
+
model_config = ConfigDict(
|
|
45
|
+
title="Config",
|
|
46
|
+
frozen=True,
|
|
47
|
+
extra="forbid",
|
|
48
|
+
strict=True,
|
|
49
|
+
hide_input_in_errors=True,
|
|
50
|
+
use_attribute_docstrings=True,
|
|
51
|
+
)
|
|
52
|
+
version: Literal[1]
|
|
53
|
+
"""Version of the config format."""
|
|
54
|
+
|
|
55
|
+
epc: dict[EPCOrigin, dict[EPCSegment, Source]] = Field(default_factory=dict)
|
|
56
|
+
"""EPC sources, keyed by origin, then by segment."""
|
|
57
|
+
|
|
58
|
+
ons: dict[ONSProduct, Source] = Field(default_factory=dict)
|
|
59
|
+
"""Sources of the Office for National Statistics, keyed by product."""
|
|
60
|
+
|
|
61
|
+
os: dict[OSProduct, Source] = Field(default_factory=dict)
|
|
62
|
+
"""Sources of Ordnance Survey, keyed by product."""
|
|
63
|
+
|
|
64
|
+
geography: _Geography = Field(default_factory=_Geography)
|
|
65
|
+
"""Areas added to each postcode from the ONS Postcode Directory."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
_current_config: _ConfigContent | None = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Config(ContextDecorator):
|
|
72
|
+
"""
|
|
73
|
+
The configuration settings for the Py-EPC library.
|
|
74
|
+
|
|
75
|
+
The defaults are shipped with the package. Change them with the `set`,
|
|
76
|
+
`reset` and `load` methods. Every change is checked against the config's
|
|
77
|
+
schema, and a rejected change leaves the config as it was. Errors never
|
|
78
|
+
show the values found, so private tokens used to fetch data cannot leak.
|
|
79
|
+
|
|
80
|
+
`save` writes only the settings that differ from the defaults, so fixes to
|
|
81
|
+
the defaults in later releases still apply to everything you have not
|
|
82
|
+
changed. If the ``PY_EPC_CONFIG`` environment variable names a saved
|
|
83
|
+
config, it is loaded on first use.
|
|
84
|
+
|
|
85
|
+
When used as a context manager or a decorator, `Config` undoes every change
|
|
86
|
+
made inside it on exit. The config is shared by every thread.
|
|
87
|
+
|
|
88
|
+
Examples
|
|
89
|
+
--------
|
|
90
|
+
>>> from py_epc import Config
|
|
91
|
+
>>> Config.get("epc.scotland.domestic.publisher")
|
|
92
|
+
'Scottish Government'
|
|
93
|
+
|
|
94
|
+
Change a setting:
|
|
95
|
+
|
|
96
|
+
>>> Config.set("epc.scotland.domestic.url", "https://example.org/a.csv")
|
|
97
|
+
|
|
98
|
+
Save the change, and load it in a later session:
|
|
99
|
+
|
|
100
|
+
>>> Config.save("my_config.json")
|
|
101
|
+
>>> Config.reset()
|
|
102
|
+
>>> Config.load("my_config.json")
|
|
103
|
+
>>> Config.get("epc.scotland.domestic.url")
|
|
104
|
+
'https://example.org/a.csv'
|
|
105
|
+
|
|
106
|
+
Change settings only inside a block:
|
|
107
|
+
|
|
108
|
+
>>> with Config():
|
|
109
|
+
... Config.set("epc.scotland.domestic.url", "https://example.org/b.csv")
|
|
110
|
+
>>> Config.get("epc.scotland.domestic.url")
|
|
111
|
+
'https://example.org/a.csv'
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def __init__(self) -> None:
|
|
115
|
+
# Configs saved on entry.
|
|
116
|
+
self._saved: list[_ConfigContent] = []
|
|
117
|
+
|
|
118
|
+
def __enter__(self) -> Self:
|
|
119
|
+
"""Save the config, so changes inside the block can be undone."""
|
|
120
|
+
self._saved.append(_config())
|
|
121
|
+
return self
|
|
122
|
+
|
|
123
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
124
|
+
"""Restore the config saved on entry."""
|
|
125
|
+
_replace(self._saved.pop())
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def get(cls, key: str) -> Any:
|
|
129
|
+
"""
|
|
130
|
+
Return the value of a setting.
|
|
131
|
+
|
|
132
|
+
Parameters
|
|
133
|
+
----------
|
|
134
|
+
key : str
|
|
135
|
+
Dotted key of the setting, such as ``"epc.scotland.domestic.url"``.
|
|
136
|
+
|
|
137
|
+
Returns
|
|
138
|
+
-------
|
|
139
|
+
Any
|
|
140
|
+
A copy of the value, so changing it does not change the config.
|
|
141
|
+
|
|
142
|
+
Raises
|
|
143
|
+
------
|
|
144
|
+
KeyError
|
|
145
|
+
If there is no setting with that key.
|
|
146
|
+
"""
|
|
147
|
+
node: Any = _config()
|
|
148
|
+
for part in _parts(key):
|
|
149
|
+
if isinstance(node, BaseModel) and part in type(node).model_fields:
|
|
150
|
+
node = getattr(node, part)
|
|
151
|
+
elif isinstance(node, dict) and part in node:
|
|
152
|
+
node = node[part]
|
|
153
|
+
else:
|
|
154
|
+
raise KeyError(f"No config setting named {key!r}.")
|
|
155
|
+
|
|
156
|
+
return copy.deepcopy(node)
|
|
157
|
+
|
|
158
|
+
@classmethod
|
|
159
|
+
def set(cls, key: str, value: object) -> None:
|
|
160
|
+
"""
|
|
161
|
+
Change a setting, or add an entry, such as a source.
|
|
162
|
+
|
|
163
|
+
Parameters
|
|
164
|
+
----------
|
|
165
|
+
key : str
|
|
166
|
+
Dotted key of the setting. New entries, such as a source, may be
|
|
167
|
+
added to parts of the config that accept them.
|
|
168
|
+
value : object
|
|
169
|
+
The new value, which must be JSON-compatible. An entry may be
|
|
170
|
+
given as a model, such as a `Source`, or as a dict.
|
|
171
|
+
|
|
172
|
+
Raises
|
|
173
|
+
------
|
|
174
|
+
KeyError
|
|
175
|
+
If the setting the key belongs to does not exist.
|
|
176
|
+
TypeError
|
|
177
|
+
If the value cannot be written as JSON.
|
|
178
|
+
pydantic.ValidationError
|
|
179
|
+
If the config would not be valid with the new entry added.
|
|
180
|
+
"""
|
|
181
|
+
*parents, last = _parts(key)
|
|
182
|
+
data = _config().model_dump(mode="json")
|
|
183
|
+
if isinstance(value, BaseModel):
|
|
184
|
+
value = value.model_dump(mode="json")
|
|
185
|
+
_node(data, parents, key)[last] = value
|
|
186
|
+
_replace(_validate(data))
|
|
187
|
+
|
|
188
|
+
@classmethod
|
|
189
|
+
def reset(cls, key: str | None = None) -> None:
|
|
190
|
+
"""
|
|
191
|
+
Put a setting, or the whole config, back to the defaults.
|
|
192
|
+
|
|
193
|
+
Parameters
|
|
194
|
+
----------
|
|
195
|
+
key : str, optional
|
|
196
|
+
Dotted key of the setting. An entry that is not in the defaults,
|
|
197
|
+
such as an added source, is removed. Without a key, the whole
|
|
198
|
+
config is reset.
|
|
199
|
+
|
|
200
|
+
Raises
|
|
201
|
+
------
|
|
202
|
+
KeyError
|
|
203
|
+
If there is no setting with that key.
|
|
204
|
+
pydantic.ValidationError
|
|
205
|
+
If the reset would leave an added entry incomplete, such as
|
|
206
|
+
resetting a field of an added source.
|
|
207
|
+
"""
|
|
208
|
+
defaults = _defaults()
|
|
209
|
+
if key is None:
|
|
210
|
+
_replace(defaults)
|
|
211
|
+
return
|
|
212
|
+
*parents, last = _parts(key)
|
|
213
|
+
data = _config().model_dump(mode="json")
|
|
214
|
+
node = _node(data, parents, key)
|
|
215
|
+
if last not in node:
|
|
216
|
+
raise KeyError(f"No config setting named {key!r}.")
|
|
217
|
+
try:
|
|
218
|
+
default = _node(defaults.model_dump(mode="json"), parents, key)
|
|
219
|
+
except KeyError:
|
|
220
|
+
default = {}
|
|
221
|
+
if last in default:
|
|
222
|
+
node[last] = default[last]
|
|
223
|
+
else:
|
|
224
|
+
del node[last]
|
|
225
|
+
_replace(_validate(data))
|
|
226
|
+
|
|
227
|
+
@classmethod
|
|
228
|
+
def state(cls) -> dict[str, Any]:
|
|
229
|
+
"""Return every setting as a dict, as it would be saved in JSON."""
|
|
230
|
+
return _config().model_dump(mode="json", exclude={"version"})
|
|
231
|
+
|
|
232
|
+
@classmethod
|
|
233
|
+
def load(cls, path: str | os.PathLike[str]) -> None:
|
|
234
|
+
"""
|
|
235
|
+
Use the defaults with a saved config laid over them.
|
|
236
|
+
|
|
237
|
+
This replaces the config in use, so changes made with `set` are lost.
|
|
238
|
+
If loading fails, the config in use is left as it was.
|
|
239
|
+
|
|
240
|
+
Parameters
|
|
241
|
+
----------
|
|
242
|
+
path : str | os.PathLike[str]
|
|
243
|
+
A saved config, holding only what differs from the defaults.
|
|
244
|
+
|
|
245
|
+
Raises
|
|
246
|
+
------
|
|
247
|
+
FileNotFoundError
|
|
248
|
+
If the file does not exist.
|
|
249
|
+
ValueError
|
|
250
|
+
If the file is not valid JSON, or not a JSON object with a
|
|
251
|
+
``"version"``.
|
|
252
|
+
pydantic.ValidationError
|
|
253
|
+
If the file does not match the config.
|
|
254
|
+
"""
|
|
255
|
+
config_file = Path(path)
|
|
256
|
+
try:
|
|
257
|
+
saved = json.loads(config_file.read_bytes())
|
|
258
|
+
if not isinstance(saved, dict) or "version" not in saved:
|
|
259
|
+
raise ValueError('a saved config is a JSON object with "version"')
|
|
260
|
+
config = _validate(_merge(_defaults().model_dump(mode="json"), saved))
|
|
261
|
+
except ValueError as error:
|
|
262
|
+
error.add_note(f"while loading {config_file}")
|
|
263
|
+
raise
|
|
264
|
+
_replace(config)
|
|
265
|
+
|
|
266
|
+
@classmethod
|
|
267
|
+
def save(cls, path: str | os.PathLike[str], *, overwrite: bool = False) -> None:
|
|
268
|
+
"""
|
|
269
|
+
Save the settings that differ from the defaults, as JSON.
|
|
270
|
+
|
|
271
|
+
The file is written under a temporary name and then renamed, so a
|
|
272
|
+
failed write never leaves a partial file.
|
|
273
|
+
|
|
274
|
+
Parameters
|
|
275
|
+
----------
|
|
276
|
+
path : str | os.PathLike[str]
|
|
277
|
+
File to write.
|
|
278
|
+
overwrite : bool, default False
|
|
279
|
+
Whether to replace the file if it already exists.
|
|
280
|
+
|
|
281
|
+
Raises
|
|
282
|
+
------
|
|
283
|
+
FileExistsError
|
|
284
|
+
If the file exists and `overwrite` is false.
|
|
285
|
+
ValueError
|
|
286
|
+
If a default entry has been removed, which a saved config,
|
|
287
|
+
holding only changes, cannot record.
|
|
288
|
+
"""
|
|
289
|
+
target = Path(path)
|
|
290
|
+
if target.exists() and not overwrite:
|
|
291
|
+
raise FileExistsError(f"{target} exists; pass overwrite=True to replace it")
|
|
292
|
+
config = _config()
|
|
293
|
+
changes = _diff(
|
|
294
|
+
_defaults().model_dump(mode="json"), config.model_dump(mode="json")
|
|
295
|
+
)
|
|
296
|
+
text = json.dumps({"version": config.version, **changes}, indent=2)
|
|
297
|
+
temporary = target.with_name(f".{target.name}.tmp")
|
|
298
|
+
try:
|
|
299
|
+
temporary.write_text(text + "\n", encoding="utf-8")
|
|
300
|
+
temporary.replace(target)
|
|
301
|
+
finally:
|
|
302
|
+
temporary.unlink(missing_ok=True)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _config() -> _ConfigContent:
|
|
306
|
+
"""Return the config in use, loading it on first use."""
|
|
307
|
+
if _current_config is None:
|
|
308
|
+
_replace(_defaults())
|
|
309
|
+
if path := os.environ.get(_CONFIG_ENV_VAR):
|
|
310
|
+
Config.load(path)
|
|
311
|
+
assert _current_config is not None
|
|
312
|
+
return _current_config
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _replace(config: _ConfigContent) -> None:
|
|
316
|
+
"""Make `config` the config in use."""
|
|
317
|
+
global _current_config
|
|
318
|
+
_current_config = config
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _defaults() -> _ConfigContent:
|
|
322
|
+
"""Read the defaults shipped with the package."""
|
|
323
|
+
return _ConfigContent.model_validate_json(_DEFAULT_CONFIG_PATH.read_bytes())
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _validate(data: object) -> _ConfigContent:
|
|
327
|
+
"""Check JSON-shaped data against the config."""
|
|
328
|
+
return _ConfigContent.model_validate_json(json.dumps(data))
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _parts(key: str) -> list[str]:
|
|
332
|
+
"""Split a dotted key, rejecting keys that name no setting."""
|
|
333
|
+
parts = key.split(".")
|
|
334
|
+
if "" in parts or parts[0] == "version":
|
|
335
|
+
raise KeyError(f"No config setting named {key!r}.")
|
|
336
|
+
return parts
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _node(data: dict[str, Any], parts: list[str], key: str) -> dict[str, Any]:
|
|
340
|
+
"""Return the dict that `parts` lead to in `data`."""
|
|
341
|
+
for part in parts:
|
|
342
|
+
if not isinstance(data, dict) or part not in data:
|
|
343
|
+
raise KeyError(f"No config setting named {key!r}.")
|
|
344
|
+
data = data[part]
|
|
345
|
+
return data
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _merge(default: object, override: object) -> object:
|
|
349
|
+
"""Lay `override` over `default`, merging nested dicts key by key."""
|
|
350
|
+
if not (isinstance(default, dict) and isinstance(override, dict)):
|
|
351
|
+
return override
|
|
352
|
+
merged = dict(default)
|
|
353
|
+
for key, value in override.items():
|
|
354
|
+
merged[key] = _merge(merged[key], value) if key in merged else value
|
|
355
|
+
return merged
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _diff(
|
|
359
|
+
default: dict[str, Any], current: dict[str, Any], location: str = ""
|
|
360
|
+
) -> dict[str, Any]:
|
|
361
|
+
"""Return the part of `current` that differs from `default`."""
|
|
362
|
+
prefix = f"{location}." if location else ""
|
|
363
|
+
if removed := default.keys() - current.keys():
|
|
364
|
+
names = ", ".join(sorted(f"{prefix}{key}" for key in removed))
|
|
365
|
+
raise ValueError(f"cannot save without {names}: a saved config holds changes")
|
|
366
|
+
changes = {}
|
|
367
|
+
for key, value in current.items():
|
|
368
|
+
if key not in default:
|
|
369
|
+
changes[key] = value
|
|
370
|
+
elif isinstance(value, dict) and isinstance(default[key], dict):
|
|
371
|
+
if nested := _diff(default[key], value, f"{prefix}{key}"):
|
|
372
|
+
changes[key] = nested
|
|
373
|
+
elif value != default[key]:
|
|
374
|
+
changes[key] = value
|
|
375
|
+
return changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Build datasets from processed builds: cleaned and ready to use."""
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Parse the phrases of a dwelling's air tightness."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Final
|
|
6
|
+
|
|
7
|
+
import polars as pl
|
|
8
|
+
|
|
9
|
+
from py_epc._dataset._phrases import (
|
|
10
|
+
ColumnParser,
|
|
11
|
+
OutputColumn,
|
|
12
|
+
ParsedPhrase,
|
|
13
|
+
prepare_phrase,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
# Tests an air permeability may be measured by, as published in brackets,
|
|
17
|
+
# such as "[AP50]".
|
|
18
|
+
_AIR_TIGHTNESS_TESTS: Final = ("ap50", "ap4")
|
|
19
|
+
|
|
20
|
+
# Air permeability, its test, value and basis, in English or Welsh.
|
|
21
|
+
_AIR_PERMEABILITY_PATTERN: Final = re.compile(
|
|
22
|
+
r"(?:air permeability|athreiddedd aer)\s*"
|
|
23
|
+
rf"(?:\[(?P<test>{'|'.join(_AIR_TIGHTNESS_TESTS)})\]\s*=\s*)?"
|
|
24
|
+
r"(?P<value>-?[0-9]+(?:\.[0-9]+)?)\s*m[3?]/h\.m[2?]\s*"
|
|
25
|
+
r"\((?P<basis>[^)]*)\)",
|
|
26
|
+
re.IGNORECASE,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# Basis of an air permeability, keyed by the phrase in its brackets.
|
|
30
|
+
_AIR_TIGHTNESS_BASIS_PHRASES: Final = {
|
|
31
|
+
"assumed": "assumed",
|
|
32
|
+
"rhagdybiaeth": "assumed",
|
|
33
|
+
"as tested": "tested",
|
|
34
|
+
"assessed average": "assessed_average",
|
|
35
|
+
"": None,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
# Phrases saying the air permeability was not tested, given as the basis.
|
|
39
|
+
_AIR_TIGHTNESS_NOT_TESTED_PHRASES: Final = frozenset({"(not tested)", "(heb ei brofi)"})
|
|
40
|
+
_NOT_TESTED_BASIS: Final = "not_tested"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class ParsedAirTightness(ParsedPhrase):
|
|
45
|
+
"""What one phrase says of a dwelling's air tightness."""
|
|
46
|
+
|
|
47
|
+
permeability: float | None = None
|
|
48
|
+
"""Air permeability, in cubic metres an hour per square metre."""
|
|
49
|
+
|
|
50
|
+
basis: str | None = None
|
|
51
|
+
"""How it was found, such as ``"tested"`` or ``"assumed"``."""
|
|
52
|
+
|
|
53
|
+
test: str | None = None
|
|
54
|
+
"""Test it was measured by, such as ``"ap50"``, if published."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def parse_air_tightness(text: str | None) -> ParsedAirTightness:
|
|
58
|
+
"""
|
|
59
|
+
Return what a cleaned phrase says of a dwelling's air tightness.
|
|
60
|
+
|
|
61
|
+
A phrase such as ``"Air permeability [AP50] = 5.4 m3/h.m2 (as tested)"``
|
|
62
|
+
gives the permeability, the test and the basis. ``"(not tested)"`` gives
|
|
63
|
+
only the basis. A permeability of zero or less is invalid. Phrases are
|
|
64
|
+
matched in any case, in English or Welsh.
|
|
65
|
+
|
|
66
|
+
Parameters
|
|
67
|
+
----------
|
|
68
|
+
text : str or None
|
|
69
|
+
A phrase, already cleaned by `clean_text`.
|
|
70
|
+
|
|
71
|
+
Returns
|
|
72
|
+
-------
|
|
73
|
+
ParsedAirTightness
|
|
74
|
+
What the phrase says. Its kind is ``"unknown"`` if it matches no
|
|
75
|
+
known phrase, or has an unknown basis or test.
|
|
76
|
+
"""
|
|
77
|
+
phrase = prepare_phrase(text)
|
|
78
|
+
if phrase is None:
|
|
79
|
+
return ParsedAirTightness("not_a_description")
|
|
80
|
+
if phrase.casefold() in _AIR_TIGHTNESS_NOT_TESTED_PHRASES:
|
|
81
|
+
return ParsedAirTightness("described", basis=_NOT_TESTED_BASIS)
|
|
82
|
+
match = _AIR_PERMEABILITY_PATTERN.fullmatch(phrase)
|
|
83
|
+
if match is None:
|
|
84
|
+
return ParsedAirTightness("unknown")
|
|
85
|
+
basis_phrase = match["basis"].strip().casefold()
|
|
86
|
+
if basis_phrase not in _AIR_TIGHTNESS_BASIS_PHRASES:
|
|
87
|
+
return ParsedAirTightness("unknown")
|
|
88
|
+
permeability = float(match["value"])
|
|
89
|
+
return ParsedAirTightness(
|
|
90
|
+
"described" if permeability > 0 else "invalid",
|
|
91
|
+
permeability=permeability,
|
|
92
|
+
basis=_AIR_TIGHTNESS_BASIS_PHRASES[basis_phrase],
|
|
93
|
+
test=match["test"].casefold() if match["test"] else None,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def air_tightness_parser() -> ColumnParser[ParsedAirTightness]:
|
|
98
|
+
"""
|
|
99
|
+
Return a parser giving a dwelling's air permeability, basis and test.
|
|
100
|
+
|
|
101
|
+
The columns given are ``air_tightness_m3_per_h_m2``, then
|
|
102
|
+
``air_tightness_basis`` and ``air_tightness_test``, each a
|
|
103
|
+
`polars.Enum`.
|
|
104
|
+
|
|
105
|
+
Returns
|
|
106
|
+
-------
|
|
107
|
+
ColumnParser
|
|
108
|
+
A parser of ``air_tightness_description`` for `parse_column`.
|
|
109
|
+
"""
|
|
110
|
+
bases = [
|
|
111
|
+
*dict.fromkeys(b for b in _AIR_TIGHTNESS_BASIS_PHRASES.values() if b),
|
|
112
|
+
_NOT_TESTED_BASIS,
|
|
113
|
+
]
|
|
114
|
+
return ColumnParser(
|
|
115
|
+
column="air_tightness_description",
|
|
116
|
+
parse_phrase=parse_air_tightness,
|
|
117
|
+
outputs=(
|
|
118
|
+
OutputColumn(
|
|
119
|
+
"air_tightness_m3_per_h_m2",
|
|
120
|
+
lambda parsed: parsed.permeability,
|
|
121
|
+
pl.Float64(),
|
|
122
|
+
),
|
|
123
|
+
OutputColumn(
|
|
124
|
+
"air_tightness_basis", lambda parsed: parsed.basis, pl.Enum(bases)
|
|
125
|
+
),
|
|
126
|
+
OutputColumn(
|
|
127
|
+
"air_tightness_test",
|
|
128
|
+
lambda parsed: parsed.test,
|
|
129
|
+
pl.Enum(list(_AIR_TIGHTNESS_TESTS)),
|
|
130
|
+
),
|
|
131
|
+
),
|
|
132
|
+
)
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Keep the English of England and Wales values published in English and Welsh.
|
|
3
|
+
|
|
4
|
+
Notes
|
|
5
|
+
-----
|
|
6
|
+
Some certificates lodged from 2009 to 2011 give each description in both
|
|
7
|
+
languages, alternating English and Welsh parts separated by ``"|"``. Only
|
|
8
|
+
England and Wales does this. In Scotland, ``"|"`` separates the parts of a
|
|
9
|
+
building instead, which `split_building_parts` handles.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import warnings
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from typing import Final
|
|
15
|
+
|
|
16
|
+
import polars as pl
|
|
17
|
+
|
|
18
|
+
# Separator between the English and Welsh parts of a bilingual value.
|
|
19
|
+
_LANGUAGE_SEPARATOR: Final = "|"
|
|
20
|
+
|
|
21
|
+
_WELSH_PART_PATTERN: Final = r"\|[^|]*(?:\||$)"
|
|
22
|
+
|
|
23
|
+
# Most values with an odd number of parts an error or warning lists.
|
|
24
|
+
_SHOWN_ODD_VALUES: Final = 20
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def english_text(value: pl.Expr) -> pl.Expr:
|
|
28
|
+
"""
|
|
29
|
+
Return the English of each value published in English and Welsh.
|
|
30
|
+
|
|
31
|
+
The English parts are the first, third and so on, and are joined as
|
|
32
|
+
published. A value without ``"|"`` is returned as it is. A value of no
|
|
33
|
+
English gives null.
|
|
34
|
+
|
|
35
|
+
Parameters
|
|
36
|
+
----------
|
|
37
|
+
value : polars.Expr
|
|
38
|
+
Text as published.
|
|
39
|
+
|
|
40
|
+
Returns
|
|
41
|
+
-------
|
|
42
|
+
polars.Expr
|
|
43
|
+
The English text.
|
|
44
|
+
|
|
45
|
+
Examples
|
|
46
|
+
--------
|
|
47
|
+
>>> import polars as pl
|
|
48
|
+
>>> value = pl.lit("Room heaters, |Gwresogyddion, |electric|trydan")
|
|
49
|
+
>>> pl.select(english_text(value)).item()
|
|
50
|
+
'Room heaters, electric'
|
|
51
|
+
"""
|
|
52
|
+
english = value.str.replace_all(_WELSH_PART_PATTERN, "").str.strip_chars()
|
|
53
|
+
return pl.when(~_is_bilingual(value)).then(value).when(english != "").then(english)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def keep_english_text(
|
|
57
|
+
frame: pl.LazyFrame, columns: Sequence[str], *, strict: bool = True
|
|
58
|
+
) -> pl.LazyFrame:
|
|
59
|
+
"""
|
|
60
|
+
Return `frame` with each of `columns` in English, by `english_text`.
|
|
61
|
+
|
|
62
|
+
The columns are first read once to find any bilingual value with an odd
|
|
63
|
+
number of parts, whose English and Welsh cannot be told apart.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
frame : polars.LazyFrame
|
|
68
|
+
Rows of an England and Wales register, as published.
|
|
69
|
+
columns : Sequence[str]
|
|
70
|
+
Text columns that may hold bilingual values.
|
|
71
|
+
strict : bool, default True
|
|
72
|
+
Whether a value with an odd number of parts raises an error. If
|
|
73
|
+
false, it gives null, with a warning listing it.
|
|
74
|
+
|
|
75
|
+
Returns
|
|
76
|
+
-------
|
|
77
|
+
polars.LazyFrame
|
|
78
|
+
The rows, in the same order, with the columns in English.
|
|
79
|
+
|
|
80
|
+
Raises
|
|
81
|
+
------
|
|
82
|
+
ValueError
|
|
83
|
+
If `strict` is true and a value has an odd number of parts. The
|
|
84
|
+
message lists them, with their columns.
|
|
85
|
+
"""
|
|
86
|
+
counts = (
|
|
87
|
+
frame.select(_has_odd_parts(pl.col(column)).sum() for column in columns)
|
|
88
|
+
.collect(engine="streaming")
|
|
89
|
+
.row(0, named=True)
|
|
90
|
+
)
|
|
91
|
+
odd = {column: count for column, count in counts.items() if count}
|
|
92
|
+
if odd:
|
|
93
|
+
listed = "; ".join(
|
|
94
|
+
f"{column}: {count} such as {_odd_values(frame, column)}"
|
|
95
|
+
for column, count in odd.items()
|
|
96
|
+
)
|
|
97
|
+
message = f"bilingual values with an odd number of parts: {listed}"
|
|
98
|
+
if strict:
|
|
99
|
+
raise ValueError(message)
|
|
100
|
+
warnings.warn(f"{message}, given null", stacklevel=2)
|
|
101
|
+
return frame.with_columns(
|
|
102
|
+
pl.when(~_has_odd_parts(pl.col(column)))
|
|
103
|
+
.then(english_text(pl.col(column)))
|
|
104
|
+
.alias(column)
|
|
105
|
+
for column in columns
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _is_bilingual(value: pl.Expr) -> pl.Expr:
|
|
110
|
+
"""Return whether each value has English and Welsh parts."""
|
|
111
|
+
return value.str.contains(_LANGUAGE_SEPARATOR, literal=True)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _has_odd_parts(value: pl.Expr) -> pl.Expr:
|
|
115
|
+
"""Return whether each bilingual value has an odd number of parts."""
|
|
116
|
+
separators = value.str.count_matches(_LANGUAGE_SEPARATOR, literal=True)
|
|
117
|
+
return (_is_bilingual(value) & (separators % 2 == 0)).fill_null(False)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _odd_values(frame: pl.LazyFrame, column: str) -> list[str]:
|
|
121
|
+
"""Return the first values of `column` with an odd number of parts."""
|
|
122
|
+
return (
|
|
123
|
+
frame.select(column)
|
|
124
|
+
.filter(_has_odd_parts(pl.col(column)))
|
|
125
|
+
.unique()
|
|
126
|
+
.sort(column)
|
|
127
|
+
.head(_SHOWN_ODD_VALUES)
|
|
128
|
+
.collect(engine="streaming")[column]
|
|
129
|
+
.to_list()
|
|
130
|
+
)
|