utility-tariffs-lib 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {utility_tariffs_lib-0.1.0/utility_tariffs_lib.egg-info → utility_tariffs_lib-0.2.0}/PKG-INFO +1 -1
- utility_tariffs_lib-0.2.0/lib/tariffstore/__init__.py +22 -0
- utility_tariffs_lib-0.2.0/lib/tariffstore/aclient.py +267 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/client.py +56 -164
- utility_tariffs_lib-0.2.0/lib/tariffstore/core.py +261 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/price_client.py +9 -29
- utility_tariffs_lib-0.2.0/lib/tariffstore/price_rules.py +31 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/pyproject.toml +1 -1
- utility_tariffs_lib-0.2.0/tests/test_async_client.py +286 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0/utility_tariffs_lib.egg-info}/PKG-INFO +1 -1
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/utility_tariffs_lib.egg-info/SOURCES.txt +4 -0
- utility_tariffs_lib-0.1.0/lib/tariffstore/__init__.py +0 -13
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/LICENSE +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/README.md +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/README.md +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/__init__.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/py.typed +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/__init__.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/accumulator.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/bands.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/bill_days.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/billing.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/compile.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/config.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/defaults.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/definition.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/definition_parse.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/definition_savings.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/demand.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/engine.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/index.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/period_bill.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/periods.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/price_curve.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/price_series.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/price_stats.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/price_timeline.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/reprice.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/savings.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/slot_history.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/slots.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/stored_energy.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariff/windows.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffranking/__init__.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffranking/search.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/compile.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/index.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/layout.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/prices.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/tariffstore/schema.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/timeutil/__init__.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/lib/timeutil/real_time.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/setup.cfg +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_build.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_client.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_compile.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_eia861_map.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_end_to_end.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_index.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_lib_release_notes.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_packaging.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_price_client.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_price_format.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_prices.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_prices_comed.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_prices_energinet.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_prices_energy_charts.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_prices_energyzero.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_prices_runner.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_repo_rules.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_schema.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_dk.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_electric_ireland.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_gb.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_hq.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_ie.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_other.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_pt.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sources_us.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_stored_tariffs.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sync_core.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_sync_run.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_tariffs_through_lib.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/tests/test_urdb_convert.py +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/utility_tariffs_lib.egg-info/dependency_links.txt +0 -0
- {utility_tariffs_lib-0.1.0 → utility_tariffs_lib-0.2.0}/utility_tariffs_lib.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""The stored tariff format: its validator, its index, the compile step and a client.
|
|
2
|
+
|
|
3
|
+
Standard library only, and no import from `lib.tariff`, so the sync scripts, the checks and any
|
|
4
|
+
client share exactly one definition of the format. `Store` and `PriceStore` are the blocking
|
|
5
|
+
clients, and `AsyncStore` and `AsyncPriceStore` the same clients for an event loop.
|
|
6
|
+
"""
|
|
7
|
+
from .aclient import (
|
|
8
|
+
AsyncFolderCache,
|
|
9
|
+
AsyncMemoryCache,
|
|
10
|
+
AsyncPriceStore,
|
|
11
|
+
AsyncStore,
|
|
12
|
+
session_reader,
|
|
13
|
+
)
|
|
14
|
+
from .client import FolderCache, MemoryCache, Store, StoreError, folder_reader, url_reader
|
|
15
|
+
from .compile import CompileError, compile_area
|
|
16
|
+
from .price_client import PriceStore
|
|
17
|
+
from .schema import checksum, validate_country, validate_tariff
|
|
18
|
+
|
|
19
|
+
__all__ = ["AsyncFolderCache", "AsyncMemoryCache", "AsyncPriceStore", "AsyncStore",
|
|
20
|
+
"CompileError", "FolderCache", "MemoryCache", "PriceStore", "Store", "StoreError",
|
|
21
|
+
"checksum", "compile_area", "folder_reader", "session_reader", "url_reader",
|
|
22
|
+
"validate_country", "validate_tariff"]
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""The store client for an event loop. Standard library only.
|
|
2
|
+
|
|
3
|
+
`AsyncStore` and `AsyncPriceStore` answer the same questions as `Store` and `PriceStore`, and
|
|
4
|
+
apply the same rules, which live in `core` and `price_rules`. They differ in two ways: the reader
|
|
5
|
+
is a coroutine (`read(path) -> bytes`, awaited), and the cache is awaited, so a slow disk or
|
|
6
|
+
network never blocks the loop.
|
|
7
|
+
|
|
8
|
+
Nothing here imports an HTTP library. `session_reader` takes the web session the host already has
|
|
9
|
+
(aiohttp's `ClientSession`, or anything with the same `get`), so its proxy, certificate and
|
|
10
|
+
connection settings apply. `AsyncFolderCache` runs the file work in a worker thread.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
from collections.abc import Awaitable, Callable
|
|
16
|
+
from datetime import date, timedelta
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Protocol
|
|
19
|
+
|
|
20
|
+
from . import core, price_rules
|
|
21
|
+
from .client import USER_AGENT, FolderCache, MemoryCache
|
|
22
|
+
from .core import DAY_SECONDS, ROOT, StoreError, Want
|
|
23
|
+
from .index import block_of
|
|
24
|
+
from .schema import Doc
|
|
25
|
+
|
|
26
|
+
AsyncReader = Callable[[str], Awaitable[bytes]]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AsyncCache(Protocol):
|
|
30
|
+
async def get(self, path: str) -> bytes | None: ...
|
|
31
|
+
async def put(self, path: str, raw: bytes) -> None: ...
|
|
32
|
+
async def age(self, path: str) -> float | None: ...
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class AsyncMemoryCache:
|
|
36
|
+
"""Keeps what was read for as long as the object lives."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, cache: MemoryCache | None = None) -> None:
|
|
39
|
+
self._cache = cache if cache is not None else MemoryCache()
|
|
40
|
+
|
|
41
|
+
async def get(self, path: str) -> bytes | None:
|
|
42
|
+
return self._cache.get(path)
|
|
43
|
+
|
|
44
|
+
async def put(self, path: str, raw: bytes) -> None:
|
|
45
|
+
self._cache.put(path, raw)
|
|
46
|
+
|
|
47
|
+
async def age(self, path: str) -> float | None:
|
|
48
|
+
return self._cache.age(path)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class AsyncFolderCache:
|
|
52
|
+
"""`FolderCache`, with every file operation run in a worker thread."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, folder: Path | str) -> None:
|
|
55
|
+
self._cache = FolderCache(folder)
|
|
56
|
+
|
|
57
|
+
async def get(self, path: str) -> bytes | None:
|
|
58
|
+
return await asyncio.to_thread(self._cache.get, path)
|
|
59
|
+
|
|
60
|
+
async def put(self, path: str, raw: bytes) -> None:
|
|
61
|
+
await asyncio.to_thread(self._cache.put, path, raw)
|
|
62
|
+
|
|
63
|
+
async def age(self, path: str) -> float | None:
|
|
64
|
+
return await asyncio.to_thread(self._cache.age, path)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Session(Protocol):
|
|
68
|
+
"""The part of an aiohttp `ClientSession` the reader uses."""
|
|
69
|
+
|
|
70
|
+
def get(self, url: str, **kwargs: Any) -> Any: ...
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def session_reader(session: Session, base: str, *, timeout: float = 30,
|
|
74
|
+
errors: tuple[type[Exception], ...] = ()) -> AsyncReader:
|
|
75
|
+
"""A reader that fetches `base/path` through a web session. `base` must be https.
|
|
76
|
+
|
|
77
|
+
A failed request raises OSError, which the client turns into a StoreError and answers from
|
|
78
|
+
its cache. Pass the session library's own error class in `errors` (aiohttp: `ClientError`),
|
|
79
|
+
because it is not an OSError itself. A timeout is an OSError already."""
|
|
80
|
+
if not base.startswith("https://"):
|
|
81
|
+
raise ValueError("the store is read over https only")
|
|
82
|
+
|
|
83
|
+
async def read(path: str) -> bytes:
|
|
84
|
+
url = f"{base.rstrip('/')}/{path}"
|
|
85
|
+
try:
|
|
86
|
+
async with asyncio.timeout(timeout):
|
|
87
|
+
async with session.get(url, headers={"User-Agent": USER_AGENT}) as reply:
|
|
88
|
+
reply.raise_for_status()
|
|
89
|
+
body: bytes = await reply.read()
|
|
90
|
+
return body
|
|
91
|
+
except errors as error:
|
|
92
|
+
raise OSError(str(error)) from error
|
|
93
|
+
return read
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class _Reading:
|
|
97
|
+
"""What both async stores share: read through the reader, with the path checked first."""
|
|
98
|
+
|
|
99
|
+
def __init__(self, read: AsyncReader, cache: AsyncCache | None = None) -> None:
|
|
100
|
+
self._read = read
|
|
101
|
+
self._cache: AsyncCache = cache if cache is not None else AsyncMemoryCache()
|
|
102
|
+
|
|
103
|
+
async def _fetch(self, path: str) -> bytes:
|
|
104
|
+
try:
|
|
105
|
+
return await self._read(core.safe(path))
|
|
106
|
+
except OSError as error:
|
|
107
|
+
raise StoreError(f"{path}: cannot be read ({error})") from error
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class AsyncStore(_Reading):
|
|
111
|
+
"""The tariff store for an event loop. See `Store`: the questions, the answers and the
|
|
112
|
+
errors are the same, and each method is awaited."""
|
|
113
|
+
|
|
114
|
+
def __init__(self, read: AsyncReader, cache: AsyncCache | None = None, *,
|
|
115
|
+
max_age: float = DAY_SECONDS) -> None:
|
|
116
|
+
super().__init__(read, cache)
|
|
117
|
+
self._max_age = max_age
|
|
118
|
+
self._root: Doc | None = None
|
|
119
|
+
self._countries: dict[str, Doc] = {}
|
|
120
|
+
self._forced = False
|
|
121
|
+
self.offline = False
|
|
122
|
+
# One read at a time, so two tasks asking for the same index fetch it once.
|
|
123
|
+
self._lock = asyncio.Lock()
|
|
124
|
+
|
|
125
|
+
async def _checked(self, path: str, wanted: str) -> Doc:
|
|
126
|
+
kept = core.kept_doc(path, await self._cache.get(path), wanted)
|
|
127
|
+
if kept is not None:
|
|
128
|
+
return kept
|
|
129
|
+
raw = await self._fetch(path)
|
|
130
|
+
doc = core.fetched_doc(path, raw, wanted)
|
|
131
|
+
await self._cache.put(path, raw)
|
|
132
|
+
return doc
|
|
133
|
+
|
|
134
|
+
async def _read_root(self) -> Doc:
|
|
135
|
+
held, age = await self._cache.get(ROOT), await self._cache.age(ROOT)
|
|
136
|
+
if held is not None and not self._forced and age is not None and age <= self._max_age:
|
|
137
|
+
return core.root_of(held)
|
|
138
|
+
try:
|
|
139
|
+
raw = await self._fetch(ROOT)
|
|
140
|
+
root = core.root_of(raw)
|
|
141
|
+
except StoreError:
|
|
142
|
+
if held is None:
|
|
143
|
+
raise
|
|
144
|
+
self.offline = True
|
|
145
|
+
return core.root_of(held)
|
|
146
|
+
await self._cache.put(ROOT, raw)
|
|
147
|
+
self._forced = self.offline = False
|
|
148
|
+
return root
|
|
149
|
+
|
|
150
|
+
def refresh(self) -> None:
|
|
151
|
+
"""Read the countries index again on the next call, whatever its age."""
|
|
152
|
+
self._root, self._countries = None, {}
|
|
153
|
+
self._forced = True
|
|
154
|
+
|
|
155
|
+
async def countries(self) -> dict[str, Doc]:
|
|
156
|
+
async with self._lock:
|
|
157
|
+
if self._root is None:
|
|
158
|
+
root = await self._read_root()
|
|
159
|
+
core.countries_of(root)
|
|
160
|
+
self._root = root
|
|
161
|
+
return core.countries_of(self._root)
|
|
162
|
+
|
|
163
|
+
async def country(self, code: str) -> Doc:
|
|
164
|
+
code = code.upper()
|
|
165
|
+
if code not in self._countries:
|
|
166
|
+
entry = core.country_entry(await self.countries(), code)
|
|
167
|
+
doc = await self._checked(entry["path"], entry["checksum"])
|
|
168
|
+
self._countries.setdefault(code, doc)
|
|
169
|
+
return self._countries[code]
|
|
170
|
+
|
|
171
|
+
async def providers(self, code: str) -> dict[str, str]:
|
|
172
|
+
return core.providers_of(await self.country(code))
|
|
173
|
+
|
|
174
|
+
async def tariffs(self, code: str, provider: str, *, direction: str = "import",
|
|
175
|
+
on: date | None = None) -> dict[str, str]:
|
|
176
|
+
return core.tariffs_of(await self.country(code), code, provider, direction, on)
|
|
177
|
+
|
|
178
|
+
async def revision(self, code: str, provider: str, tariff: str,
|
|
179
|
+
on: date) -> tuple[int, Doc]:
|
|
180
|
+
return core.revision_of(await self.country(code), code, provider, tariff, on)
|
|
181
|
+
|
|
182
|
+
async def revision_starts(self, code: str, provider: str, tariff: str, # noqa: PLR0913
|
|
183
|
+
on: date, until: date) -> list[tuple[date, int]]:
|
|
184
|
+
return core.starts_of(await self.country(code), code, provider, tariff, on, until)
|
|
185
|
+
|
|
186
|
+
async def tariff(self, code: str, provider: str, tariff: str,
|
|
187
|
+
on: date) -> tuple[int, Doc]:
|
|
188
|
+
number, entry = await self.revision(code, provider, tariff, on)
|
|
189
|
+
path = core.tariff_path(code, provider, tariff, number)
|
|
190
|
+
return number, core.valid_tariff(path, await self._checked(path, entry["checksum"]))
|
|
191
|
+
|
|
192
|
+
async def fetch_country(self, code: str, *, on: date | None = None) -> int:
|
|
193
|
+
wanted = core.revisions_to_fetch(await self.country(code), code, on)
|
|
194
|
+
for path, wanted_sum in wanted:
|
|
195
|
+
await self._checked(path, wanted_sum)
|
|
196
|
+
return len(wanted)
|
|
197
|
+
|
|
198
|
+
async def is_holiday(self, code: str, region: str | None = None) -> Callable[[date], bool]:
|
|
199
|
+
return core.holiday_fn(await self.country(code), code, region)
|
|
200
|
+
|
|
201
|
+
async def areas(self, code: str, provider: str, tariff: str, on: date) -> dict[str, str]:
|
|
202
|
+
return core.areas_of((await self.tariff(code, provider, tariff, on))[1])
|
|
203
|
+
|
|
204
|
+
async def _export_file(self, code: str, target: str, on: date) -> Doc:
|
|
205
|
+
provider, tariff = core.split_export(target)
|
|
206
|
+
return core.export_file(target, (await self.tariff(code, provider, tariff, on))[1])
|
|
207
|
+
|
|
208
|
+
async def definition(self, code: str, provider: str, tariff: str, on: date, *, # noqa: PLR0913
|
|
209
|
+
area: str = "all", export: str | None = None) -> Doc:
|
|
210
|
+
code = code.upper()
|
|
211
|
+
number, doc = await self.tariff(code, provider, tariff, on)
|
|
212
|
+
target = export or doc.get("export")
|
|
213
|
+
export_doc = await self._export_file(code, target, on) if target and \
|
|
214
|
+
doc.get("direction") != "export" else None
|
|
215
|
+
rev = (await self.revision(code, provider, tariff, on))[1]
|
|
216
|
+
want = Want(code, provider, tariff, area, on)
|
|
217
|
+
return core.compiled(want, number, doc, rev, await self.country(code), export_doc)
|
|
218
|
+
|
|
219
|
+
async def _custom_export(self, code: str, export: str | Doc | None,
|
|
220
|
+
on: date) -> Doc | None:
|
|
221
|
+
if export is None:
|
|
222
|
+
return None
|
|
223
|
+
if isinstance(export, str):
|
|
224
|
+
provider, tariff = core.split_export(export)
|
|
225
|
+
found = (await self.tariff(code, provider, tariff, on))[1]
|
|
226
|
+
else:
|
|
227
|
+
found = core.custom_doc(export)
|
|
228
|
+
return core.custom_export_ok(found)
|
|
229
|
+
|
|
230
|
+
async def custom_definition(self, code: str, doc: Doc, on: date, *, # noqa: PLR0913
|
|
231
|
+
area: str = "all", export: str | Doc | None = None,
|
|
232
|
+
block: Doc | None = None) -> Doc:
|
|
233
|
+
code = code.upper()
|
|
234
|
+
full = core.custom_import_doc(doc)
|
|
235
|
+
block = block_of(await self.country(code)) if block is None else core.custom_block(block)
|
|
236
|
+
export_doc = await self._custom_export(code, export, on)
|
|
237
|
+
return core.custom_compiled(code, full, block, on, area, export_doc)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class AsyncPriceStore(_Reading):
|
|
241
|
+
"""Dynamic prices for an event loop. See `PriceStore`."""
|
|
242
|
+
|
|
243
|
+
async def latest(self, country: str, feed: str, area: str) -> date | None:
|
|
244
|
+
path = price_rules.index_path(country, feed, area)
|
|
245
|
+
return price_rules.latest_of(path, await self._fetch(path))
|
|
246
|
+
|
|
247
|
+
async def day(self, country: str, feed: str, area: str, day: date) -> Doc:
|
|
248
|
+
path = price_rules.day_path(country, feed, area, day)
|
|
249
|
+
held = await self._cache.get(path)
|
|
250
|
+
raw = held if held is not None else await self._fetch(path)
|
|
251
|
+
doc = price_rules.valid_day(path, raw, feed, area, day)
|
|
252
|
+
if held is None:
|
|
253
|
+
await self._cache.put(path, raw)
|
|
254
|
+
return doc
|
|
255
|
+
|
|
256
|
+
async def days(self, country: str, feed: str, area: str, first: date, # noqa: PLR0913
|
|
257
|
+
last: date) -> tuple[list[Doc], list[date]]:
|
|
258
|
+
held: list[Doc] = []
|
|
259
|
+
missing: list[date] = []
|
|
260
|
+
day = first
|
|
261
|
+
while day <= last:
|
|
262
|
+
try:
|
|
263
|
+
held.append(await self.day(country, feed, area, day))
|
|
264
|
+
except StoreError:
|
|
265
|
+
missing.append(day)
|
|
266
|
+
day += timedelta(days=1)
|
|
267
|
+
return held, missing
|
|
@@ -19,30 +19,31 @@ countries index has an age. `custom_definition()` prices a tariff the store does
|
|
|
19
19
|
"""
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
-
import json
|
|
23
22
|
import os
|
|
24
|
-
import re
|
|
25
23
|
import time
|
|
26
24
|
import urllib.request
|
|
27
25
|
from collections.abc import Callable
|
|
28
26
|
from datetime import date
|
|
29
27
|
from pathlib import Path
|
|
30
28
|
|
|
31
|
-
from .
|
|
32
|
-
from .
|
|
33
|
-
|
|
29
|
+
from . import core
|
|
30
|
+
from .core import (
|
|
31
|
+
CUSTOM,
|
|
32
|
+
CUSTOM_SOURCE,
|
|
33
|
+
DAY_SECONDS,
|
|
34
|
+
OPEN,
|
|
35
|
+
ROOT,
|
|
36
|
+
SAFE_PATH,
|
|
37
|
+
StoreError,
|
|
38
|
+
Want,
|
|
39
|
+
)
|
|
40
|
+
from .index import block_of
|
|
41
|
+
from .schema import Doc, checksum
|
|
42
|
+
|
|
43
|
+
__all__ = ["CUSTOM", "CUSTOM_SOURCE", "DAY_SECONDS", "OPEN", "ROOT", "SAFE_PATH", "FolderCache",
|
|
44
|
+
"MemoryCache", "Store", "StoreError", "checksum", "folder_reader", "url_reader"]
|
|
34
45
|
|
|
35
|
-
OPEN = "9999-12-31"
|
|
36
46
|
USER_AGENT = "tariffstore-client/1"
|
|
37
|
-
DAY_SECONDS = 86400.0
|
|
38
|
-
ROOT = "index.json"
|
|
39
|
-
SAFE_PATH = re.compile(r"^[a-z0-9-]+(/[a-z0-9-]+)*\.json$")
|
|
40
|
-
CUSTOM = "custom"
|
|
41
|
-
CUSTOM_SOURCE = {"id": CUSTOM, "url": "https://example.invalid/custom"}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
class StoreError(LookupError):
|
|
45
|
-
"""Something asked for is not in the store, or a file is not the one its index describes."""
|
|
46
47
|
|
|
47
48
|
|
|
48
49
|
def folder_reader(tariffs: Path) -> Callable[[str], bytes]:
|
|
@@ -64,10 +65,7 @@ def url_reader(base: str, timeout: float = 30) -> Callable[[str], bytes]:
|
|
|
64
65
|
return read
|
|
65
66
|
|
|
66
67
|
|
|
67
|
-
|
|
68
|
-
if not SAFE_PATH.match(path):
|
|
69
|
-
raise StoreError(f"{path}: not a path inside the store")
|
|
70
|
-
return path
|
|
68
|
+
_safe = core.safe
|
|
71
69
|
|
|
72
70
|
|
|
73
71
|
class MemoryCache:
|
|
@@ -124,7 +122,8 @@ class Store:
|
|
|
124
122
|
|
|
125
123
|
The countries index is read again once it is older than `max_age` seconds. Everything below
|
|
126
124
|
it is read again only when its checksum in the level above has changed. When the store cannot
|
|
127
|
-
be reached, what the cache holds is used and `offline` is true.
|
|
125
|
+
be reached, what the cache holds is used and `offline` is true. `AsyncStore` is the same
|
|
126
|
+
client for an event loop, and both apply the rules in `core`."""
|
|
128
127
|
|
|
129
128
|
def __init__(self, read: Callable[[str], bytes],
|
|
130
129
|
cache: MemoryCache | FolderCache | None = None, *,
|
|
@@ -142,53 +141,36 @@ class Store:
|
|
|
142
141
|
"""A store read from an https address, with a cache in `folder`."""
|
|
143
142
|
return cls(url_reader(base), FolderCache(folder), max_age=max_age)
|
|
144
143
|
|
|
145
|
-
def _parsed(self, path: str, raw: bytes) -> Doc:
|
|
146
|
-
try:
|
|
147
|
-
doc = json.loads(raw)
|
|
148
|
-
except ValueError as error:
|
|
149
|
-
raise StoreError(f"{path}: not JSON ({error})") from error
|
|
150
|
-
if not isinstance(doc, dict):
|
|
151
|
-
raise StoreError(f"{path}: not a JSON object")
|
|
152
|
-
return doc
|
|
153
|
-
|
|
154
144
|
def _fetch(self, path: str) -> bytes:
|
|
155
145
|
try:
|
|
156
|
-
return self._read(
|
|
146
|
+
return self._read(core.safe(path))
|
|
157
147
|
except OSError as error:
|
|
158
148
|
raise StoreError(f"{path}: cannot be read ({error})") from error
|
|
159
149
|
|
|
160
150
|
def _checked(self, path: str, wanted: str) -> Doc:
|
|
161
151
|
"""The file at `path` whose checksum is `wanted`: from the cache when it holds that very
|
|
162
152
|
file, else from the store, and then kept."""
|
|
163
|
-
|
|
164
|
-
if
|
|
165
|
-
kept
|
|
166
|
-
try:
|
|
167
|
-
kept = self._parsed(path, held)
|
|
168
|
-
except StoreError:
|
|
169
|
-
kept = None
|
|
170
|
-
if kept is not None and checksum(kept) == wanted:
|
|
171
|
-
return kept
|
|
153
|
+
kept = core.kept_doc(path, self._cache.get(path), wanted)
|
|
154
|
+
if kept is not None:
|
|
155
|
+
return kept
|
|
172
156
|
raw = self._fetch(path)
|
|
173
|
-
doc =
|
|
174
|
-
if checksum(doc) != wanted:
|
|
175
|
-
raise StoreError(f"{path}: does not match its checksum")
|
|
157
|
+
doc = core.fetched_doc(path, raw, wanted)
|
|
176
158
|
self._cache.put(path, raw)
|
|
177
159
|
return doc
|
|
178
160
|
|
|
179
161
|
def _read_root(self) -> Doc:
|
|
180
162
|
held, age = self._cache.get(ROOT), self._cache.age(ROOT)
|
|
181
163
|
if held is not None and not self._forced and age is not None and age <= self._max_age:
|
|
182
|
-
return
|
|
164
|
+
return core.root_of(held)
|
|
183
165
|
try:
|
|
184
166
|
raw = self._fetch(ROOT)
|
|
185
|
-
root =
|
|
167
|
+
root = core.root_of(raw)
|
|
186
168
|
except StoreError:
|
|
187
169
|
# Unreachable, or an answer that is not the index (a sign-in page, say).
|
|
188
170
|
if held is None:
|
|
189
171
|
raise
|
|
190
172
|
self.offline = True
|
|
191
|
-
return
|
|
173
|
+
return core.root_of(held)
|
|
192
174
|
self._cache.put(ROOT, raw)
|
|
193
175
|
self._forced = self.offline = False
|
|
194
176
|
return root
|
|
@@ -205,61 +187,36 @@ class Store:
|
|
|
205
187
|
"""{country code: {availability, reason, providers, tariffs}} for every country listed."""
|
|
206
188
|
if self._root is None:
|
|
207
189
|
root = self._read_root()
|
|
208
|
-
|
|
209
|
-
raise StoreError(f"index.json: not a schema {SCHEMA} countries index")
|
|
190
|
+
core.countries_of(root)
|
|
210
191
|
self._root = root
|
|
211
|
-
|
|
212
|
-
return countries
|
|
192
|
+
return core.countries_of(self._root)
|
|
213
193
|
|
|
214
194
|
def country(self, code: str) -> Doc:
|
|
215
195
|
"""The country's index, checked against the countries index: its values and its tariffs."""
|
|
216
196
|
code = code.upper()
|
|
217
197
|
if code not in self._countries:
|
|
218
|
-
entry = self.countries()
|
|
219
|
-
if entry is None or "path" not in entry:
|
|
220
|
-
reason = f": {entry['reason']}" if entry else ""
|
|
221
|
-
raise StoreError(f"{code}: no tariffs are held{reason}")
|
|
198
|
+
entry = core.country_entry(self.countries(), code)
|
|
222
199
|
self._countries[code] = self._checked(entry["path"], entry["checksum"])
|
|
223
200
|
return self._countries[code]
|
|
224
201
|
|
|
225
202
|
def providers(self, code: str) -> dict[str, str]:
|
|
226
203
|
"""{provider id: provider name} in one country."""
|
|
227
|
-
return
|
|
204
|
+
return core.providers_of(self.country(code))
|
|
228
205
|
|
|
229
206
|
def tariffs(self, code: str, provider: str, *, direction: str = "import",
|
|
230
207
|
on: date | None = None) -> dict[str, str]:
|
|
231
208
|
"""{tariff id: name} for one provider. `on` keeps only tariffs with a revision in force."""
|
|
232
|
-
|
|
233
|
-
entries = self.country(code)["providers"][provider]["tariffs"]
|
|
234
|
-
except KeyError:
|
|
235
|
-
raise StoreError(f"{code}: no provider {provider}") from None
|
|
236
|
-
return {key: t["name"] for key, t in entries.items()
|
|
237
|
-
if t.get("direction", "import") == direction
|
|
238
|
-
and (on is None or _in_force(t, on) is not None)}
|
|
209
|
+
return core.tariffs_of(self.country(code), code, provider, direction, on)
|
|
239
210
|
|
|
240
211
|
def revision(self, code: str, provider: str, tariff: str, on: date) -> tuple[int, Doc]:
|
|
241
212
|
"""(revision number, its index entry) in force on a day."""
|
|
242
|
-
|
|
243
|
-
entry = self.country(code)["providers"][provider]["tariffs"][tariff]
|
|
244
|
-
except KeyError:
|
|
245
|
-
raise StoreError(f"{code}: no tariff {provider}/{tariff}") from None
|
|
246
|
-
number = _in_force(entry, on)
|
|
247
|
-
if number is None:
|
|
248
|
-
raise StoreError(f"{code}/{provider}/{tariff}: no revision is in force on {on}")
|
|
249
|
-
return number, entry["revisions"][str(number)]
|
|
213
|
+
return core.revision_of(self.country(code), code, provider, tariff, on)
|
|
250
214
|
|
|
251
215
|
def revision_starts(self, code: str, provider: str, tariff: str, on: date, # noqa: PLR0913
|
|
252
216
|
until: date) -> list[tuple[date, int]]:
|
|
253
217
|
"""(first day, revision number) of every revision that starts after `on`, up to and
|
|
254
218
|
including `until`, earliest first. The revision in force on `on` is not listed."""
|
|
255
|
-
|
|
256
|
-
entry = self.country(code)["providers"][provider]["tariffs"][tariff]
|
|
257
|
-
except KeyError:
|
|
258
|
-
raise StoreError(f"{code}: no tariff {provider}/{tariff}") from None
|
|
259
|
-
found = [(date.fromisoformat(r["effective_from"]), int(number))
|
|
260
|
-
for number, r in entry["revisions"].items()
|
|
261
|
-
if on < date.fromisoformat(r["effective_from"]) <= until]
|
|
262
|
-
return sorted(found)
|
|
219
|
+
return core.starts_of(self.country(code), code, provider, tariff, on, until)
|
|
263
220
|
|
|
264
221
|
# Fetching
|
|
265
222
|
|
|
@@ -267,27 +224,17 @@ class Store:
|
|
|
267
224
|
"""(revision number, tariff file) in force on a day, checked against its checksum and
|
|
268
225
|
valid."""
|
|
269
226
|
number, entry = self.revision(code, provider, tariff, on)
|
|
270
|
-
path =
|
|
271
|
-
|
|
272
|
-
errors = validate_tariff(doc)
|
|
273
|
-
if errors:
|
|
274
|
-
raise StoreError(f"{path}: " + "; ".join(errors))
|
|
275
|
-
return number, doc
|
|
227
|
+
path = core.tariff_path(code, provider, tariff, number)
|
|
228
|
+
return number, core.valid_tariff(path, self._checked(path, entry["checksum"]))
|
|
276
229
|
|
|
277
230
|
def fetch_country(self, code: str, *, on: date | None = None) -> int:
|
|
278
231
|
"""Bring a whole country into the cache, for comparing many tariffs or for use with no
|
|
279
232
|
network. `on` keeps it to the revisions in force on that day. Returns the number of
|
|
280
233
|
tariff files now held."""
|
|
281
|
-
|
|
282
|
-
for
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
if on is not None and _in_force(listed, on) != int(number):
|
|
286
|
-
continue
|
|
287
|
-
path = f"{code.lower()}/{provider}/{tariff}/{number}.json"
|
|
288
|
-
self._checked(path, revision["checksum"])
|
|
289
|
-
count += 1
|
|
290
|
-
return count
|
|
234
|
+
wanted = core.revisions_to_fetch(self.country(code), code, on)
|
|
235
|
+
for path, wanted_sum in wanted:
|
|
236
|
+
self._checked(path, wanted_sum)
|
|
237
|
+
return len(wanted)
|
|
291
238
|
|
|
292
239
|
def is_holiday(self, code: str, region: str | None = None) -> Callable[[date], bool]:
|
|
293
240
|
"""Whether a day is a public holiday in the country, for the engine to price a band that
|
|
@@ -296,28 +243,16 @@ class Store:
|
|
|
296
243
|
|
|
297
244
|
A country that holds no holidays is refused here. A day in a year that is not held is
|
|
298
245
|
refused when it is asked about, because a missing year is not a year without holidays."""
|
|
299
|
-
|
|
300
|
-
if years is None:
|
|
301
|
-
raise StoreError(f"{code.upper()}: no public holidays are held")
|
|
302
|
-
|
|
303
|
-
def is_holiday(day: date) -> bool:
|
|
304
|
-
if day.year not in years:
|
|
305
|
-
raise StoreError(f"{code.upper()}: no public holidays are held for {day.year}")
|
|
306
|
-
return day.isoformat() in years[day.year]
|
|
307
|
-
return is_holiday
|
|
246
|
+
return core.holiday_fn(self.country(code), code, region)
|
|
308
247
|
|
|
309
248
|
def areas(self, code: str, provider: str, tariff: str, on: date) -> dict[str, str]:
|
|
310
249
|
"""{area key: label} a customer chooses from. A tariff with one price has the area `all`."""
|
|
311
|
-
|
|
312
|
-
return {key: row.get("label", doc["name"]) for key, row in doc["areas"].items()}
|
|
250
|
+
return core.areas_of(self.tariff(code, provider, tariff, on)[1])
|
|
313
251
|
|
|
314
252
|
def _export_file(self, code: str, target: str, on: date) -> Doc:
|
|
315
253
|
"""The stored export tariff named `provider-id/tariff-id`."""
|
|
316
|
-
provider,
|
|
317
|
-
|
|
318
|
-
if doc.get("direction") != "export":
|
|
319
|
-
raise StoreError(f"{target}: not an export tariff")
|
|
320
|
-
return doc
|
|
254
|
+
provider, tariff = core.split_export(target)
|
|
255
|
+
return core.export_file(target, self.tariff(code, provider, tariff, on)[1])
|
|
321
256
|
|
|
322
257
|
def definition(self, code: str, provider: str, tariff: str, on: date, *, # noqa: PLR0913
|
|
323
258
|
area: str = "all", export: str | None = None) -> Doc:
|
|
@@ -327,16 +262,12 @@ class Store:
|
|
|
327
262
|
out, the export tariff the import tariff itself names is used, if it names one."""
|
|
328
263
|
code = code.upper()
|
|
329
264
|
number, doc = self.tariff(code, provider, tariff, on)
|
|
330
|
-
if doc.get("direction") == "export":
|
|
331
|
-
raise StoreError(f"{provider}/{tariff}: an export tariff is paired with an import "
|
|
332
|
-
"tariff, it has no bill of its own")
|
|
333
265
|
target = export or doc.get("export")
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
return
|
|
339
|
-
on=on.isoformat(), export_file=export_file)
|
|
266
|
+
export_doc = self._export_file(code, target, on) if target and \
|
|
267
|
+
doc.get("direction") != "export" else None
|
|
268
|
+
rev = self.revision(code, provider, tariff, on)[1]
|
|
269
|
+
want = Want(code, provider, tariff, area, on)
|
|
270
|
+
return core.compiled(want, number, doc, rev, self.country(code), export_doc)
|
|
340
271
|
|
|
341
272
|
# A tariff the store does not hold
|
|
342
273
|
|
|
@@ -344,13 +275,11 @@ class Store:
|
|
|
344
275
|
if export is None:
|
|
345
276
|
return None
|
|
346
277
|
if isinstance(export, str):
|
|
347
|
-
provider,
|
|
278
|
+
provider, tariff = core.split_export(export)
|
|
348
279
|
found = self.tariff(code, provider, tariff, on)[1]
|
|
349
280
|
else:
|
|
350
|
-
found =
|
|
351
|
-
|
|
352
|
-
raise StoreError("custom tariff: the export given is not an export tariff")
|
|
353
|
-
return found
|
|
281
|
+
found = core.custom_doc(export)
|
|
282
|
+
return core.custom_export_ok(found)
|
|
354
283
|
|
|
355
284
|
def custom_definition(self, code: str, doc: Doc, on: date, *, # noqa: PLR0913
|
|
356
285
|
area: str = "all", export: str | Doc | None = None,
|
|
@@ -363,44 +292,7 @@ class Store:
|
|
|
363
292
|
file. For a country the store does not hold, pass its values as `block`, written like the
|
|
364
293
|
top of a country index. The document is checked by the same rules as a stored tariff."""
|
|
365
294
|
code = code.upper()
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
if block is None:
|
|
371
|
-
block = block_of(self.country(code))
|
|
372
|
-
else:
|
|
373
|
-
_refuse("custom country", validate_country(block))
|
|
374
|
-
export_file = self._custom_export(code, export, on)
|
|
375
|
-
meta = {"country": code, "provider": CUSTOM, "tariff": CUSTOM, "revision": 1,
|
|
376
|
-
"effective_to": doc.get("effective_to")}
|
|
377
|
-
try:
|
|
378
|
-
out = compile_area(doc, area, block=block, meta=meta, on=on.isoformat(),
|
|
379
|
-
export_file=export_file)
|
|
380
|
-
except CompileError as error:
|
|
381
|
-
raise StoreError(f"custom tariff: {error}") from error
|
|
382
|
-
if doc["source"] == CUSTOM_SOURCE:
|
|
383
|
-
out["source_url"] = None
|
|
384
|
-
return out
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
def _refuse(what: str, errors: list[str]) -> None:
|
|
388
|
-
if errors:
|
|
389
|
-
raise StoreError(f"{what}: " + "; ".join(errors))
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
def _custom(doc: Doc) -> Doc:
|
|
393
|
-
"""A supplied tariff with the fields a person need not type filled in, then validated."""
|
|
394
|
-
if not isinstance(doc, dict):
|
|
395
|
-
raise StoreError("custom tariff: not a JSON object")
|
|
396
|
-
full = {"schema": 1, "provider": "Custom", "source": dict(CUSTOM_SOURCE), **doc}
|
|
397
|
-
_refuse("custom tariff", validate_tariff(full))
|
|
398
|
-
return full
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
def _in_force(entry: Doc, on: date) -> int | None:
|
|
402
|
-
day = on.isoformat()
|
|
403
|
-
for number, revision in entry["revisions"].items():
|
|
404
|
-
if revision["effective_from"] <= day < (revision["effective_to"] or OPEN):
|
|
405
|
-
return int(number)
|
|
406
|
-
return None
|
|
295
|
+
full = core.custom_import_doc(doc)
|
|
296
|
+
block = block_of(self.country(code)) if block is None else core.custom_block(block)
|
|
297
|
+
export_doc = self._custom_export(code, export, on)
|
|
298
|
+
return core.custom_compiled(code, full, block, on, area, export_doc)
|