python-esios 2.0.1__tar.gz → 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {python_esios-2.0.1 → python_esios-2.0.2}/PKG-INFO +1 -1
- {python_esios-2.0.1 → python_esios-2.0.2}/pyproject.toml +1 -1
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/.agents/skills/esios/SKILL.md +49 -4
- python_esios-2.0.2/src/esios/cli/archives.py +217 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/managers/indicators.py +5 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/processing/i90.py +25 -4
- python_esios-2.0.2/tests/test_i90.py +167 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_managers.py +47 -0
- python_esios-2.0.1/src/esios/cli/archives.py +0 -66
- {python_esios-2.0.1 → python_esios-2.0.2}/.github/workflows/publish.yml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/.gitignore +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/CHANGELOG.md +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/LICENSE +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/README.md +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/.gitignore +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/01_Quickstart/01_Setup and First Query.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/01_Search Indicators.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/02_Historical Data.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/03_Multi-Geography Indicators.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/04_Compare Indicators.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/05_Market Prices.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/06_Generation and Demand.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/03_Archives/01_Download Archives.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/03_Archives/02_I90 Settlement Files.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/04_Caching/01_Cache Management.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/05_Advanced/01_Ad-hoc Pandas Expressions.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/05_Advanced/02_Async Client.ipynb +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/01_Quickstart/01_Setup and First Query.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/01_Search Indicators.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/02_Historical Data.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/03_Multi-Geography Indicators.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/04_Compare Indicators.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/05_Market Prices.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/06_Generation and Demand.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/03_Archives/01_Download Archives.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/03_Archives/02_I90 Settlement Files.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/04_Caching/01_Cache Management.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/05_Advanced/01_Ad-hoc Pandas Expressions.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/05_Advanced/02_Async Client.yaml +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/examples/generate.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/async_client.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cache.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/app.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/cache_cmd.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/config.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/config_cmd.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/exec_cmd.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/cli/indicators.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/client.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/constants.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/data/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/data/catalogs/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/data/catalogs/archives/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/data/catalogs/archives/catalog.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/data/catalogs/archives/refresh.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/exceptions.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/managers/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/managers/archives.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/managers/base.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/managers/offer_indicators.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/models/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/models/archive.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/models/indicator.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/models/offer_indicator.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/processing/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/processing/dataframes.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/src/esios/processing/zip.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/__init__.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/conftest.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_cache.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_client.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_dataframes.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_exceptions.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_models.py +0 -0
- {python_esios-2.0.1 → python_esios-2.0.2}/tests/test_zip.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-esios
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.2
|
|
4
4
|
Summary: A Python wrapper for the ESIOS API (Spanish electricity market)
|
|
5
5
|
Project-URL: Homepage, https://github.com/datons/python-esios
|
|
6
6
|
Project-URL: Repository, https://github.com/datons/python-esios
|
|
@@ -39,8 +39,22 @@ esios indicators exec 600 -s 2025-01-01 -e 2025-01-31 --expr "df.resample('D').m
|
|
|
39
39
|
### Archives
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
|
+
# List all available archives
|
|
42
43
|
esios archives list
|
|
43
|
-
|
|
44
|
+
|
|
45
|
+
# Download archive files
|
|
46
|
+
esios archives download 34 --start 2025-05-01 --end 2025-05-31 --output ./data
|
|
47
|
+
esios archives download 34 --date 2025-06-01
|
|
48
|
+
|
|
49
|
+
# List sheets (table of contents) in an I90 file
|
|
50
|
+
esios archives sheets 34 --date 2025-06-01
|
|
51
|
+
|
|
52
|
+
# Parse and query archive data (like indicators exec but for archives)
|
|
53
|
+
esios archives exec 34 --sheet I90DIA03 --date 2025-06-01
|
|
54
|
+
esios archives exec 34 --sheet I90DIA03 --date 2025-06-01 -x "df.describe()"
|
|
55
|
+
esios archives exec 34 --sheet I90DIA03 -s 2025-05-05 -e 2025-06-08 \
|
|
56
|
+
-x "df[df['Sentido']=='Bajar'].groupby('Unidad de Programación')['value'].sum().sort_values()"
|
|
57
|
+
esios archives exec 34 --sheet I90DIA26 --date 2025-06-01 --format csv --output pbf.csv
|
|
44
58
|
```
|
|
45
59
|
|
|
46
60
|
### Cache Management
|
|
@@ -132,11 +146,42 @@ df = client.indicators.compare([600, 10034, 10035], "2025-01-01", "2025-01-07")
|
|
|
132
146
|
|
|
133
147
|
## I90 Settlement Files
|
|
134
148
|
|
|
149
|
+
### CLI (quickest path)
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
# Discover available sheets
|
|
153
|
+
esios archives sheets 34 --date 2025-06-01
|
|
154
|
+
|
|
155
|
+
# Key I90DIA sheets:
|
|
156
|
+
# I90DIA03 — Restricciones en el Mercado Diario (curtailment)
|
|
157
|
+
# I90DIA08 — Restricciones en Tiempo Real
|
|
158
|
+
# I90DIA26 — Programa Base de Funcionamiento (PBF, generation program)
|
|
159
|
+
# I90DIA01 — Programa PVP
|
|
160
|
+
# I90DIA07 — Regulación Terciaria (mFRR)
|
|
161
|
+
|
|
162
|
+
# Total curtailment by direction
|
|
163
|
+
esios archives exec 34 --sheet I90DIA03 --date 2025-06-01 \
|
|
164
|
+
-x "df.groupby('Sentido')['value'].sum()"
|
|
165
|
+
|
|
166
|
+
# Multi-day curtailment analysis
|
|
167
|
+
esios archives exec 34 --sheet I90DIA03 -s 2025-05-05 -e 2025-06-08 \
|
|
168
|
+
-x "df[df['Sentido']=='Bajar'].groupby('Unidad de Programación')['value'].sum().sort_values().head(20)"
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Python library
|
|
172
|
+
|
|
135
173
|
```python
|
|
136
|
-
from esios
|
|
174
|
+
from esios import ESIOSClient
|
|
175
|
+
from esios.processing.i90 import I90Book
|
|
176
|
+
|
|
177
|
+
# Download + parse in one step
|
|
178
|
+
client = ESIOSClient()
|
|
179
|
+
archive = client.archives.get(34)
|
|
180
|
+
books = I90Book.from_archive(archive, start="2025-05-05", end="2025-06-08")
|
|
137
181
|
|
|
138
|
-
|
|
139
|
-
|
|
182
|
+
# Access a sheet
|
|
183
|
+
book = books[0]
|
|
184
|
+
sheet = book["I90DIA03"]
|
|
140
185
|
df = sheet.df # Preprocessed DataFrame with datetime index
|
|
141
186
|
print(sheet.frequency) # "hourly" or "hourly-quarterly"
|
|
142
187
|
```
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""CLI subcommands for archives."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
archives_app = typer.Typer(no_args_is_help=True)
|
|
12
|
+
console = Console()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@archives_app.command("list")
|
|
16
|
+
def list_archives(
|
|
17
|
+
token: Optional[str] = typer.Option(None, "--token", "-t"),
|
|
18
|
+
format: str = typer.Option("table", "--format", "-f"),
|
|
19
|
+
):
|
|
20
|
+
"""List all available ESIOS archives."""
|
|
21
|
+
from esios.cli.app import get_client
|
|
22
|
+
|
|
23
|
+
client = get_client(token)
|
|
24
|
+
df = client.archives.list()
|
|
25
|
+
|
|
26
|
+
if format == "csv":
|
|
27
|
+
typer.echo(df.to_csv())
|
|
28
|
+
elif format == "json":
|
|
29
|
+
typer.echo(df.to_json(orient="records", indent=2))
|
|
30
|
+
else:
|
|
31
|
+
table = Table(title="ESIOS Archives")
|
|
32
|
+
cols = list(df.columns)[:8]
|
|
33
|
+
for col in cols:
|
|
34
|
+
table.add_column(str(col))
|
|
35
|
+
for _, row in df.head(50).iterrows():
|
|
36
|
+
table.add_row(*[str(row[c]) for c in cols])
|
|
37
|
+
if len(df) > 50:
|
|
38
|
+
table.caption = f"Showing 50 of {len(df)} rows"
|
|
39
|
+
console.print(table)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@archives_app.command("download")
|
|
43
|
+
def download_archive(
|
|
44
|
+
archive_id: int = typer.Argument(..., help="Archive ID"),
|
|
45
|
+
start: Optional[str] = typer.Option(None, "--start", "-s", help="Start date (YYYY-MM-DD)"),
|
|
46
|
+
end: Optional[str] = typer.Option(None, "--end", "-e", help="End date (YYYY-MM-DD)"),
|
|
47
|
+
date: Optional[str] = typer.Option(None, "--date", "-d", help="Single date (YYYY-MM-DD)"),
|
|
48
|
+
output: str = typer.Option(".", "--output", "-o", help="Output directory"),
|
|
49
|
+
token: Optional[str] = typer.Option(None, "--token", "-t"),
|
|
50
|
+
):
|
|
51
|
+
"""Download an archive for a date or date range."""
|
|
52
|
+
from esios.cli.app import get_client
|
|
53
|
+
|
|
54
|
+
if not date and not (start and end):
|
|
55
|
+
typer.echo("Provide --date or both --start and --end.", err=True)
|
|
56
|
+
raise typer.Exit(1)
|
|
57
|
+
|
|
58
|
+
client = get_client(token)
|
|
59
|
+
client.archives.download(
|
|
60
|
+
archive_id,
|
|
61
|
+
start=start,
|
|
62
|
+
end=end,
|
|
63
|
+
date=date,
|
|
64
|
+
output_dir=output,
|
|
65
|
+
)
|
|
66
|
+
typer.echo(f"Download complete → {output}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _download_files(
|
|
70
|
+
archive_id: int,
|
|
71
|
+
token: str | None,
|
|
72
|
+
date: str | None,
|
|
73
|
+
start: str | None,
|
|
74
|
+
end: str | None,
|
|
75
|
+
) -> list:
|
|
76
|
+
"""Download archive files (shared by sheets and exec commands)."""
|
|
77
|
+
from esios.cli.app import get_client
|
|
78
|
+
|
|
79
|
+
if not date and not (start and end):
|
|
80
|
+
typer.echo("Provide --date or both --start and --end.", err=True)
|
|
81
|
+
raise typer.Exit(1)
|
|
82
|
+
|
|
83
|
+
client = get_client(token)
|
|
84
|
+
|
|
85
|
+
if date:
|
|
86
|
+
files = client.archives.download(archive_id, date=date)
|
|
87
|
+
else:
|
|
88
|
+
files = client.archives.download(archive_id, start=start, end=end)
|
|
89
|
+
|
|
90
|
+
if not files:
|
|
91
|
+
typer.echo("No files found for the given date range.", err=True)
|
|
92
|
+
raise typer.Exit(1)
|
|
93
|
+
|
|
94
|
+
return files
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@archives_app.command("sheets")
|
|
98
|
+
def sheets(
|
|
99
|
+
archive_id: int = typer.Argument(..., help="Archive ID (e.g. 34 for I90DIA)"),
|
|
100
|
+
date: Optional[str] = typer.Option(None, "--date", "-d", help="Single date (YYYY-MM-DD)"),
|
|
101
|
+
start: Optional[str] = typer.Option(None, "--start", "-s", help="Start date (YYYY-MM-DD)"),
|
|
102
|
+
end: Optional[str] = typer.Option(None, "--end", "-e", help="End date (YYYY-MM-DD)"),
|
|
103
|
+
token: Optional[str] = typer.Option(None, "--token", "-t"),
|
|
104
|
+
):
|
|
105
|
+
"""List sheets (table of contents) in an archive file.
|
|
106
|
+
|
|
107
|
+
Downloads and parses the first file to show available sheets.
|
|
108
|
+
|
|
109
|
+
\b
|
|
110
|
+
Examples:
|
|
111
|
+
esios archives sheets 34 --date 2025-06-01
|
|
112
|
+
esios archives sheets 34 --start 2025-06-01 --end 2025-06-01
|
|
113
|
+
"""
|
|
114
|
+
from esios.processing.i90 import I90Book
|
|
115
|
+
|
|
116
|
+
files = _download_files(archive_id, token, date, start, end)
|
|
117
|
+
|
|
118
|
+
try:
|
|
119
|
+
book = I90Book(files[0])
|
|
120
|
+
except Exception as exc:
|
|
121
|
+
typer.echo(f"Error parsing {files[0].name}: {exc}", err=True)
|
|
122
|
+
raise typer.Exit(1)
|
|
123
|
+
|
|
124
|
+
table = Table(title=f"Sheets in {files[0].name}")
|
|
125
|
+
table.add_column("Sheet", style="cyan")
|
|
126
|
+
table.add_column("Description")
|
|
127
|
+
|
|
128
|
+
for sheet_name, description in book.table_of_contents.items():
|
|
129
|
+
if not isinstance(sheet_name, str) or not sheet_name.strip():
|
|
130
|
+
continue
|
|
131
|
+
table.add_row(str(sheet_name), str(description))
|
|
132
|
+
|
|
133
|
+
console.print(table)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@archives_app.command("exec")
|
|
137
|
+
def exec_archive(
|
|
138
|
+
archive_id: int = typer.Argument(..., help="Archive ID (e.g. 34 for I90DIA)"),
|
|
139
|
+
sheet: str = typer.Option(..., "--sheet", help="Sheet name (e.g. I90DIA03, I90DIA26)"),
|
|
140
|
+
date: Optional[str] = typer.Option(None, "--date", "-d", help="Single date (YYYY-MM-DD)"),
|
|
141
|
+
start: Optional[str] = typer.Option(None, "--start", "-s", help="Start date (YYYY-MM-DD)"),
|
|
142
|
+
end: Optional[str] = typer.Option(None, "--end", "-e", help="End date (YYYY-MM-DD)"),
|
|
143
|
+
expr: str = typer.Option("df", "--expr", "-x", help="Python expression to evaluate (df, pd, np available)"),
|
|
144
|
+
format: str = typer.Option("table", "--format", "-f", help="Output format: table, csv, json"),
|
|
145
|
+
output: Optional[str] = typer.Option(None, "--output", "-o", help="Output file path"),
|
|
146
|
+
token: Optional[str] = typer.Option(None, "--token", "-t"),
|
|
147
|
+
):
|
|
148
|
+
"""Parse archive files and evaluate a Python expression on the data.
|
|
149
|
+
|
|
150
|
+
Downloads archive files (cached), parses the specified sheet from each
|
|
151
|
+
file using I90Book, concatenates the DataFrames, and evaluates the
|
|
152
|
+
expression. The parsed data is available as `df` (pandas DataFrame).
|
|
153
|
+
`pd` (pandas) and `np` (numpy) are also available.
|
|
154
|
+
|
|
155
|
+
\b
|
|
156
|
+
Examples:
|
|
157
|
+
# Show curtailment data for a single day
|
|
158
|
+
esios archives exec 34 --sheet I90DIA03 --date 2025-06-01
|
|
159
|
+
|
|
160
|
+
# Curtailment by technology over a month
|
|
161
|
+
esios archives exec 34 --sheet I90DIA03 -s 2025-05-05 -e 2025-06-08 \\
|
|
162
|
+
-x "df[df['Sentido']=='Bajar'].groupby('Tecnología')['value'].sum().sort_values()"
|
|
163
|
+
|
|
164
|
+
# Export PBF generation program to CSV
|
|
165
|
+
esios archives exec 34 --sheet I90DIA26 --date 2025-06-01 -f csv -o pbf.csv
|
|
166
|
+
|
|
167
|
+
# Descriptive statistics
|
|
168
|
+
esios archives exec 34 --sheet I90DIA03 --date 2025-06-01 -x "df.describe()"
|
|
169
|
+
"""
|
|
170
|
+
import numpy as np
|
|
171
|
+
import pandas as pd
|
|
172
|
+
|
|
173
|
+
from esios.processing.i90 import I90Book
|
|
174
|
+
|
|
175
|
+
files = _download_files(archive_id, token, date, start, end)
|
|
176
|
+
|
|
177
|
+
# Parse all files and extract the requested sheet
|
|
178
|
+
all_dfs = []
|
|
179
|
+
for f in files:
|
|
180
|
+
try:
|
|
181
|
+
book = I90Book(f)
|
|
182
|
+
s = book[sheet]
|
|
183
|
+
if s.df is not None and not s.df.empty:
|
|
184
|
+
all_dfs.append(s.df.reset_index())
|
|
185
|
+
except KeyError:
|
|
186
|
+
# Sheet not found — show available sheets
|
|
187
|
+
try:
|
|
188
|
+
book = I90Book(f)
|
|
189
|
+
available = list(book.table_of_contents.keys())
|
|
190
|
+
except Exception:
|
|
191
|
+
available = []
|
|
192
|
+
typer.echo(f"Sheet '{sheet}' not found in {f.name}.", err=True)
|
|
193
|
+
if available:
|
|
194
|
+
typer.echo(f"Available sheets: {', '.join(str(s) for s in available)}", err=True)
|
|
195
|
+
raise typer.Exit(1)
|
|
196
|
+
except Exception as exc:
|
|
197
|
+
typer.echo(f"Warning: skipping {f.name}: {exc}", err=True)
|
|
198
|
+
continue
|
|
199
|
+
|
|
200
|
+
if not all_dfs:
|
|
201
|
+
typer.echo("No data extracted from the specified sheet.", err=True)
|
|
202
|
+
raise typer.Exit(1)
|
|
203
|
+
|
|
204
|
+
df = pd.concat(all_dfs, ignore_index=True)
|
|
205
|
+
|
|
206
|
+
# Evaluate expression
|
|
207
|
+
namespace = {"df": df, "pd": pd, "np": np}
|
|
208
|
+
try:
|
|
209
|
+
result = eval(expr, {"__builtins__": {}}, namespace) # noqa: S307
|
|
210
|
+
except Exception as exc:
|
|
211
|
+
typer.echo(f"Error evaluating expression: {exc}", err=True)
|
|
212
|
+
raise typer.Exit(1)
|
|
213
|
+
|
|
214
|
+
# Render output — reuse _render from exec_cmd
|
|
215
|
+
from esios.cli.exec_cmd import _render
|
|
216
|
+
|
|
217
|
+
_render(result, format, output)
|
|
@@ -385,6 +385,11 @@ class IndicatorsManager(BaseManager):
|
|
|
385
385
|
indicator = Indicator.from_api(raw)
|
|
386
386
|
handle = IndicatorHandle(self, indicator)
|
|
387
387
|
|
|
388
|
+
# Enrich geos from values returned in metadata response.
|
|
389
|
+
# Some indicators (e.g. province-level breakdown) have an empty
|
|
390
|
+
# "geos" field in the metadata but geo_id/geo_name in the values.
|
|
391
|
+
handle._enrich_geo_map(raw.get("values", []))
|
|
392
|
+
|
|
388
393
|
# Persist metadata and geos
|
|
389
394
|
if cache.config.enabled:
|
|
390
395
|
handle._persist_meta()
|
|
@@ -30,7 +30,7 @@ def _get_idx_column_start(columns: np.ndarray) -> int:
|
|
|
30
30
|
|
|
31
31
|
def _any_value_greater_than_30(series: np.ndarray) -> bool:
|
|
32
32
|
"""Check if any numeric value exceeds 30 (quarter-hourly indicator)."""
|
|
33
|
-
return any(v > 30 for v in series if isinstance(v, (int, float)) and not np.isnan(v))
|
|
33
|
+
return any(v > 30 for v in series if isinstance(v, (int, float, np.integer, np.floating)) and not np.isnan(v))
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
class I90Book:
|
|
@@ -164,15 +164,36 @@ class I90Sheet:
|
|
|
164
164
|
return arr
|
|
165
165
|
|
|
166
166
|
def _normalize_datetime_columns(self, columns: np.ndarray) -> np.ndarray:
|
|
167
|
-
"""Normalize time column headers to integer period indices.
|
|
167
|
+
"""Normalize time column headers to integer period indices.
|
|
168
|
+
|
|
169
|
+
Handles three column formats found in I90 files:
|
|
170
|
+
- Sequential integers 1–24 (hourly) or 1–96 (quarterly)
|
|
171
|
+
- H-Q format with dash notation: "1-1", "1-2", "1-3", "1-4", "2-1", …
|
|
172
|
+
- NaN-filler format: [1, NaN, NaN, NaN, 2, …] (one label per hour,
|
|
173
|
+
three trailing NaNs for quarters 2–4)
|
|
174
|
+
"""
|
|
168
175
|
if any(pd.isna(columns)):
|
|
169
176
|
self._n_columns_totals = 3
|
|
170
177
|
else:
|
|
171
178
|
self._n_columns_totals = 2
|
|
172
179
|
|
|
173
180
|
series = pd.Series(columns, dtype=str).ffill()
|
|
174
|
-
|
|
175
|
-
|
|
181
|
+
parts = series.str.split("-")
|
|
182
|
+
hours = parts.str[0].astype(float).astype(int)
|
|
183
|
+
|
|
184
|
+
# H-Q format: any column carries an explicit quarter suffix (e.g. "1-2")
|
|
185
|
+
if (parts.str.len() > 1).any():
|
|
186
|
+
quarters = parts.apply(
|
|
187
|
+
lambda x: int(x[1]) if len(x) > 1 and str(x[1]).isdigit() else 1
|
|
188
|
+
)
|
|
189
|
+
return ((hours - 1) * 4 + quarters).values
|
|
190
|
+
|
|
191
|
+
# NaN-filler quarterly format: after ffill the same hour number repeats
|
|
192
|
+
# four times (quarters share the hour label). Assign sequential indices.
|
|
193
|
+
if hours.duplicated().any():
|
|
194
|
+
return np.arange(1, len(hours) + 1)
|
|
195
|
+
|
|
196
|
+
return hours.values
|
|
176
197
|
|
|
177
198
|
def _preprocess_double_index(
|
|
178
199
|
self, idx: int, columns: np.ndarray
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Tests for I90 file processing — frequency detection and column normalisation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from unittest.mock import MagicMock
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from esios.processing.i90 import I90Sheet, _any_value_greater_than_30
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# ---------------------------------------------------------------------------
|
|
14
|
+
# Helpers
|
|
15
|
+
# ---------------------------------------------------------------------------
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _make_sheet() -> I90Sheet:
|
|
19
|
+
"""Return a bare I90Sheet instance backed by mocked objects."""
|
|
20
|
+
wb = MagicMock()
|
|
21
|
+
sheet = MagicMock()
|
|
22
|
+
sheet.to_python.return_value = [[""]]
|
|
23
|
+
wb.get_sheet_by_name.return_value = sheet
|
|
24
|
+
return I90Sheet("test", wb, "I90DIA_20241001.xls", {})
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _full_nan_filler_columns(n_hours: int = 24) -> np.ndarray:
|
|
28
|
+
"""Build NaN-filler quarterly columns: [1, NaN, NaN, NaN, 2, NaN, …]."""
|
|
29
|
+
cols: list = []
|
|
30
|
+
for h in range(1, n_hours + 1):
|
|
31
|
+
cols.append(h)
|
|
32
|
+
cols.extend([np.nan, np.nan, np.nan])
|
|
33
|
+
return np.array(cols, dtype=object)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _full_hq_columns(n_hours: int = 24) -> np.ndarray:
|
|
37
|
+
"""Build explicit H-Q columns: ['1-1', '1-2', '1-3', '1-4', '2-1', …]."""
|
|
38
|
+
return np.array(
|
|
39
|
+
[f"{h}-{q}" for h in range(1, n_hours + 1) for q in range(1, 5)],
|
|
40
|
+
dtype=object,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _mixed_hq_columns(n_hours: int = 24) -> np.ndarray:
|
|
45
|
+
"""First quarter is unlabelled ('1', '2', …); rest carry '-Q' suffix."""
|
|
46
|
+
cols: list = []
|
|
47
|
+
for h in range(1, n_hours + 1):
|
|
48
|
+
cols.append(str(h))
|
|
49
|
+
for q in range(2, 5):
|
|
50
|
+
cols.append(f"{h}-{q}")
|
|
51
|
+
return np.array(cols, dtype=object)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
# _any_value_greater_than_30
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class TestAnyValueGreaterThan30:
|
|
60
|
+
def test_returns_true_for_value_above_30(self):
|
|
61
|
+
assert _any_value_greater_than_30(np.array([1, 31, 24])) is True
|
|
62
|
+
|
|
63
|
+
def test_returns_false_for_all_values_24_or_less(self):
|
|
64
|
+
assert _any_value_greater_than_30(np.arange(1, 25)) is False
|
|
65
|
+
|
|
66
|
+
def test_works_with_numpy_int64(self):
|
|
67
|
+
"""numpy 2.x broke isinstance(np.int64, int) — make sure we handle it."""
|
|
68
|
+
arr = np.array([1, 2, 31, 96], dtype=np.int64)
|
|
69
|
+
assert _any_value_greater_than_30(arr) is True
|
|
70
|
+
|
|
71
|
+
def test_ignores_nan(self):
|
|
72
|
+
arr = np.array([np.nan, 5.0, 20.0])
|
|
73
|
+
assert _any_value_greater_than_30(arr) is False
|
|
74
|
+
|
|
75
|
+
def test_sequential_quarterly_1_to_96(self):
|
|
76
|
+
assert _any_value_greater_than_30(np.arange(1, 97)) is True
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ---------------------------------------------------------------------------
|
|
80
|
+
# _normalize_datetime_columns — hourly (unchanged behaviour)
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class TestNormalizeHourly:
|
|
85
|
+
def test_sequential_1_to_24(self):
|
|
86
|
+
s = _make_sheet()
|
|
87
|
+
cols = np.array([float(i) for i in range(1, 25)], dtype=object)
|
|
88
|
+
result = s._normalize_datetime_columns(cols)
|
|
89
|
+
assert list(result) == list(range(1, 25))
|
|
90
|
+
assert not _any_value_greater_than_30(result)
|
|
91
|
+
|
|
92
|
+
def test_n_columns_totals_is_2_when_no_nan(self):
|
|
93
|
+
s = _make_sheet()
|
|
94
|
+
cols = np.array([float(i) for i in range(1, 25)], dtype=object)
|
|
95
|
+
s._normalize_datetime_columns(cols)
|
|
96
|
+
assert s._n_columns_totals == 2
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
# _normalize_datetime_columns — quarterly (new behaviour)
|
|
101
|
+
# ---------------------------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class TestNormalizeQuarterlySequential:
|
|
105
|
+
"""Columns already in 1–96 sequential form (already worked before the fix)."""
|
|
106
|
+
|
|
107
|
+
def test_sequential_1_to_96(self):
|
|
108
|
+
s = _make_sheet()
|
|
109
|
+
cols = np.array([float(i) for i in range(1, 97)], dtype=object)
|
|
110
|
+
result = s._normalize_datetime_columns(cols)
|
|
111
|
+
assert list(result) == list(range(1, 97))
|
|
112
|
+
assert _any_value_greater_than_30(result)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class TestNormalizeQuarterlyHQFormat:
|
|
116
|
+
"""Columns in explicit 'H-Q' dash notation: '1-1', '1-2', …, '24-4'."""
|
|
117
|
+
|
|
118
|
+
def test_full_day_96_periods(self):
|
|
119
|
+
s = _make_sheet()
|
|
120
|
+
result = s._normalize_datetime_columns(_full_hq_columns())
|
|
121
|
+
assert len(result) == 96
|
|
122
|
+
assert list(result) == list(range(1, 97))
|
|
123
|
+
assert _any_value_greater_than_30(result)
|
|
124
|
+
|
|
125
|
+
def test_time_deltas_are_correct(self):
|
|
126
|
+
"""period 1 → 0 min (00:00), period 96 → 1425 min (23:45)."""
|
|
127
|
+
s = _make_sheet()
|
|
128
|
+
result = s._normalize_datetime_columns(_full_hq_columns())
|
|
129
|
+
time_deltas = (result - 1) * 15
|
|
130
|
+
assert time_deltas[0] == 0
|
|
131
|
+
assert time_deltas[-1] == 1425
|
|
132
|
+
|
|
133
|
+
def test_mixed_hq_first_quarter_unlabelled(self):
|
|
134
|
+
"""'1', '1-2', '1-3', '1-4', '2', '2-2', … is treated the same."""
|
|
135
|
+
s = _make_sheet()
|
|
136
|
+
result = s._normalize_datetime_columns(_mixed_hq_columns())
|
|
137
|
+
assert len(result) == 96
|
|
138
|
+
assert list(result) == list(range(1, 97))
|
|
139
|
+
assert _any_value_greater_than_30(result)
|
|
140
|
+
|
|
141
|
+
def test_n_columns_totals_is_2_for_explicit_hq(self):
|
|
142
|
+
s = _make_sheet()
|
|
143
|
+
s._normalize_datetime_columns(_full_hq_columns())
|
|
144
|
+
assert s._n_columns_totals == 2
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class TestNormalizeQuarterlyNaNFiller:
|
|
148
|
+
"""Columns in NaN-filler form: [1, NaN, NaN, NaN, 2, NaN, …]."""
|
|
149
|
+
|
|
150
|
+
def test_full_day_96_periods(self):
|
|
151
|
+
s = _make_sheet()
|
|
152
|
+
result = s._normalize_datetime_columns(_full_nan_filler_columns())
|
|
153
|
+
assert len(result) == 96
|
|
154
|
+
assert list(result) == list(range(1, 97))
|
|
155
|
+
assert _any_value_greater_than_30(result)
|
|
156
|
+
|
|
157
|
+
def test_time_deltas_are_correct(self):
|
|
158
|
+
s = _make_sheet()
|
|
159
|
+
result = s._normalize_datetime_columns(_full_nan_filler_columns())
|
|
160
|
+
time_deltas = (result - 1) * 15
|
|
161
|
+
assert time_deltas[0] == 0
|
|
162
|
+
assert time_deltas[-1] == 1425
|
|
163
|
+
|
|
164
|
+
def test_n_columns_totals_is_3_when_nan_present(self):
|
|
165
|
+
s = _make_sheet()
|
|
166
|
+
s._normalize_datetime_columns(_full_nan_filler_columns())
|
|
167
|
+
assert s._n_columns_totals == 3
|
|
@@ -165,6 +165,53 @@ class TestIndicatorsCaching:
|
|
|
165
165
|
assert "1" in geos
|
|
166
166
|
assert geos["1"] == "Portugal"
|
|
167
167
|
|
|
168
|
+
def test_get_enriches_geos_from_values_when_metadata_geos_empty(
|
|
169
|
+
self, cached_client, mock_httpx,
|
|
170
|
+
):
|
|
171
|
+
"""indicators.get() should populate geos from values when metadata geos is empty.
|
|
172
|
+
|
|
173
|
+
Reproduces the bug reported for indicators with province-level breakdown
|
|
174
|
+
(e.g. indicator 1161) where the API returns ``"geos": []`` in the
|
|
175
|
+
metadata field but each value carries ``geo_id`` / ``geo_name``.
|
|
176
|
+
"""
|
|
177
|
+
indicator_response = {
|
|
178
|
+
"indicator": {
|
|
179
|
+
"id": 1161,
|
|
180
|
+
"name": "Generación medida Solar fotovoltaica",
|
|
181
|
+
"geos": [],
|
|
182
|
+
"values": [
|
|
183
|
+
{
|
|
184
|
+
"value": 100.0,
|
|
185
|
+
"datetime": "2025-01-01T00:00:00.000+01:00",
|
|
186
|
+
"datetime_utc": "2024-12-31T23:00:00Z",
|
|
187
|
+
"tz_time": "2025-01-01T00:00:00.000+01:00",
|
|
188
|
+
"geo_id": 4,
|
|
189
|
+
"geo_name": "Almería",
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
"value": 200.0,
|
|
193
|
+
"datetime": "2025-01-01T00:00:00.000+01:00",
|
|
194
|
+
"datetime_utc": "2024-12-31T23:00:00Z",
|
|
195
|
+
"tz_time": "2025-01-01T00:00:00.000+01:00",
|
|
196
|
+
"geo_id": 5,
|
|
197
|
+
"geo_name": "Cádiz",
|
|
198
|
+
},
|
|
199
|
+
],
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
response = MagicMock()
|
|
203
|
+
response.status_code = 200
|
|
204
|
+
response.json.return_value = indicator_response
|
|
205
|
+
mock_httpx.get.return_value = response
|
|
206
|
+
|
|
207
|
+
handle = cached_client.indicators.get(1161)
|
|
208
|
+
|
|
209
|
+
# geos must be populated even though the metadata field was empty
|
|
210
|
+
assert len(handle.geos) == 2
|
|
211
|
+
geo_ids = {g["geo_id"] for g in handle.geos}
|
|
212
|
+
assert 4 in geo_ids
|
|
213
|
+
assert 5 in geo_ids
|
|
214
|
+
|
|
168
215
|
def test_enrich_geo_map_persists(
|
|
169
216
|
self, cached_client, mock_httpx,
|
|
170
217
|
):
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
"""CLI subcommands for archives."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from typing import Optional
|
|
6
|
-
|
|
7
|
-
import typer
|
|
8
|
-
from rich.console import Console
|
|
9
|
-
from rich.table import Table
|
|
10
|
-
|
|
11
|
-
archives_app = typer.Typer(no_args_is_help=True)
|
|
12
|
-
console = Console()
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
@archives_app.command("list")
|
|
16
|
-
def list_archives(
|
|
17
|
-
token: Optional[str] = typer.Option(None, "--token", "-t"),
|
|
18
|
-
format: str = typer.Option("table", "--format", "-f"),
|
|
19
|
-
):
|
|
20
|
-
"""List all available ESIOS archives."""
|
|
21
|
-
from esios.cli.app import get_client
|
|
22
|
-
|
|
23
|
-
client = get_client(token)
|
|
24
|
-
df = client.archives.list()
|
|
25
|
-
|
|
26
|
-
if format == "csv":
|
|
27
|
-
typer.echo(df.to_csv())
|
|
28
|
-
elif format == "json":
|
|
29
|
-
typer.echo(df.to_json(orient="records", indent=2))
|
|
30
|
-
else:
|
|
31
|
-
table = Table(title="ESIOS Archives")
|
|
32
|
-
cols = list(df.columns)[:8]
|
|
33
|
-
for col in cols:
|
|
34
|
-
table.add_column(str(col))
|
|
35
|
-
for _, row in df.head(50).iterrows():
|
|
36
|
-
table.add_row(*[str(row[c]) for c in cols])
|
|
37
|
-
if len(df) > 50:
|
|
38
|
-
table.caption = f"Showing 50 of {len(df)} rows"
|
|
39
|
-
console.print(table)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
@archives_app.command("download")
|
|
43
|
-
def download_archive(
|
|
44
|
-
archive_id: int = typer.Argument(..., help="Archive ID"),
|
|
45
|
-
start: Optional[str] = typer.Option(None, "--start", "-s", help="Start date (YYYY-MM-DD)"),
|
|
46
|
-
end: Optional[str] = typer.Option(None, "--end", "-e", help="End date (YYYY-MM-DD)"),
|
|
47
|
-
date: Optional[str] = typer.Option(None, "--date", "-d", help="Single date (YYYY-MM-DD)"),
|
|
48
|
-
output: str = typer.Option(".", "--output", "-o", help="Output directory"),
|
|
49
|
-
token: Optional[str] = typer.Option(None, "--token", "-t"),
|
|
50
|
-
):
|
|
51
|
-
"""Download an archive for a date or date range."""
|
|
52
|
-
from esios.cli.app import get_client
|
|
53
|
-
|
|
54
|
-
if not date and not (start and end):
|
|
55
|
-
typer.echo("Provide --date or both --start and --end.", err=True)
|
|
56
|
-
raise typer.Exit(1)
|
|
57
|
-
|
|
58
|
-
client = get_client(token)
|
|
59
|
-
client.archives.download(
|
|
60
|
-
archive_id,
|
|
61
|
-
start=start,
|
|
62
|
-
end=end,
|
|
63
|
-
date=date,
|
|
64
|
-
output_dir=output,
|
|
65
|
-
)
|
|
66
|
-
typer.echo(f"Download complete → {output}")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/01_Quickstart/01_Setup and First Query.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/03_Multi-Geography Indicators.ipynb
RENAMED
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/04_Compare Indicators.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/02_Indicators/06_Generation and Demand.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/03_Archives/02_I90 Settlement Files.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/05_Advanced/01_Ad-hoc Pandas Expressions.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/01_Search Indicators.yaml
RENAMED
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/02_Historical Data.yaml
RENAMED
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/04_Compare Indicators.yaml
RENAMED
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/02_Indicators/05_Market Prices.yaml
RENAMED
|
File without changes
|
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/03_Archives/01_Download Archives.yaml
RENAMED
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/03_Archives/02_I90 Settlement Files.yaml
RENAMED
|
File without changes
|
{python_esios-2.0.1 → python_esios-2.0.2}/examples/_specs/04_Caching/01_Cache Management.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|