sistat 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sistat-0.1.0/.gitignore +8 -0
- sistat-0.1.0/LICENSE +21 -0
- sistat-0.1.0/PKG-INFO +115 -0
- sistat-0.1.0/README.md +90 -0
- sistat-0.1.0/pyproject.toml +71 -0
- sistat-0.1.0/src/sistat/__init__.py +8 -0
- sistat-0.1.0/src/sistat/client.py +89 -0
- sistat-0.1.0/src/sistat/errors.py +14 -0
- sistat-0.1.0/src/sistat/pxweb/__init__.py +7 -0
- sistat-0.1.0/src/sistat/pxweb/client.py +98 -0
- sistat-0.1.0/src/sistat/pxweb/jsonstat.py +67 -0
- sistat-0.1.0/src/sistat/pxweb/models.py +89 -0
- sistat-0.1.0/src/sistat/pxweb/query.py +148 -0
- sistat-0.1.0/src/sistat/pxweb/session.py +129 -0
- sistat-0.1.0/src/sistat/py.typed +0 -0
- sistat-0.1.0/tests/__init__.py +0 -0
- sistat-0.1.0/tests/conftest.py +79 -0
- sistat-0.1.0/tests/fixtures/data_0156101S.json +1 -0
- sistat-0.1.0/tests/fixtures/meta_0156101S.json +1 -0
- sistat-0.1.0/tests/fixtures/tables.json +32 -0
- sistat-0.1.0/tests/test_client.py +48 -0
- sistat-0.1.0/tests/test_jsonstat.py +37 -0
- sistat-0.1.0/tests/test_matching.py +49 -0
- sistat-0.1.0/tests/test_pxweb.py +111 -0
- sistat-0.1.0/tests/test_session.py +235 -0
sistat-0.1.0/.gitignore
ADDED
sistat-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jošt Prevc
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
sistat-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sistat
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python client for SiStat, the database of the Statistical Office of Slovenia (SURS), via its PxWeb API
|
|
5
|
+
Project-URL: Homepage, https://github.com/jprevc/sistat
|
|
6
|
+
Project-URL: Repository, https://github.com/jprevc/sistat
|
|
7
|
+
Project-URL: Issues, https://github.com/jprevc/sistat/issues
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: open-data,pxweb,sistat,slovenia,statistics,surs
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# sistat
|
|
27
|
+
|
|
28
|
+
A small Python client for [SiStat](https://pxweb.stat.si/SiStat), the statistical database of the
|
|
29
|
+
Statistical Office of the Republic of Slovenia (SURS). Use it to search tables, inspect their
|
|
30
|
+
variables, and download data as plain records, ready for pandas, polars or anything else.
|
|
31
|
+
|
|
32
|
+
SiStat runs [PxWeb](https://www.scb.se/en/services/statistical-programs-for-px-files/px-web/),
|
|
33
|
+
whose v1 API is documented in
|
|
34
|
+
[SURS's API help (PDF)](https://www.stat.si/StatWeb/File/DocSysFile/10722/px_web_api_helpEN.pdf).
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install sistat
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Its only dependency is `httpx`.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import sistat
|
|
48
|
+
|
|
49
|
+
c = sistat.Client(lang="en") # or "sl"
|
|
50
|
+
|
|
51
|
+
c.search("air temperatures") # case- and accent-insensitive, searched locally
|
|
52
|
+
# [TableInfo(id='0156101S.px', text='Average annual and monthly air temperatures [°C] by ...', ...)]
|
|
53
|
+
|
|
54
|
+
meta = c.metadata("0156101S") # the ".px" suffix is optional
|
|
55
|
+
for v in meta.variables:
|
|
56
|
+
print(v.code, "|", v.text, "|", v.value_texts[:3])
|
|
57
|
+
# METEOROLOŠKA POSTAJA | METEOROLOGICAL STATION | ('SLOVENIA', 'Bilje', 'Bovec')
|
|
58
|
+
# OBDOBJE, LETO | PERIOD, YEAR | ('1981-1990', '1991-2000', '2001-2010')
|
|
59
|
+
# MERITVE | MEASURE | ('Average temperature - ANNUAL', 'January', 'February')
|
|
60
|
+
|
|
61
|
+
records = c.get_records(
|
|
62
|
+
"0156101S",
|
|
63
|
+
{
|
|
64
|
+
"METEOROLOGICAL STATION": ["Ljubljana, Bežigrad", "Maribor, Tabor"],
|
|
65
|
+
"PERIOD, YEAR": "201*",
|
|
66
|
+
},
|
|
67
|
+
)
|
|
68
|
+
# [{'METEOROLOGICAL STATION': 'Ljubljana, Bežigrad', 'PERIOD, YEAR': '2010',
|
|
69
|
+
# 'MEASURE': 'Average temperature - ANNUAL', 'value': 10.7}, ...]
|
|
70
|
+
|
|
71
|
+
import pandas as pd # or polars: pl.DataFrame(records)
|
|
72
|
+
df = pd.DataFrame(records)
|
|
73
|
+
# METEOROLOGICAL STATION PERIOD, YEAR MEASURE value
|
|
74
|
+
# 0 Ljubljana, Bežigrad 2010 Average temperature - ANNUAL 10.7
|
|
75
|
+
# ...
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Selections
|
|
79
|
+
|
|
80
|
+
Keys are variables, given as their code (often Slovenian, even in the English API) or their
|
|
81
|
+
label, case-insensitively. Values can be:
|
|
82
|
+
|
|
83
|
+
| Selection | Meaning |
|
|
84
|
+
|-----------------------------------|--------------------------------------------|
|
|
85
|
+
| `"Ljubljana, Bežigrad"` or `"10"` | one value, by label or code |
|
|
86
|
+
| `["10", "Maribor, Tabor"]` | several values |
|
|
87
|
+
| `"2020*"`, `"*"` | glob pattern over codes and labels |
|
|
88
|
+
| `"top:5"` | the first 5 values in the table's order |
|
|
89
|
+
|
|
90
|
+
The library sends every variable you leave out with all its values selected. PxWeb would
|
|
91
|
+
otherwise drop it from the result without telling you.
|
|
92
|
+
|
|
93
|
+
Pass `codes=True` to also get a `<variable>_code` key with the raw value codes. Queries that
|
|
94
|
+
exceed the server's cell limit are split into several requests and merged. Requests are kept
|
|
95
|
+
under the server's rate limit automatically.
|
|
96
|
+
|
|
97
|
+
## Layout
|
|
98
|
+
|
|
99
|
+
- `sistat.pxweb`: a generic PxWeb **v1** client that works on its own:
|
|
100
|
+
`PxWebClient.get_records(table, selections)` resolves selections by label or code, splits
|
|
101
|
+
queries at the server's cell limit, rate-limits, and returns merged records. It never imports
|
|
102
|
+
from the SiStat layer, so it could be split into its own package.
|
|
103
|
+
- `sistat.Client`: SiStat specifics such as the base URL, table-id normalisation, cached table
|
|
104
|
+
search.
|
|
105
|
+
|
|
106
|
+
## Development
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
uv sync
|
|
110
|
+
uv run pytest # offline tests against recorded fixtures
|
|
111
|
+
uv run pytest -m live # hits the real API
|
|
112
|
+
uv run ruff check .
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Data © SURS, see their [terms of use](https://www.stat.si/StatWeb/en/LegalNotice).
|
sistat-0.1.0/README.md
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# sistat
|
|
2
|
+
|
|
3
|
+
A small Python client for [SiStat](https://pxweb.stat.si/SiStat), the statistical database of the
|
|
4
|
+
Statistical Office of the Republic of Slovenia (SURS). Use it to search tables, inspect their
|
|
5
|
+
variables, and download data as plain records, ready for pandas, polars or anything else.
|
|
6
|
+
|
|
7
|
+
SiStat runs [PxWeb](https://www.scb.se/en/services/statistical-programs-for-px-files/px-web/),
|
|
8
|
+
whose v1 API is documented in
|
|
9
|
+
[SURS's API help (PDF)](https://www.stat.si/StatWeb/File/DocSysFile/10722/px_web_api_helpEN.pdf).
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install sistat
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Its only dependency is `httpx`.
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
import sistat
|
|
23
|
+
|
|
24
|
+
c = sistat.Client(lang="en") # or "sl"
|
|
25
|
+
|
|
26
|
+
c.search("air temperatures") # case- and accent-insensitive, searched locally
|
|
27
|
+
# [TableInfo(id='0156101S.px', text='Average annual and monthly air temperatures [°C] by ...', ...)]
|
|
28
|
+
|
|
29
|
+
meta = c.metadata("0156101S") # the ".px" suffix is optional
|
|
30
|
+
for v in meta.variables:
|
|
31
|
+
print(v.code, "|", v.text, "|", v.value_texts[:3])
|
|
32
|
+
# METEOROLOŠKA POSTAJA | METEOROLOGICAL STATION | ('SLOVENIA', 'Bilje', 'Bovec')
|
|
33
|
+
# OBDOBJE, LETO | PERIOD, YEAR | ('1981-1990', '1991-2000', '2001-2010')
|
|
34
|
+
# MERITVE | MEASURE | ('Average temperature - ANNUAL', 'January', 'February')
|
|
35
|
+
|
|
36
|
+
records = c.get_records(
|
|
37
|
+
"0156101S",
|
|
38
|
+
{
|
|
39
|
+
"METEOROLOGICAL STATION": ["Ljubljana, Bežigrad", "Maribor, Tabor"],
|
|
40
|
+
"PERIOD, YEAR": "201*",
|
|
41
|
+
},
|
|
42
|
+
)
|
|
43
|
+
# [{'METEOROLOGICAL STATION': 'Ljubljana, Bežigrad', 'PERIOD, YEAR': '2010',
|
|
44
|
+
# 'MEASURE': 'Average temperature - ANNUAL', 'value': 10.7}, ...]
|
|
45
|
+
|
|
46
|
+
import pandas as pd # or polars: pl.DataFrame(records)
|
|
47
|
+
df = pd.DataFrame(records)
|
|
48
|
+
# METEOROLOGICAL STATION PERIOD, YEAR MEASURE value
|
|
49
|
+
# 0 Ljubljana, Bežigrad 2010 Average temperature - ANNUAL 10.7
|
|
50
|
+
# ...
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Selections
|
|
54
|
+
|
|
55
|
+
Keys are variables, given as their code (often Slovenian, even in the English API) or their
|
|
56
|
+
label, case-insensitively. Values can be:
|
|
57
|
+
|
|
58
|
+
| Selection | Meaning |
|
|
59
|
+
|-----------------------------------|--------------------------------------------|
|
|
60
|
+
| `"Ljubljana, Bežigrad"` or `"10"` | one value, by label or code |
|
|
61
|
+
| `["10", "Maribor, Tabor"]` | several values |
|
|
62
|
+
| `"2020*"`, `"*"` | glob pattern over codes and labels |
|
|
63
|
+
| `"top:5"` | the first 5 values in the table's order |
|
|
64
|
+
|
|
65
|
+
The library sends every variable you leave out with all its values selected. PxWeb would
|
|
66
|
+
otherwise drop it from the result without telling you.
|
|
67
|
+
|
|
68
|
+
Pass `codes=True` to also get a `<variable>_code` key with the raw value codes. Queries that
|
|
69
|
+
exceed the server's cell limit are split into several requests and merged. Requests are kept
|
|
70
|
+
under the server's rate limit automatically.
|
|
71
|
+
|
|
72
|
+
## Layout
|
|
73
|
+
|
|
74
|
+
- `sistat.pxweb`: a generic PxWeb **v1** client that works on its own:
|
|
75
|
+
`PxWebClient.get_records(table, selections)` resolves selections by label or code, splits
|
|
76
|
+
queries at the server's cell limit, rate-limits, and returns merged records. It never imports
|
|
77
|
+
from the SiStat layer, so it could be split into its own package.
|
|
78
|
+
- `sistat.Client`: SiStat specifics such as the base URL, table-id normalisation, cached table
|
|
79
|
+
search.
|
|
80
|
+
|
|
81
|
+
## Development
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
uv sync
|
|
85
|
+
uv run pytest # offline tests against recorded fixtures
|
|
86
|
+
uv run pytest -m live # hits the real API
|
|
87
|
+
uv run ruff check .
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Data © SURS, see their [terms of use](https://www.stat.si/StatWeb/en/LegalNotice).
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "sistat"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "Python client for SiStat, the database of the Statistical Office of Slovenia (SURS), via its PxWeb API"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
dependencies = ["httpx>=0.27"]
|
|
10
|
+
keywords = ["slovenia", "statistics", "surs", "sistat", "pxweb", "open-data"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Operating System :: OS Independent",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
16
|
+
"Programming Language :: Python :: 3.10",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
"Intended Audience :: Science/Research",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/jprevc/sistat"
|
|
27
|
+
Repository = "https://github.com/jprevc/sistat"
|
|
28
|
+
Issues = "https://github.com/jprevc/sistat/issues"
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = [
|
|
32
|
+
"mypy>=2.4.1",
|
|
33
|
+
"pytest>=8",
|
|
34
|
+
"ruff>=0.6",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[build-system]
|
|
38
|
+
requires = ["hatchling>=1.26"]
|
|
39
|
+
build-backend = "hatchling.build"
|
|
40
|
+
|
|
41
|
+
[tool.hatch.version]
|
|
42
|
+
path = "src/sistat/__init__.py"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.sdist]
|
|
45
|
+
include = ["src/sistat", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
46
|
+
|
|
47
|
+
[tool.pytest.ini_options]
|
|
48
|
+
testpaths = ["tests"]
|
|
49
|
+
markers = ["live: hits the real SiStat API (run with: pytest -m live)"]
|
|
50
|
+
addopts = "-m 'not live'"
|
|
51
|
+
|
|
52
|
+
[tool.ruff]
|
|
53
|
+
line-length = 140
|
|
54
|
+
target-version = "py310"
|
|
55
|
+
extend-exclude = ["*.md"]
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint]
|
|
58
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
59
|
+
|
|
60
|
+
[tool.mypy]
|
|
61
|
+
files = ["src", "tests"]
|
|
62
|
+
python_version = "3.10"
|
|
63
|
+
strict = true
|
|
64
|
+
|
|
65
|
+
[[tool.mypy.overrides]]
|
|
66
|
+
# Tests needn't be annotated, but annotated-or-not their bodies are still checked.
|
|
67
|
+
module = "tests.*"
|
|
68
|
+
disallow_untyped_defs = false
|
|
69
|
+
disallow_incomplete_defs = false
|
|
70
|
+
disallow_untyped_calls = false
|
|
71
|
+
check_untyped_defs = true
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Python client for SiStat, the database of the Statistical Office of Slovenia (SURS)."""
|
|
2
|
+
|
|
3
|
+
from .client import Client, table_id
|
|
4
|
+
from .errors import APIError, QueryError, SiStatError
|
|
5
|
+
from .pxweb import TableInfo, TableMetadata, Variable
|
|
6
|
+
|
|
7
|
+
__all__ = ["APIError", "Client", "QueryError", "SiStatError", "TableInfo", "TableMetadata", "Variable", "table_id"]
|
|
8
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""SiStat-specific client built on the generic PxWeb v1 layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from types import TracebackType
|
|
7
|
+
from typing import Any, Literal
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
from .pxweb import PxWebClient, Selection, TableInfo, TableMetadata, fold
|
|
12
|
+
|
|
13
|
+
API_ROOT = "https://pxweb.stat.si/SiStatData/api/v1"
|
|
14
|
+
DATABASE = "Data"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def table_id(table: str) -> str:
|
|
18
|
+
"""Normalise ``"0156101S"`` / ``"0156101s.px"`` to ``"0156101S.px"``."""
|
|
19
|
+
t = table.strip()
|
|
20
|
+
if t.lower().endswith(".px"):
|
|
21
|
+
t = t[:-3]
|
|
22
|
+
return t.upper() + ".px"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Client:
|
|
26
|
+
"""Client for the SiStat database of the Statistical Office of Slovenia (SURS).
|
|
27
|
+
|
|
28
|
+
>>> c = Client(lang="en")
|
|
29
|
+
>>> c.search("air temperature")
|
|
30
|
+
>>> records = c.get_records("0156101S", {"METEOROLOGICAL STATION": "Ljubljana, Bežigrad"})
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
lang: Literal["en", "sl"] = "en",
|
|
36
|
+
*,
|
|
37
|
+
api_root: str = API_ROOT,
|
|
38
|
+
timeout: float = 60.0,
|
|
39
|
+
transport: httpx.BaseTransport | None = None,
|
|
40
|
+
):
|
|
41
|
+
self.px = PxWebClient(api_root, DATABASE, lang, timeout=timeout, transport=transport)
|
|
42
|
+
self._tables: list[TableInfo] | None = None
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def lang(self) -> str:
|
|
46
|
+
return self.px.lang
|
|
47
|
+
|
|
48
|
+
def close(self) -> None:
|
|
49
|
+
self.px.close()
|
|
50
|
+
|
|
51
|
+
def __enter__(self) -> Client:
|
|
52
|
+
return self
|
|
53
|
+
|
|
54
|
+
def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None) -> None:
|
|
55
|
+
self.close()
|
|
56
|
+
|
|
57
|
+
# -- discovery ----------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
def tables(self, refresh: bool = False) -> list[TableInfo]:
|
|
60
|
+
"""All tables in SiStat (one request, cached on the client)."""
|
|
61
|
+
if self._tables is None or refresh:
|
|
62
|
+
self._tables = [t for t in self.px.list() if t.type == "t"]
|
|
63
|
+
return self._tables
|
|
64
|
+
|
|
65
|
+
def search(self, text: str) -> list[TableInfo]:
|
|
66
|
+
"""Tables whose id or title contains every word of ``text`` (case- and accent-insensitive)."""
|
|
67
|
+
words = fold(text).split()
|
|
68
|
+
return [t for t in self.tables() if all(w in fold(f"{t.id} {t.text}") for w in words)]
|
|
69
|
+
|
|
70
|
+
def metadata(self, table: str) -> TableMetadata:
|
|
71
|
+
return self.px.metadata(table_id(table))
|
|
72
|
+
|
|
73
|
+
# -- data ---------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
def get_records(self, table: str, selections: Mapping[str, Selection] | None = None, *, codes: bool = False) -> list[dict[str, Any]]:
|
|
76
|
+
"""Fetch data as a list of dicts, one per cell (long format).
|
|
77
|
+
|
|
78
|
+
``selections`` maps a variable (code or English/Slovenian label, case-insensitive) to:
|
|
79
|
+
|
|
80
|
+
- a value code or label, or a list of them: ``["Ljubljana, Bežigrad", "11"]``
|
|
81
|
+
- a glob pattern over codes or labels: ``"2020*"``; ``"*"`` means all
|
|
82
|
+
- ``"top:N"``: the first N values in the table's order
|
|
83
|
+
|
|
84
|
+
Variables you don't mention are fully selected. Large queries are split
|
|
85
|
+
automatically to respect the server's cell limit.
|
|
86
|
+
|
|
87
|
+
For a DataFrame, pass the result to ``pandas.DataFrame(...)`` or ``polars.DataFrame(...)``.
|
|
88
|
+
"""
|
|
89
|
+
return self.px.get_records(table_id(table), selections, codes=codes)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class SiStatError(Exception):
|
|
2
|
+
"""Base class for all errors raised by this package."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class APIError(SiStatError):
|
|
6
|
+
"""The server returned an error response."""
|
|
7
|
+
|
|
8
|
+
def __init__(self, message: str, status_code: int | None = None):
|
|
9
|
+
super().__init__(message)
|
|
10
|
+
self.status_code = status_code
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class QueryError(SiStatError, ValueError):
|
|
14
|
+
"""A selection does not match the table's variables or values."""
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Generic PxWeb API v1 client. Must not import from the SiStat layer."""
|
|
2
|
+
|
|
3
|
+
from .client import PxWebClient
|
|
4
|
+
from .models import ServerLimits, TableInfo, TableMetadata, Variable
|
|
5
|
+
from .query import Selection, fold
|
|
6
|
+
|
|
7
|
+
__all__ = ["PxWebClient", "Selection", "ServerLimits", "TableInfo", "TableMetadata", "Variable", "fold"]
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Generic PxWeb API v1 client.
|
|
2
|
+
|
|
3
|
+
Knows nothing about SiStat specifically: give it any PxWeb v1 base URL
|
|
4
|
+
(``https://<host>/<app>/api/v1/<lang>/<database>``).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import builtins
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from types import TracebackType
|
|
12
|
+
from typing import Any
|
|
13
|
+
from urllib.parse import quote
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
from . import query as q
|
|
18
|
+
from .jsonstat import to_records
|
|
19
|
+
from .models import ServerLimits, TableInfo, TableMetadata
|
|
20
|
+
from .session import Session
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class PxWebClient:
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
api_root: str,
|
|
27
|
+
database: str,
|
|
28
|
+
lang: str,
|
|
29
|
+
*,
|
|
30
|
+
timeout: float = 30.0,
|
|
31
|
+
retries: int = 3,
|
|
32
|
+
transport: httpx.BaseTransport | None = None,
|
|
33
|
+
):
|
|
34
|
+
"""``api_root`` is the part before the language, e.g. ``https://host/App/api/v1``."""
|
|
35
|
+
self.api_root = api_root.rstrip("/")
|
|
36
|
+
self.database = database
|
|
37
|
+
self.lang = lang
|
|
38
|
+
self.retries = retries
|
|
39
|
+
self._http = httpx.Client(timeout=timeout, transport=transport, headers={"User-Agent": "sistat-python"})
|
|
40
|
+
self._session = Session(self._http, f"{self.api_root}/{lang}/?config", retries=retries)
|
|
41
|
+
self._meta: dict[str, TableMetadata] = {}
|
|
42
|
+
|
|
43
|
+
# -- plumbing -----------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def base_url(self) -> str:
|
|
47
|
+
return f"{self.api_root}/{self.lang}/{self.database}"
|
|
48
|
+
|
|
49
|
+
def close(self) -> None:
|
|
50
|
+
self._http.close()
|
|
51
|
+
|
|
52
|
+
def __enter__(self) -> PxWebClient:
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None) -> None:
|
|
56
|
+
self.close()
|
|
57
|
+
|
|
58
|
+
def _url(self, path: str = "") -> str:
|
|
59
|
+
path = "/".join(quote(p) for p in path.strip("/").split("/") if p)
|
|
60
|
+
return f"{self.base_url}/{path}" if path else self.base_url
|
|
61
|
+
|
|
62
|
+
# -- API ----------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def limits(self) -> ServerLimits:
|
|
66
|
+
"""The server's cell limit and rate limit, read from ``?config`` on first use (defaults if unavailable)."""
|
|
67
|
+
return self._session.limits
|
|
68
|
+
|
|
69
|
+
def _post(self, table: str, query: builtins.list[dict[str, Any]]) -> dict[str, Any]:
|
|
70
|
+
"""POST a raw PxWeb query (list of ``{"code", "selection"}``) and return the json-stat2 dataset."""
|
|
71
|
+
body = {"query": query, "response": {"format": "json-stat2"}}
|
|
72
|
+
dataset: dict[str, Any] = self._session.send("POST", self._url(table), json=body).json()
|
|
73
|
+
return dataset
|
|
74
|
+
|
|
75
|
+
def list(self, path: str = "") -> list[TableInfo]:
|
|
76
|
+
"""List tables (and folders) at ``path`` within the database."""
|
|
77
|
+
return [TableInfo.from_json(d) for d in self._session.send("GET", self._url(path)).json()]
|
|
78
|
+
|
|
79
|
+
def metadata(self, table: str) -> TableMetadata:
|
|
80
|
+
"""The table's variables and values (fetched once, cached on the client)."""
|
|
81
|
+
if table not in self._meta:
|
|
82
|
+
self._meta[table] = TableMetadata.from_json(self._session.send("GET", self._url(table)).json())
|
|
83
|
+
return self._meta[table]
|
|
84
|
+
|
|
85
|
+
def get_records(
|
|
86
|
+
self, table: str, selections: Mapping[str, q.Selection] | None = None, *, codes: bool = False
|
|
87
|
+
) -> builtins.list[dict[str, Any]]:
|
|
88
|
+
"""Fetch data as a list of records, one per cell.
|
|
89
|
+
|
|
90
|
+
``selections`` maps a variable (code or label, case-insensitive) to a value code or label,
|
|
91
|
+
a list of them, a glob pattern (``"20*"``, ``"*"``), or ``"top:N"``. Variables not mentioned
|
|
92
|
+
are fully selected. Queries over the server's cell limit are split and the results merged.
|
|
93
|
+
"""
|
|
94
|
+
resolved = q.resolve(self.metadata(table), selections)
|
|
95
|
+
records: builtins.list[dict[str, Any]] = []
|
|
96
|
+
for chunk in q.split(resolved, self.limits.cell_limit):
|
|
97
|
+
records.extend(to_records(self._post(table, [r.to_json() for r in chunk]), codes=codes))
|
|
98
|
+
return records
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Minimal json-stat2 dataset -> records conversion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from itertools import product
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _ordered_categories(dim: dict[str, Any]) -> list[str]:
|
|
10
|
+
index = dim["category"].get("index")
|
|
11
|
+
if index is None: # single-category dimension may omit index
|
|
12
|
+
return list(dim["category"]["label"])
|
|
13
|
+
if isinstance(index, list):
|
|
14
|
+
return list(index)
|
|
15
|
+
return sorted(index, key=index.__getitem__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _cell_lookup(obj: Any, n: int) -> list[Any]:
|
|
19
|
+
"""json-stat allows value/status as a dense list, a sparse {index: v} dict, or (status) a single scalar."""
|
|
20
|
+
if obj is None:
|
|
21
|
+
return [None] * n
|
|
22
|
+
if isinstance(obj, list):
|
|
23
|
+
return obj
|
|
24
|
+
if isinstance(obj, dict):
|
|
25
|
+
out: list[Any] = [None] * n
|
|
26
|
+
for k, v in obj.items():
|
|
27
|
+
out[int(k)] = v
|
|
28
|
+
return out
|
|
29
|
+
return [obj] * n
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def to_records(dataset: dict[str, Any], *, codes: bool = False) -> list[dict[str, Any]]:
|
|
33
|
+
"""Flatten a json-stat2 dataset into one dict per cell (long/tidy format).
|
|
34
|
+
|
|
35
|
+
Columns are the dimension labels holding value labels, plus ``value``.
|
|
36
|
+
With ``codes=True`` a ``<label>_code`` column with the raw value code is added per dimension.
|
|
37
|
+
A ``status`` column is added only when the dataset carries status symbols.
|
|
38
|
+
"""
|
|
39
|
+
ids: list[str] = dataset["id"]
|
|
40
|
+
sizes: list[int] = dataset["size"]
|
|
41
|
+
n = 1
|
|
42
|
+
for s in sizes:
|
|
43
|
+
n *= s
|
|
44
|
+
|
|
45
|
+
dims = []
|
|
46
|
+
for dim_id in ids:
|
|
47
|
+
dim = dataset["dimension"][dim_id]
|
|
48
|
+
cats = _ordered_categories(dim)
|
|
49
|
+
labels = dim["category"].get("label", {})
|
|
50
|
+
dims.append((dim.get("label", dim_id), cats, labels))
|
|
51
|
+
|
|
52
|
+
values = _cell_lookup(dataset.get("value"), n)
|
|
53
|
+
statuses = _cell_lookup(dataset.get("status"), n) if "status" in dataset else None
|
|
54
|
+
|
|
55
|
+
records = []
|
|
56
|
+
for i, combo in enumerate(product(*(range(len(c)) for _, c, _ in dims))):
|
|
57
|
+
rec: dict[str, Any] = {}
|
|
58
|
+
for (label, cats, cat_labels), j in zip(dims, combo, strict=True):
|
|
59
|
+
code = cats[j]
|
|
60
|
+
rec[label] = cat_labels.get(code, code)
|
|
61
|
+
if codes:
|
|
62
|
+
rec[f"{label}_code"] = code
|
|
63
|
+
rec["value"] = values[i]
|
|
64
|
+
if statuses is not None:
|
|
65
|
+
rec["status"] = statuses[i]
|
|
66
|
+
records.append(rec)
|
|
67
|
+
return records
|