ssb-nr-utils 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ssb_nr_utils-0.0.1/PKG-INFO +103 -0
- ssb_nr_utils-0.0.1/README.md +76 -0
- ssb_nr_utils-0.0.1/pyproject.toml +174 -0
- ssb_nr_utils-0.0.1/src/nr_utils/__init__.py +1 -0
- ssb_nr_utils-0.0.1/src/nr_utils/__main__.py +13 -0
- ssb_nr_utils-0.0.1/src/nr_utils/fame/__init__.py +0 -0
- ssb_nr_utils-0.0.1/src/nr_utils/fame/fame.py +225 -0
- ssb_nr_utils-0.0.1/src/nr_utils/fame/fame_prog/lag_csv_template.inp +35 -0
- ssb_nr_utils-0.0.1/src/nr_utils/fame/fame_prog/oppdater_db_template.inp +5 -0
- ssb_nr_utils-0.0.1/src/nr_utils/fame/fame_prog/opprett_db_template.inp +5 -0
- ssb_nr_utils-0.0.1/src/nr_utils/functions.py +35 -0
- ssb_nr_utils-0.0.1/src/nr_utils/py.typed +0 -0
- ssb_nr_utils-0.0.1/src/nr_utils/sesongjustering/__init__.py +0 -0
- ssb_nr_utils-0.0.1/src/nr_utils/sesongjustering/bin/x13as +0 -0
- ssb_nr_utils-0.0.1/src/nr_utils/sesongjustering/bin/x13as_html +0 -0
- ssb_nr_utils-0.0.1/src/nr_utils/sesongjustering/x13.py +222 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ssb-nr-utils
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: SSB Nr Utils
|
|
5
|
+
Author: Edvard Garmannslund
|
|
6
|
+
Author-email: Edvard Garmannslund <ged@ssb.no>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Dist: click>=8.0.1
|
|
17
|
+
Requires-Dist: ipykernel>=7.3.0
|
|
18
|
+
Requires-Dist: matplotlib>=3.11.0
|
|
19
|
+
Requires-Dist: pandas>=3.0.3
|
|
20
|
+
Maintainer: Statistics Norway, National accounts Department (210)
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Project-URL: homepage, https://github.com/statisticsnorway/ssb-nr-utils
|
|
23
|
+
Project-URL: repository, https://github.com/statisticsnorway/ssb-nr-utils
|
|
24
|
+
Project-URL: documentation, https://statisticsnorway.github.io/ssb-nr-utils
|
|
25
|
+
Project-URL: Changelog, https://github.com/statisticsnorway/ssb-nr-utils/releases
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# SSB Nr Utils
|
|
29
|
+
|
|
30
|
+
[][pypi status]
|
|
31
|
+
[][pypi status]
|
|
32
|
+
[][pypi status]
|
|
33
|
+
[][license]
|
|
34
|
+
|
|
35
|
+
[][documentation]
|
|
36
|
+
[][tests]
|
|
37
|
+
[][sonarcov]
|
|
38
|
+
[][sonarquality]
|
|
39
|
+
|
|
40
|
+
[][pre-commit]
|
|
41
|
+
[][black]
|
|
42
|
+
[](https://github.com/astral-sh/ruff)
|
|
43
|
+
[][poetry]
|
|
44
|
+
|
|
45
|
+
[pypi status]: https://pypi.org/project/ssb-nr-utils/
|
|
46
|
+
[documentation]: https://statisticsnorway.github.io/ssb-nr-utils
|
|
47
|
+
[tests]: https://github.com/statisticsnorway/ssb-nr-utils/actions?workflow=Tests
|
|
48
|
+
[sonarcov]: https://sonarcloud.io/summary/overall?id=statisticsnorway_ssb-nr-utils
|
|
49
|
+
[sonarquality]: https://sonarcloud.io/summary/overall?id=statisticsnorway_ssb-nr-utils
|
|
50
|
+
[pre-commit]: https://github.com/pre-commit/pre-commit
|
|
51
|
+
[black]: https://github.com/psf/black
|
|
52
|
+
[poetry]: https://python-poetry.org/
|
|
53
|
+
|
|
54
|
+
## Features
|
|
55
|
+
|
|
56
|
+
- TODO
|
|
57
|
+
|
|
58
|
+
## Requirements
|
|
59
|
+
|
|
60
|
+
- TODO
|
|
61
|
+
|
|
62
|
+
## Installation
|
|
63
|
+
|
|
64
|
+
You can install _SSB Nr Utils_ via [pip] from [PyPI]:
|
|
65
|
+
|
|
66
|
+
```console
|
|
67
|
+
pip install ssb-nr-utils
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Usage
|
|
71
|
+
|
|
72
|
+
Please see the [Reference Guide] for details.
|
|
73
|
+
|
|
74
|
+
## Contributing
|
|
75
|
+
|
|
76
|
+
Contributions are very welcome.
|
|
77
|
+
To learn more, see the [Contributor Guide].
|
|
78
|
+
|
|
79
|
+
## License
|
|
80
|
+
|
|
81
|
+
Distributed under the terms of the [MIT license][license],
|
|
82
|
+
_SSB Nr Utils_ is free and open source software.
|
|
83
|
+
|
|
84
|
+
## Issues
|
|
85
|
+
|
|
86
|
+
If you encounter any problems,
|
|
87
|
+
please [file an issue] along with a detailed description.
|
|
88
|
+
|
|
89
|
+
## Credits
|
|
90
|
+
|
|
91
|
+
This project was generated from [Statistics Norway]'s [SSB PyPI Template].
|
|
92
|
+
|
|
93
|
+
[statistics norway]: https://www.ssb.no/en
|
|
94
|
+
[pypi]: https://pypi.org/
|
|
95
|
+
[ssb pypi template]: https://github.com/statisticsnorway/ssb-pypitemplate
|
|
96
|
+
[file an issue]: https://github.com/statisticsnorway/ssb-nr-utils/issues
|
|
97
|
+
[pip]: https://pip.pypa.io/
|
|
98
|
+
|
|
99
|
+
<!-- github-only -->
|
|
100
|
+
|
|
101
|
+
[license]: https://github.com/statisticsnorway/ssb-nr-utils/blob/main/LICENSE
|
|
102
|
+
[contributor guide]: https://github.com/statisticsnorway/ssb-nr-utils/blob/main/CONTRIBUTING.md
|
|
103
|
+
[reference guide]: https://statisticsnorway.github.io/ssb-nr-utils/reference.html
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# SSB Nr Utils
|
|
2
|
+
|
|
3
|
+
[][pypi status]
|
|
4
|
+
[][pypi status]
|
|
5
|
+
[][pypi status]
|
|
6
|
+
[][license]
|
|
7
|
+
|
|
8
|
+
[][documentation]
|
|
9
|
+
[][tests]
|
|
10
|
+
[][sonarcov]
|
|
11
|
+
[][sonarquality]
|
|
12
|
+
|
|
13
|
+
[][pre-commit]
|
|
14
|
+
[][black]
|
|
15
|
+
[](https://github.com/astral-sh/ruff)
|
|
16
|
+
[][poetry]
|
|
17
|
+
|
|
18
|
+
[pypi status]: https://pypi.org/project/ssb-nr-utils/
|
|
19
|
+
[documentation]: https://statisticsnorway.github.io/ssb-nr-utils
|
|
20
|
+
[tests]: https://github.com/statisticsnorway/ssb-nr-utils/actions?workflow=Tests
|
|
21
|
+
[sonarcov]: https://sonarcloud.io/summary/overall?id=statisticsnorway_ssb-nr-utils
|
|
22
|
+
[sonarquality]: https://sonarcloud.io/summary/overall?id=statisticsnorway_ssb-nr-utils
|
|
23
|
+
[pre-commit]: https://github.com/pre-commit/pre-commit
|
|
24
|
+
[black]: https://github.com/psf/black
|
|
25
|
+
[poetry]: https://python-poetry.org/
|
|
26
|
+
|
|
27
|
+
## Features
|
|
28
|
+
|
|
29
|
+
- TODO
|
|
30
|
+
|
|
31
|
+
## Requirements
|
|
32
|
+
|
|
33
|
+
- TODO
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
You can install _SSB Nr Utils_ via [pip] from [PyPI]:
|
|
38
|
+
|
|
39
|
+
```console
|
|
40
|
+
pip install ssb-nr-utils
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
Please see the [Reference Guide] for details.
|
|
46
|
+
|
|
47
|
+
## Contributing
|
|
48
|
+
|
|
49
|
+
Contributions are very welcome.
|
|
50
|
+
To learn more, see the [Contributor Guide].
|
|
51
|
+
|
|
52
|
+
## License
|
|
53
|
+
|
|
54
|
+
Distributed under the terms of the [MIT license][license],
|
|
55
|
+
_SSB Nr Utils_ is free and open source software.
|
|
56
|
+
|
|
57
|
+
## Issues
|
|
58
|
+
|
|
59
|
+
If you encounter any problems,
|
|
60
|
+
please [file an issue] along with a detailed description.
|
|
61
|
+
|
|
62
|
+
## Credits
|
|
63
|
+
|
|
64
|
+
This project was generated from [Statistics Norway]'s [SSB PyPI Template].
|
|
65
|
+
|
|
66
|
+
[statistics norway]: https://www.ssb.no/en
|
|
67
|
+
[pypi]: https://pypi.org/
|
|
68
|
+
[ssb pypi template]: https://github.com/statisticsnorway/ssb-pypitemplate
|
|
69
|
+
[file an issue]: https://github.com/statisticsnorway/ssb-nr-utils/issues
|
|
70
|
+
[pip]: https://pip.pypa.io/
|
|
71
|
+
|
|
72
|
+
<!-- github-only -->
|
|
73
|
+
|
|
74
|
+
[license]: https://github.com/statisticsnorway/ssb-nr-utils/blob/main/LICENSE
|
|
75
|
+
[contributor guide]: https://github.com/statisticsnorway/ssb-nr-utils/blob/main/CONTRIBUTING.md
|
|
76
|
+
[reference guide]: https://statisticsnorway.github.io/ssb-nr-utils/reference.html
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "ssb-nr-utils"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "SSB Nr Utils"
|
|
5
|
+
authors = [{ name = "Edvard Garmannslund", email = "ged@ssb.no" }]
|
|
6
|
+
maintainers = [{ name = "Statistics Norway, National accounts Department (210)" }]
|
|
7
|
+
license = "MIT"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Programming Language :: Python :: 3.10",
|
|
13
|
+
"Programming Language :: Python :: 3.11",
|
|
14
|
+
"Programming Language :: Python :: 3.12",
|
|
15
|
+
"Programming Language :: Python :: 3.13",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Typing :: Typed",
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"click>=8.0.1",
|
|
22
|
+
"ipykernel>=7.3.0",
|
|
23
|
+
"matplotlib>=3.11.0",
|
|
24
|
+
"pandas>=3.0.3",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
homepage = "https://github.com/statisticsnorway/ssb-nr-utils"
|
|
29
|
+
repository = "https://github.com/statisticsnorway/ssb-nr-utils"
|
|
30
|
+
documentation = "https://statisticsnorway.github.io/ssb-nr-utils"
|
|
31
|
+
Changelog = "https://github.com/statisticsnorway/ssb-nr-utils/releases"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
ssb-nr-utils = "nr_utils.__main__:main"
|
|
35
|
+
|
|
36
|
+
[dependency-groups]
|
|
37
|
+
lint = [
|
|
38
|
+
"black[jupyter]>=25.11.0",
|
|
39
|
+
"pre-commit>=4.4.0",
|
|
40
|
+
"pre-commit-hooks>=6.0.0",
|
|
41
|
+
"pydoclint>=0.8.3",
|
|
42
|
+
"ruff>=0.14.5",
|
|
43
|
+
]
|
|
44
|
+
doc = [
|
|
45
|
+
"furo>=2025.9.25",
|
|
46
|
+
"myst-parser>=4.0.1",
|
|
47
|
+
"sphinx>=8.2.3",
|
|
48
|
+
"sphinx-autobuild>=2025.8.25",
|
|
49
|
+
"sphinx-autodoc-typehints>=3.5.2",
|
|
50
|
+
"sphinx-click>=6.1.0",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
# Type hint stubs should be placed in the dev dependency group
|
|
54
|
+
dev = [
|
|
55
|
+
{include-group = "lint"},
|
|
56
|
+
{include-group = "doc"},
|
|
57
|
+
"coverage[toml]>=7.11.3",
|
|
58
|
+
"deptry>=0.24.0; python_version < '4.0'",
|
|
59
|
+
"mypy>=1.18.2",
|
|
60
|
+
"pygments>=2.19.0",
|
|
61
|
+
"pytest>=9.0.0",
|
|
62
|
+
"typeguard>=2.13.3",
|
|
63
|
+
"xdoctest[colors]>=1.3.0",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
[tool.pytest.ini_options]
|
|
68
|
+
pythonpath = ["src"]
|
|
69
|
+
|
|
70
|
+
[tool.coverage.paths]
|
|
71
|
+
source = ["src", "*/site-packages"]
|
|
72
|
+
tests = ["tests", "*/tests"]
|
|
73
|
+
|
|
74
|
+
[tool.coverage.run]
|
|
75
|
+
branch = true
|
|
76
|
+
source = ["nr_utils", "tests"]
|
|
77
|
+
relative_files = true
|
|
78
|
+
|
|
79
|
+
[tool.coverage.report]
|
|
80
|
+
show_missing = true
|
|
81
|
+
fail_under = 20
|
|
82
|
+
|
|
83
|
+
[tool.deptry.per_rule_ignores]
|
|
84
|
+
DEP001 = ["nox"] # packages available by default
|
|
85
|
+
|
|
86
|
+
[tool.mypy]
|
|
87
|
+
strict = true
|
|
88
|
+
warn_unreachable = true
|
|
89
|
+
pretty = true
|
|
90
|
+
show_column_numbers = true
|
|
91
|
+
show_error_context = true
|
|
92
|
+
|
|
93
|
+
[tool.pydoclint]
|
|
94
|
+
style = "google"
|
|
95
|
+
exclude = '\.git|\.nox'
|
|
96
|
+
arg-type-hints-in-docstring = false
|
|
97
|
+
check-return-types = false
|
|
98
|
+
check-yield-types = false
|
|
99
|
+
allow-init-docstring = true
|
|
100
|
+
quiet = true
|
|
101
|
+
|
|
102
|
+
[tool.ruff]
|
|
103
|
+
force-exclude = true # Apply excludes to pre-commit
|
|
104
|
+
show-fixes = true
|
|
105
|
+
src = ["src", "tests"]
|
|
106
|
+
target-version = "py313" # Minimum Python version supported
|
|
107
|
+
include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"]
|
|
108
|
+
extend-exclude = [
|
|
109
|
+
"__pycache__",
|
|
110
|
+
"old",
|
|
111
|
+
".ipynb_checkpoints",
|
|
112
|
+
"noxfile.py",
|
|
113
|
+
"docs/conf.py",
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
# Ruff rules may be customized as desired: https://docs.astral.sh/ruff/rules/
|
|
117
|
+
[tool.ruff.lint]
|
|
118
|
+
select = [
|
|
119
|
+
"E", # pycodestyle
|
|
120
|
+
"F", # pyflakes
|
|
121
|
+
]
|
|
122
|
+
ignore = [
|
|
123
|
+
"ANN202", # Don't require return type annotation for private functions.
|
|
124
|
+
"ANN401", # Allow type annotation with type Any.
|
|
125
|
+
"D100", # Supress undocumented-public-module. Only doc of public api required.
|
|
126
|
+
"FBT001", # Allow boolean positional arguments in a function.
|
|
127
|
+
"FBT002", # Allow boolean default positional arguments in a function.
|
|
128
|
+
"E402", # Supress module-import-not-at-top-of-file, needed in jupyter notebooks.
|
|
129
|
+
"E501", # Supress line-too-long warnings: trust black's judgement on this one.
|
|
130
|
+
"PLR2004", # Allow to compare with unnamed numerical constants.
|
|
131
|
+
]
|
|
132
|
+
external = ["DOC"] # Avoiding removing any codes starting with `DOC` from any `# noqa` directives.
|
|
133
|
+
|
|
134
|
+
[tool.ruff.lint.isort]
|
|
135
|
+
force-single-line = true
|
|
136
|
+
|
|
137
|
+
[tool.ruff.lint.mccabe]
|
|
138
|
+
max-complexity = 15
|
|
139
|
+
|
|
140
|
+
[tool.ruff.lint.pydocstyle]
|
|
141
|
+
convention = "google" # You can also use "numpy".
|
|
142
|
+
|
|
143
|
+
[tool.ruff.lint.pylint]
|
|
144
|
+
max-args = 8
|
|
145
|
+
|
|
146
|
+
[tool.ruff.lint.pep8-naming]
|
|
147
|
+
classmethod-decorators = ["classmethod", "validator", "root_validator", "pydantic.validator"]
|
|
148
|
+
|
|
149
|
+
[tool.ruff.lint.per-file-ignores]
|
|
150
|
+
"*/__init__.py" = ["F401"]
|
|
151
|
+
"**/tests/*" = [
|
|
152
|
+
"ANN001", # type annotations don't add value for test functions
|
|
153
|
+
"ANN002", # type annotations don't add value for test functions
|
|
154
|
+
"ANN003", # type annotations don't add value for test functions
|
|
155
|
+
"ANN201", # type annotations don't add value for test functions
|
|
156
|
+
"ANN204", # type annotations don't add value for test functions
|
|
157
|
+
"ANN205", # type annotations don't add value for test functions
|
|
158
|
+
"ANN206", # type annotations don't add value for test functions
|
|
159
|
+
"D100", # docstrings are overkill for test functions
|
|
160
|
+
"D101",
|
|
161
|
+
"D102",
|
|
162
|
+
"D103",
|
|
163
|
+
"S101", # asserts are encouraged in pytest
|
|
164
|
+
]
|
|
165
|
+
|
|
166
|
+
[tool.uv]
|
|
167
|
+
exclude-newer = "7 days"
|
|
168
|
+
|
|
169
|
+
[tool.uv.build-backend]
|
|
170
|
+
module-name = "nr_utils"
|
|
171
|
+
|
|
172
|
+
[build-system]
|
|
173
|
+
requires = ["uv_build>=0.11.0,<0.12.0"]
|
|
174
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""SSB Nr Utils."""
|
|
File without changes
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Fame utility functions"""
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
import tempfile
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _run_fame_script(script: str, famedb: str) -> None:
|
|
10
|
+
with tempfile.TemporaryDirectory(prefix="fame_prog_nr_utils") as tmp:
|
|
11
|
+
spec_path = Path(tmp) / "run.inp"
|
|
12
|
+
spec_path.write_text(script)
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
with open(spec_path, "r") as spec_file:
|
|
16
|
+
result = subprocess.run(
|
|
17
|
+
["ssh", famedb, "fame"],
|
|
18
|
+
stdin=spec_file,
|
|
19
|
+
stdout=subprocess.DEVNULL,
|
|
20
|
+
stderr=subprocess.PIPE,
|
|
21
|
+
text=True,
|
|
22
|
+
)
|
|
23
|
+
except OSError as e:
|
|
24
|
+
# ssh binary missing, permissions issue, etc.
|
|
25
|
+
raise RuntimeError(f"Failed to launch ssh: {e}") from e
|
|
26
|
+
|
|
27
|
+
if result.returncode != 0:
|
|
28
|
+
raise RuntimeError(
|
|
29
|
+
f"ssh/fame failed (exit code {result.returncode}): {result.stderr.strip()}"
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _inject_params(script: str, params: dict[str, str]) -> str:
|
|
34
|
+
"""Function that puts in params in str obj of fame script."""
|
|
35
|
+
for key, value in params.items():
|
|
36
|
+
script = script.replace(f"<<{key}>>", value)
|
|
37
|
+
return script
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _fame_period(date: pd.Timestamp, freq: str) -> str:
|
|
41
|
+
"""Format a timestamp as a FAME date literal for the given frequency.
|
|
42
|
+
|
|
43
|
+
NOTE: "<year>:<month>" confirmed working for monthly via a tested .inp
|
|
44
|
+
file. Other frequencies below are still unverified -- test before use.
|
|
45
|
+
"""
|
|
46
|
+
if freq == "monthly":
|
|
47
|
+
return f"{date.year}:{date.month}"
|
|
48
|
+
if freq == "quarterly":
|
|
49
|
+
return f"{date.year}:{date.quarter}"
|
|
50
|
+
if freq == "annual":
|
|
51
|
+
return f"{date.year}"
|
|
52
|
+
if freq == "daily":
|
|
53
|
+
return f"{date.year}:{date.month}:{date.day}"
|
|
54
|
+
raise ValueError(f"Unhandled frequency: {freq}")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _assert_no_gaps(data: pd.DataFrame) -> None:
|
|
58
|
+
"""Guard against silent value-list misalignment from a gapped index."""
|
|
59
|
+
expected = pd.date_range(
|
|
60
|
+
data.index[0], data.index[-1], freq=data.index.freq or "MS"
|
|
61
|
+
)
|
|
62
|
+
assert data.index.equals(
|
|
63
|
+
expected
|
|
64
|
+
), "gaps detected in data.index -- would misalign value-list"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _create_series_lines(
|
|
68
|
+
data: pd.DataFrame, db_alias: str, precision: bool = True
|
|
69
|
+
) -> str:
|
|
70
|
+
"""Build one SERIES statement per column, creating and populating each object.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
data: wide DataFrame, datetime index (one row per period, no gaps,
|
|
74
|
+
sorted ascending), one column per series (column name = FAME
|
|
75
|
+
object name).
|
|
76
|
+
db_alias: channel alias from the OPEN ... AS clause.
|
|
77
|
+
precision: use :precision (15 sig. figures) rather than :numeric
|
|
78
|
+
(7 sig. figures).
|
|
79
|
+
"""
|
|
80
|
+
type_clause = ":precision" if precision else ":numeric"
|
|
81
|
+
lines = []
|
|
82
|
+
for name in data.columns:
|
|
83
|
+
values = ", ".join(f"{v:g}" for v in data[name])
|
|
84
|
+
lines.append(f"series {name} {type_clause} indexed by date = {values}")
|
|
85
|
+
return "\n".join(lines)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _update_series_lines(data: pd.DataFrame, db_alias: str) -> str:
|
|
89
|
+
"""Build one UPDATE statement per column, writing into existing objects.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
data: wide DataFrame, datetime index (one row per period, no gaps,
|
|
93
|
+
sorted ascending), one column per series (column name = FAME
|
|
94
|
+
object name).
|
|
95
|
+
db_alias: channel alias from the OPEN ... AS clause.
|
|
96
|
+
"""
|
|
97
|
+
lines = []
|
|
98
|
+
for name in data.columns:
|
|
99
|
+
values = ", ".join(f"{v:g}" for v in data[name])
|
|
100
|
+
lines.append(f"update {name} = {values}")
|
|
101
|
+
return "\n".join(lines)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def create_fame_db(
|
|
105
|
+
data: pd.DataFrame,
|
|
106
|
+
freq: str,
|
|
107
|
+
start_date: str,
|
|
108
|
+
end_date: str,
|
|
109
|
+
db_path: str,
|
|
110
|
+
db_alias: str = "mydb",
|
|
111
|
+
famedb: str = "sl-fame-p1",
|
|
112
|
+
precision: bool = True,
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Create a brand-new FAME database and populate it with the given series.
|
|
115
|
+
|
|
116
|
+
WARNING: uses ACCESS OVERWRITE, which replaces any existing file at
|
|
117
|
+
db_path entirely. Call this only for a genuinely new database -- use
|
|
118
|
+
update_fame_db for writing to one that already exists.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
data: wide DataFrame, datetime index, one column per series.
|
|
122
|
+
freq: FAME frequency keyword, e.g. "monthly".
|
|
123
|
+
start_date: Date str for from date, format YYYY:MM.
|
|
124
|
+
end_date: Date str for to date.
|
|
125
|
+
db_path: path to the FAME database file to create.
|
|
126
|
+
db_alias: channel alias for the OPEN ... AS clause.
|
|
127
|
+
famedb: FAME server hostname.
|
|
128
|
+
precision: type used for the newly created series.
|
|
129
|
+
Returns:
|
|
130
|
+
None
|
|
131
|
+
"""
|
|
132
|
+
_assert_no_gaps(data)
|
|
133
|
+
MODULE_DIR = Path(__file__).resolve().parent
|
|
134
|
+
original_script = (MODULE_DIR / "fame_prog" / "opprett_db_template.inp").read_text()
|
|
135
|
+
|
|
136
|
+
params = {
|
|
137
|
+
"DB_PATH": db_path,
|
|
138
|
+
"FREQ": freq,
|
|
139
|
+
"FROM_DATE": start_date,
|
|
140
|
+
"TO_DATE": end_date,
|
|
141
|
+
"MYDB": db_alias,
|
|
142
|
+
"SERIES_LINES": _create_series_lines(data, db_alias, precision),
|
|
143
|
+
}
|
|
144
|
+
_run_fame_script(_inject_params(original_script, params), famedb)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def update_fame_db(
|
|
148
|
+
data: pd.DataFrame,
|
|
149
|
+
freq: str,
|
|
150
|
+
start_date: str,
|
|
151
|
+
end_date: str,
|
|
152
|
+
db_path: str,
|
|
153
|
+
db_alias: str = "mydb",
|
|
154
|
+
famedb: str = "sl-fame-1.ssb.no",
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Write a range of values into an existing FAME database's existing series.
|
|
157
|
+
|
|
158
|
+
Uses ACCESS SHARED, so the write can proceed alongside a concurrent
|
|
159
|
+
reader. Fails against series that don't already exist -- this function
|
|
160
|
+
assumes a fixed, known set of series (see create_fame_db for new ones).
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
data: wide DataFrame, datetime index, one column per series.
|
|
164
|
+
freq: FAME frequency keyword, e.g. "monthly".
|
|
165
|
+
start_date: Date str for from date, format YYYY:MM.
|
|
166
|
+
end_date: Date str for to date.
|
|
167
|
+
db_path: path to the existing FAME database file.
|
|
168
|
+
db_alias: channel alias for the OPEN ... AS clause.
|
|
169
|
+
famedb: FAME server hostname.
|
|
170
|
+
Returns:
|
|
171
|
+
None
|
|
172
|
+
"""
|
|
173
|
+
# _assert_no_gaps(data)
|
|
174
|
+
MODULE_DIR = Path(__file__).resolve().parent
|
|
175
|
+
original_script = (
|
|
176
|
+
MODULE_DIR / "fame_prog" / "oppdater_db_template.inp"
|
|
177
|
+
).read_text()
|
|
178
|
+
|
|
179
|
+
params = {
|
|
180
|
+
"DB_PATH": db_path,
|
|
181
|
+
"FREQ": freq,
|
|
182
|
+
"FROM_DATE": start_date,
|
|
183
|
+
"TO_DATE": end_date,
|
|
184
|
+
"MYDB": db_alias,
|
|
185
|
+
"SERIES_LINES": _update_series_lines(data, db_alias),
|
|
186
|
+
}
|
|
187
|
+
_run_fame_script(_inject_params(original_script, params), famedb)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def get_fame(
|
|
191
|
+
db_path: str,
|
|
192
|
+
csv_path: str,
|
|
193
|
+
freq: str,
|
|
194
|
+
start_date: str,
|
|
195
|
+
end_date: str,
|
|
196
|
+
rounding: str = "auto",
|
|
197
|
+
famedb: str = "sl-fame-p1",
|
|
198
|
+
) -> None:
|
|
199
|
+
"""Read data from a FAME database and write it out to a CSV file.
|
|
200
|
+
|
|
201
|
+
Args:
|
|
202
|
+
db_path: Path to the existing FAME database file.
|
|
203
|
+
csv_path: Path to store csv file.
|
|
204
|
+
freq: FAME frequency keyword, e.g. "monthly".
|
|
205
|
+
start_date: Date str for from date, format YYYY:MM.
|
|
206
|
+
end_date: Date str for to date.
|
|
207
|
+
rounding: Number of decimals, eiher auto or number as str.
|
|
208
|
+
famedb: FAME server hostname.
|
|
209
|
+
Returns:
|
|
210
|
+
None
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
params = {
|
|
214
|
+
"TARGET_DB": db_path,
|
|
215
|
+
"FREQUENCY": freq,
|
|
216
|
+
"START_DATE": start_date,
|
|
217
|
+
"END_DATE": end_date,
|
|
218
|
+
"OUTPUT_FILE": csv_path,
|
|
219
|
+
"DECIMAL": rounding,
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
MODULE_DIR = Path(__file__).resolve().parent
|
|
223
|
+
original_script = (MODULE_DIR / "fame_prog" / "lag_csv_template.inp").read_text()
|
|
224
|
+
script = _inject_params(original_script, params)
|
|
225
|
+
_run_fame_script(script, famedb)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
open <acc rea> "<<TARGET_DB>>" as famedb
|
|
2
|
+
|
|
3
|
+
freq <<FREQUENCY>>
|
|
4
|
+
csv on, delimiter ";"
|
|
5
|
+
over on
|
|
6
|
+
|
|
7
|
+
replace nd null
|
|
8
|
+
replace nc null
|
|
9
|
+
replace na null
|
|
10
|
+
|
|
11
|
+
reset only item
|
|
12
|
+
item class off, series on
|
|
13
|
+
item index case off
|
|
14
|
+
decimal <<DECIMAL>>
|
|
15
|
+
|
|
16
|
+
image date "<year>-<pz>"
|
|
17
|
+
|
|
18
|
+
open <kind text; access overwrite> file("<<OUTPUT_FILE>>") as output_csv
|
|
19
|
+
|
|
20
|
+
loop for x in wildlist(famedb, "?")
|
|
21
|
+
write <crlf off> ";"+lower(name(x)) to output_csv
|
|
22
|
+
end loop
|
|
23
|
+
write null to output_csv
|
|
24
|
+
|
|
25
|
+
loop for s = <<START_DATE>> to <<END_DATE>>
|
|
26
|
+
write <crlf off> datefmt(s) to output_csv
|
|
27
|
+
loop for x in wildlist(famedb, "?")
|
|
28
|
+
write <crlf off> ";"+numfmt(x[s], *, @decimals) to output_csv
|
|
29
|
+
end loop
|
|
30
|
+
write null to output_csv
|
|
31
|
+
end loop
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
type "ferdig"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""A collection of useful functions.
|
|
2
|
+
|
|
3
|
+
The template and this example uses Google style docstrings as described at:
|
|
4
|
+
https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def example_function(number1: int, number2: int) -> str:
|
|
10
|
+
"""Compare two integers.
|
|
11
|
+
|
|
12
|
+
This is merely an example function can be deleted. It is used to show and test generating
|
|
13
|
+
documentation from code, type hinting, testing, and testing examples
|
|
14
|
+
in the code.
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
number1: The first number.
|
|
19
|
+
number2: The second number, which will be compared to number1.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
A string describing which number is the greatest.
|
|
23
|
+
|
|
24
|
+
Examples:
|
|
25
|
+
Examples should be written in doctest format, and should illustrate how
|
|
26
|
+
to use the function.
|
|
27
|
+
|
|
28
|
+
>>> example_function(1, 2)
|
|
29
|
+
1 is less than 2
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
if number1 < number2:
|
|
33
|
+
return f"{number1} is less than {number2}"
|
|
34
|
+
|
|
35
|
+
return f"{number1} is greater than or equal to {number2}"
|
|
File without changes
|
|
File without changes
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Sesonjustering funksjoner for MNR
|
|
2
|
+
|
|
3
|
+
Bruker x13 fra US Census https://www.census.gov/data/software/x13as.X-13ARIMA-SEATS.html#accordion-bfdec081de-item-fb93ad60cf
|
|
4
|
+
|
|
5
|
+
Programmet kjører fra Linux, men vi bruker python til å orkestrere kjøringen og sende inn data.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
import subprocess
|
|
11
|
+
import tempfile
|
|
12
|
+
import re
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
# Directory containing this .py file
|
|
16
|
+
MODULE_DIR = Path(__file__).resolve().parent
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _call_x13(spec: str, out_prefix: str, x13_bin: str) -> None:
|
|
20
|
+
"""Function to call on x13 binary program"""
|
|
21
|
+
result = subprocess.run(
|
|
22
|
+
[str(x13_bin), "-i", spec, "-o", out_prefix], capture_output=True, text=True
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
# print("returncode:", result.returncode)
|
|
26
|
+
# print("STDOUT:\n", result.stdout)
|
|
27
|
+
# print("STDERR:\n", result.stderr)
|
|
28
|
+
|
|
29
|
+
if result.returncode != 0:
|
|
30
|
+
print("STDERR:", result.stderr)
|
|
31
|
+
for f in tmp.glob("*.err"):
|
|
32
|
+
print(f.read_text())
|
|
33
|
+
raise RuntimeError("X-13 failed — see output above")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def run_x13(spec: str, series: str = "x13", hmtl: bool = False, outdir: str = None):
|
|
37
|
+
"""Function to run the x13 binary file.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
spec: Path to spec.
|
|
41
|
+
series: name for series.
|
|
42
|
+
html: Bool of wether or not to run the html version, default false.
|
|
43
|
+
outdir: Dir for temp output from the x13 run.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
None
|
|
47
|
+
"""
|
|
48
|
+
if hmtl:
|
|
49
|
+
x13_bin = MODULE_DIR / "bin" / "x13as_html"
|
|
50
|
+
else:
|
|
51
|
+
x13_bin = MODULE_DIR / "bin" / "x13as"
|
|
52
|
+
|
|
53
|
+
if outdir is None:
|
|
54
|
+
with tempfile.TemporaryDirectory(prefix="x13_") as tmpdir:
|
|
55
|
+
tmpdir = Path(tmpdir)
|
|
56
|
+
out_prefix = tmpdir / series
|
|
57
|
+
|
|
58
|
+
_call_x13(spec=spec, out_prefix=out_prefix, x13_bin=x13_bin)
|
|
59
|
+
|
|
60
|
+
else:
|
|
61
|
+
_call_x13(spec=spec, out_prefix=f"{outdir}/{series}", x13_bin=x13_bin)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
DATA_LINE_PATTERN = re.compile(
|
|
65
|
+
r"^\s*data\s*=\s*\([^)]*\)\s*\n?", re.IGNORECASE | re.MULTILINE
|
|
66
|
+
)
|
|
67
|
+
SERIES_BLOCK_PATTERN = re.compile(r"(series\s*\{)", re.IGNORECASE)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _build_data_line(s: pd.Series) -> str:
|
|
71
|
+
values = [str(v) for v in s.dropna().values]
|
|
72
|
+
lines = "\n".join(f" {v}" for v in values)
|
|
73
|
+
return f" data = (\n{lines}\n )"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _insert_data(spec_text: str, data_line: str) -> str:
|
|
77
|
+
if not SERIES_BLOCK_PATTERN.search(spec_text):
|
|
78
|
+
raise ValueError("No series{ block found in spec file")
|
|
79
|
+
return SERIES_BLOCK_PATTERN.sub(
|
|
80
|
+
lambda m: f"{m.group(1)}\n{data_line}", spec_text, count=1
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _compute_start(s: pd.Series) -> str:
|
|
85
|
+
"""
|
|
86
|
+
Returns the X-13 start= string (e.g. '2016.1') based on the first
|
|
87
|
+
non-null observation in the series, matching what _build_data_line
|
|
88
|
+
actually sends (since it drops NaNs).
|
|
89
|
+
"""
|
|
90
|
+
valid = s.dropna()
|
|
91
|
+
if valid.empty:
|
|
92
|
+
raise ValueError("Series has no valid (non-NaN) observations")
|
|
93
|
+
|
|
94
|
+
first_date = pd.to_datetime(valid.index[0])
|
|
95
|
+
|
|
96
|
+
return f"{first_date.year}.{first_date.month}"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
START_LINE_PATTERN = re.compile(
|
|
100
|
+
r"^\s*start\s*=\s*\S+\s*\n?", re.IGNORECASE | re.MULTILINE
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _set_start(spec_text: str, start: str) -> str:
|
|
105
|
+
spec_text = START_LINE_PATTERN.sub("", spec_text)
|
|
106
|
+
return re.sub(
|
|
107
|
+
r"(data\s*=\s*\([^)]*\))", # <-- single closing paren now, not two
|
|
108
|
+
lambda m: f"{m.group(1)}\n start = {start}",
|
|
109
|
+
spec_text,
|
|
110
|
+
count=1,
|
|
111
|
+
flags=re.DOTALL,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _read_saved_series(path: Path) -> pd.Series:
|
|
116
|
+
df = pd.read_csv(
|
|
117
|
+
path,
|
|
118
|
+
sep=r"\s+",
|
|
119
|
+
skiprows=2, # skip header row + dashed separator
|
|
120
|
+
header=None,
|
|
121
|
+
names=["date", "value"],
|
|
122
|
+
dtype={"date": str},
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# date can be yyyymm (monthly) or yyyyq (quarterly, 5 digits)
|
|
126
|
+
if df["date"].str.len().iloc[0] == 6:
|
|
127
|
+
idx = pd.to_datetime(df["date"], format="%Y%m")
|
|
128
|
+
elif df["date"].str.len().iloc[0] == 5:
|
|
129
|
+
years = df["date"].str[:4].astype(int)
|
|
130
|
+
quarters = df["date"].str[4].astype(int)
|
|
131
|
+
idx = pd.PeriodIndex(
|
|
132
|
+
[f"{y}Q{q}" for y, q in zip(years, quarters)], freq="Q"
|
|
133
|
+
).to_timestamp()
|
|
134
|
+
else:
|
|
135
|
+
raise ValueError(f"Unrecognized date format in {path}: {df['date'].iloc[0]!r}")
|
|
136
|
+
|
|
137
|
+
values = df["value"].astype(float)
|
|
138
|
+
return pd.Series(values.values, index=idx)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _call_x13_from_df(
|
|
142
|
+
df: pd.DataFrame, spec_folder: str, out_prefix: str, x13_bin: str
|
|
143
|
+
) -> pd.DataFrame:
|
|
144
|
+
|
|
145
|
+
all_series = {}
|
|
146
|
+
i = 0
|
|
147
|
+
length = len(df.columns)
|
|
148
|
+
|
|
149
|
+
for col in df.columns:
|
|
150
|
+
try:
|
|
151
|
+
spec_path = Path(f"{spec_folder}/{col}.spc")
|
|
152
|
+
original_text = spec_path.read_text()
|
|
153
|
+
except FileNotFoundError:
|
|
154
|
+
continue
|
|
155
|
+
|
|
156
|
+
pd_series = df[col]
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
modified_text = _insert_data(original_text, _build_data_line(pd_series))
|
|
160
|
+
modified_text = _set_start(modified_text, _compute_start(pd_series))
|
|
161
|
+
spec_path.write_text(modified_text)
|
|
162
|
+
|
|
163
|
+
_call_x13(
|
|
164
|
+
spec=str(spec_path)[:-4],
|
|
165
|
+
out_prefix=f"{out_prefix}/{col}",
|
|
166
|
+
x13_bin=x13_bin,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
finally:
|
|
170
|
+
spec_path.write_text(original_text)
|
|
171
|
+
|
|
172
|
+
all_series[f"{col}.s"] = _read_saved_series(f"{out_prefix}/{col}.d11")
|
|
173
|
+
all_series[f"{col}.t"] = _read_saved_series(f"{out_prefix}/{col}.d12")
|
|
174
|
+
all_series[f"{col}.i"] = _read_saved_series(f"{out_prefix}/{col}.d13")
|
|
175
|
+
|
|
176
|
+
i += 1
|
|
177
|
+
|
|
178
|
+
if i % 100 == 0:
|
|
179
|
+
print(f"Seasonaly adjusted series {i} of {length}")
|
|
180
|
+
|
|
181
|
+
return pd.concat(all_series, axis=1)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def run_x13_from_df(
|
|
185
|
+
df: pd.DataFrame,
|
|
186
|
+
spec_folder: str,
|
|
187
|
+
series: str = "x13",
|
|
188
|
+
hmtl: bool = False,
|
|
189
|
+
outdir: str = None,
|
|
190
|
+
):
|
|
191
|
+
"""Function to run the x13 binary file.
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
df: Pandas df with data to adjust.
|
|
195
|
+
spec: Path to spec.
|
|
196
|
+
series: name for series.
|
|
197
|
+
html: Bool of wether or not to run the html version, default false.
|
|
198
|
+
outdir: Dir for temp output from the x13 run.
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
None
|
|
202
|
+
"""
|
|
203
|
+
if hmtl:
|
|
204
|
+
x13_bin = MODULE_DIR / "bin" / "x13as_html"
|
|
205
|
+
else:
|
|
206
|
+
x13_bin = MODULE_DIR / "bin" / "x13as"
|
|
207
|
+
|
|
208
|
+
if outdir is None:
|
|
209
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
210
|
+
tmpdir = Path(tmpdir)
|
|
211
|
+
out_prefix = tmpdir
|
|
212
|
+
|
|
213
|
+
df_seasonal = _call_x13_from_df(
|
|
214
|
+
df=df, spec_folder=spec_folder, out_prefix=out_prefix, x13_bin=x13_bin
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
else:
|
|
218
|
+
df_seasonal = _call_x13_from_df(
|
|
219
|
+
df=df, spec_folder=spec_folder, out_prefix=f"{outdir}", x13_bin=x13_bin
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
return df_seasonal
|