weatherdb 1.1.0__py3-none-any.whl

docs/source/conf.py

@@ -0,0 +1,137 @@
+# Configuration file for the Sphinx documentation builder.
+
+import sys
+from pathlib import Path
+
+# -- Path setup --------------------------------------------------------------
+
+src_path = Path(__file__).parent
+base_path = src_path.parents[1]
+sys.path.insert(0, base_path.resolve().as_posix())
+
+import weatherdb
+
+# -- Project information -----------------------------------------------------
+
+project = 'WeatherDB'
+copyright = weatherdb.__copyright__
+author = weatherdb.__author__
+author_email = weatherdb.__email__
+
+# The full version, including alpha/beta/rc tags
+release = weatherdb.__version__
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.napoleon',
+    'sphinx_rtd_theme',
+    'sphinx.ext.autodoc',
+    'nbsphinx',
+    'myst_parser',
+    'sphinx.ext.viewcode',
+    'autoclasstoc',
+    'sphinx.ext.autosummary',
+    'sphinx_click',
+    "sphinx.ext.autosectionlabel",
+    'sphinx_copybutton',
+    'sphinx_design',
+    'sphinx.ext.intersphinx',
+    "sphinx_new_tab_link",
+    "sphinx.ext.mathjax"
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sync.ffs_db']
+
+source_suffix = [".rst", ".md"]
+
+source_parsers = {
+   '.md': "markdown"
+}
+
+myst_enable_extensions = [
+    "dollarmath",
+    "colon_fence",
+    "attrs_inline"
+]
+
+# Autodoc options
+autodoc_default_options = {
+    'member-order': 'bysource',
+    'undoc-members': True,
+    'show-inheritance': True,
+    'inherited-members':True,
+    'collapse_navigation': False,
+}
+autodoc_default_flags=["show-inheritance"]
+autoclass_content= "both"
+autodoc_inherit_docstrings= True
+
+autoclasstoc_sections = [
+    'public-methods',
+]
+
+# Autosummary options
+autosummary_generate = True
+autosummary_generate_overwrite = True
+
+# intersphinx
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+    'sqlalchemy': ('https://docs.sqlalchemy.org/en/20/', None),
+    'pandas': ('https://pandas.pydata.org/docs/', None),}
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "sphinx_rtd_theme"
+
+html_theme_options = {
+    'version_selector': True,
+    'prev_next_buttons_location': 'bottom',
+    'style_external_links': True,
+    # TOC options
+    'collapse_navigation': True,
+    'sticky_navigation': True,
+    'navigation_depth': -1,
+}
+
+html_css_files = [
+    'custom.css',
+]
+html_logo = "_static/logo.png"
+html_favicon = "_static/favicon.ico"
+
+html_context = {
+    # display gitlab
+    "display_gitlab": True,
+    "gitlab_host": "https://gitlab.uni-freiburg.de",
+    "gitlab_user": "hydrology",
+    "gitlab_repo": "weatherDB",
+    "gitlab_version": "master",
+    "conf_py_path": "/docs/source/", # Path in the checkout to the docs root
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for PDF Output --------------------------------------------------
+latex_engine = "pdflatex"
+latex_theme = "manual"
+
+
+# run with command: sphinx-build -b html .\source .\html

docs/source/index.rst

@@ -0,0 +1,33 @@
+#####################################
+Welcome to WeatherDB's documentation!
+#####################################
+
+
+The WeatherDB module offers an API to interact with the automatically filled WeatherDB Database.
+
+There are 4 different main sub modules with their corresponding classes you might need to use.
+
+- :py:data:`weatherdb.config`:
+   This submodule has only one main object :py:class:`config <weatherdb.config.ConfigParser>`. This object is used to configure the module, like e.g. saving the database connection.
+- :py:mod:`weatherdb.station`:
+   Has a class for every type of station. E.g. :py:class:`StationP <weatherdb.station.StationP>` for precipitation.
+   One object represents one Station with one parameter.
+   This object can get used to get the corresponding timeserie.
+   There is also a :py:class:`GroupStation <weatherdb.station.GroupStation>` class that groups the three parameters precipitation, temperature and potential evapotranspiration together for one station. If one parameter is not available for a specific station this one won't get grouped.
+- :py:mod:`weatherdb.stations`:
+   Is a grouping class for all the stations of one measurement parameter. E.g. :py:class:`StationsP <weatherdb.stations.StationsP>`.
+   Can get used to do actions on all the stations.
+- :py:mod:`weatherdb.broker`:
+   This submodule has only one class :py:class:`Broker <weatherdb.broker.Broker>`. This one is used to do actions on all the stations together. Mainly only used for updating the Database.
+
+To get started follow th installation guide and the setup guide first. After that you can use the quickstart guide to get a first impression of the module.
+
+.. toctree::
+   :maxdepth: 6
+   :hidden:
+
+   Setup <setup/setup>
+   Method <Methode.md>
+	API reference <api/api>
+   Change-log <Changelog.md>
+   License <License>

docs/source/setup/Configuration.md

@@ -0,0 +1,127 @@
+# Configuration
+
+The WeatherDB module has several configurations you can make. Those are stored in a INI-File.
+
+## create user configuration
+
+To configure your WeatherDB module, you need to create a user configuration file somewhere on your system. To do so:
+
+::::{tab-set}
+:sync-group: category
+
+:::{tab-item} Python
+:sync: python
+
+```{code-block} python
+import weatherdb as wdb
+wdb.config.create_user_config()
+```
+:::
+
+:::{tab-item} Bash
+:sync: bash
+
+```{code-block} bash
+weatherdb create-user-config
+```
+:::
+
+::::
+
+This now created a INI-File with all possible configurations, but all values are commented out and you will see the default value.
+After creating the file, you can edit the configuration file with any Texteditor and WeatherDB will use those configurations on your next import.
+
+## Setup database
+
+To get started you need to setup your PostGreSQL database the WeatherDB module should use.
+
+There are two usage scenarios:
+1. your working with an existing WeatherDB-database someone else did setup.
+   Then continue with this guide
+2. you want to create your own WeatherDB database instance.
+   Then first follow the [hosting instructions](<Hosting.md>).
+
+(setup-main-con)=
+### setup main connection
+
+To setup you database connection open the previously created user configuration file find the `[database:main]`{l=ini} section and uncomment the options and fill them out with your credentials:
+
+```ini
+[database:main]
+; setup a database connection
+; The password should not be stored in the config file, as it is a security risk.
+; the module will ask for the password on first execution and store it in the keyring.
+HOST = localhost
+PORT = 5432
+DATABASE = weatherdb
+USER = weatherdb
+```
+
+To set your password you will just have to use the module for the first time and will then get prompted to enter the password. This password is then securely stored in your keyring.
+
+:::{warning}
+Don't add a setting for your password in the ini file as this would be a security risk.
+:::
+
+:::{tip}
+If you use the database at the hydrology department of Freiburg, please go to the [apps.hydro.intra.uni-freiburg.de/weatherdb](https://apps.hydro.intra.uni-freiburg.de/weatherdb). There you can create yourself an account and download your login credentials from your profile page ("API Password").
+:::
+
+#### example
+
+For example for the user *philip*, who's using the UNI Freiburg internal database *weatherdb* on host *fuhys017.public.ads.uni-freiburg.de* with port *5432* this part will look like:
+
+```ini
+[database:main]
+; setup a database connection
+; The password should not be stored in the config file, as it is a security risk.
+; the module will ask for the password on first execution and store it in the keyring.
+HOST = fuhys017.public.ads.uni-freiburg.de
+PORT = 5432
+DATABASE = weatherdb
+USER = philip
+```
+
+### multiple connections
+
+The WeatherDB configuration also allows you to work with multiple database connections. This works similarly as with the [main database](#setup-main-connection), but you don't use the `[connection:main]`{l=ini} section of the configuration file, but add a custom connection subsection by giving it any name of your choice `[connection:my_con_name]`{l=ini}.
+
+Then you have to tell WeatherDB to use this connection:
+
+```python
+import weatherdb as wdb
+
+wdb.config.set("database", "connection", "my_con_name")
+```
+
+Alternatively you can also set this custom database connection to be used as the default connection. To do so look for the `[connection:main]`{l=ini} section in the user configuration file and change the `connection=`{l=ini} value to your **my_con_name**:
+
+
+```ini
+[database]
+; These are the main database settings
+; The database is created with the cli command create-db-schema or the weatherdb.Broker().create_db_schema() method
+; you can define multiple database connections by adding a new section like [database.connection_name], where you can define your connection name
+; The connection setting defines which connection is used if not specified
+connection = my_con_name
+```
+
+:::{admonition} Example
+:class: tip
+
+```ini
+[database]
+connection = my_con_name
+
+[database:my_con_name]
+HOST = fuhys017.public.ads.uni-freiburg.de
+PORT = 5432
+DATABASE = weatherdb
+USER = ada
+```
+:::
+
+
+:::{attention}
+ make sure to replace **my_con_name** with your chosen name in every statement.
+:::

docs/source/setup/Hosting.md

@@ -0,0 +1,9 @@
+# Hosting
+
+This documentation is yet to come
+
+## create database
+
+## download rasters
+
+%TODO: create this

docs/source/setup/Install.md

@@ -0,0 +1,49 @@
+# Installation
+
+## Install WeatherDB module
+
+To install the package use PIP to install the Github repository:
+
+```batch
+pip install weatherdb
+```
+
+If you also want to install the optional dependencies use:
+
+```batch
+pip install weatherdb[optionals]
+```
+
+Or to upgrade use:
+
+```batch
+pip install weatherdb --upgrade
+```
+
+After installing the package you need to [configure the module](<project:Configuration.md>).
+
+## How-to install python
+
+If you never used python before, this section is for you. 
+
+To use this package you obviously need Python with several packages installed.
+
+One way to install python is by installing [Anaconda](https://www.anaconda.com/products/distribution).
+
+After the installation you should create yourself a virtual environment. This is basically a folder with all your packages installed and some definition files to set the appropriate environment variables...
+To do so use (in Anaconda Terminal):
+
+```batch
+conda create --name your_environment_name python=3.8
+```
+
+
+Afterwards you need to activate your environment and then install the requirements:
+
+```batch
+conda activate your_environment_name
+conda install shapely numpy geopandas pandas sqlalchemy
+conda install -c conda-forge rasterio psycopg2 pip
+pip install progressbar2
+```
+

docs/source/setup/Quickstart.md

@@ -0,0 +1,183 @@
+# Quick-start
+
+After installing and setting up the user configuration file with your database credentials you are ready to use the package.
+This page should show you the basic usage of the package.
+
+The package is divided in 2 main submodules:
+- **weatherdb.station:**<br>
+  This module has a class for every type of station. E.g. StationP (or StationP).
+  One object represents one Station with one parameter.
+  This object can get used to get the corresponding timeserie.
+  There is also a StationGroup class that groups the three parameters precipitation, temperature and evapotranspiration together for one station.
+- **weatherdb.stations:**<br>
+  This module has grouping classes for all the stations of one parameter. E.G. StationsP (or StationsP) groups all the Precipitation Stations available.
+  Those classes can get used to do actions on all the stations.
+
+The basics of those modules are explained here, but every class and method has way more parameters to get exactly what you want. PLease use the API-reference for more information.
+
+## download
+### single station
+
+If you want to download data for a single station, you have to create an object of the respective class, by gibing the station id you are interested in. This station object can then get used to download the data from the database.
+
+If you want e.g. to download all the 10 minute, filled and Richter corrected precipitation data for the DWD station in Freiburg(station ID = 1443), then you can go like:
+
+```
+from weatherdb import StationP
+stat_p = StationP(1443)
+df = stat_p.get_corr()
+```
+
+If you are only interested in a small timespan, provide the period parameter with the upper and lower time limit. If e.g. you want the data for the years 2000-2010 do:
+```
+df = stat_p.get_corr(period=("2000-01-01", "2010-12-31"))
+```
+
+If you are not interested in the filled and Richter-corrected data, but want e.g. the raw data, add the kind parameter to your query. Like e.g.:
+```
+df = stat_p.get_raw()
+```
+Or use the more general function with the wanted kind parameter.
+```
+df = stat_p.get_df(kinds=["raw"])
+```
+There are 3-5 different kinds of timeseries available per station object depending on the class.
+So there is:
+- "raw" : the raw measurements as on the DWD server
+- "qc"  : The quality checked data
+- "filled" : The filled timeseries
+- "filled_by" : The station ID of the station from which the data was taken to fill the measurements
+- "corr"    : The Richter corrected timeserie.
+
+If you want more than just one kind of timeseries, e.g. the filled timeseries, together with the id from which station the respective field got filled with use:
+```
+df = stat_p.get_df(kinds=["filled", "filled_by"])
+```
+
+If you only need daily values, you can hand in the agg_to parameter. This will also make your query faster, because not as much data has to get transmitted over the network.
+```
+df = stat_p.get_df(agg_to="day")
+```
+
+Similar to the precipitation, you can also work with the Temperature and potential Evapotranspiration data:
+```
+from weatherdb import StationT, StationET
+stat_t = StationT(1443)
+stat_et = StationET(1443)
+period = ("2000-01-01", "2010-12-31")
+df_t = stat_t.get_df(
+    period=period,
+    kinds=["raw", "filled"])
+df_et = stat_t.get_df(
+    period=period,
+    kinds=["raw", "filled"])
+```
+
+So to download the 3 parameters N, T and ET from one station you could create the 3 single station objects and then have 3 different timeseries. But the better solution is to use the GroupStation class. This class groups all the available parameters for one location. Here is an example, how you could use it to get a Dataframe with the filled data:
+```
+from weatherdb import GroupStation
+stat = GroupStation(1443)
+df = stat.get_df(
+    period=("2000-01-01", "2010-12-31"),
+    kind="filled",
+    agg_to="day")
+```
+
+### multiple stations
+If you want to download the data for multiple stations. Like e.g. the station in Freiburg (1443) and the station on the Feldberg (1346) it is recommended to use the classes in the stations module.
+
+To use the stations-module, you first have to create an object and then hand the station ids you are interested in when downloading it:
+```
+from weatherdb import StationsP
+stats_n = StationsP()
+df = stats_n.get_df(
+    stids=[1443, 1346],
+    period=("2000-01-01", "2010-12-31"),
+    kind="filled")
+```
+
+## create timeseries files
+You can also use the module to quickly create the csv-timeseries needed by RoGeR. Either for one station:
+
+```
+from weatherdb import GroupStation
+stat = GroupStation(1443)
+df = stat.create_roger_ts(
+    dir="path/to/the/directory/where/to/save")
+```
+
+or for multiple stations, you can use the GroupStations. This will create a subdirectory for ever station. It is also possible to save in a zip file, by simply giving the path to a zip file. (will get created):
+
+```
+from weatherdb import GroupStations
+stats = GroupStations()
+df = stats.create_roger_ts(
+    stids=[1443, 1346],
+    dir="path/to/the/directory/where/to/save")
+```
+If you don't want to use the RoGeR format for the timestamp you can use the `.create_ts()` method. This method also offers you way more possibilities to define the output, like e.g. adding the share of NAs in the aggregation step or adding the filled_by column.
+
+```
+from weatherdb import GroupStations
+stats = GroupStations()
+df = stats.create_ts(
+    stids=[1443, 1346],
+    dir="path/to/the/directory/where/to/save")
+
+# or for one station
+from weatherdb import GroupStation
+stat = GroupStation(1443)
+df = stat.create_ts(
+    dir="path/to/the/directory/where/to/save")
+```
+
+## get meta information
+If you need more information about the stations you can get the meta data for a single station:
+
+```
+from weatherdb import StationP
+stat = StationP(1443)
+meta_dict = stat.get_meta()
+```
+
+or for multiple stations, you can use the Stations class and get a GeoDataFrame as output with all the stations information.
+
+```
+from weatherdb import StationsP
+stats = StationsP(
+    stids=[1443, 1346])
+df = stats.get_meta()
+```
+
+Furthermore you can also get all the information for every parameter of one station by using the GroupStation class:
+```
+from weatherdb import GroupStation
+gstat = GroupStation(1443)
+df = gstat.get_meta()
+```
+
+To get an explanation about the available meta information you can use the get_meta_explanation method:
+```
+from weatherdb import StationP, StationsP
+stat = StationP(1443)
+explain_df = stat.get_meta_explanation()
+# or
+stats = StationsP()
+explain_df = stats.get_meta_explanation()
+```
+
+If you are only interested in some information you can use the infos parameter like:
+```
+from weatherdb import StationP
+stat = StationP(1443)
+filled_period_1443 = stat.get_meta(infos=["filled_from", "filled_until"])
+```
+but to get the filled period you can also use the get_period method, like:
+```
+from weatherdb import StationP
+stat = StationP(1443)
+filled_period_1443 = stat.get_period_meta(kind="filled")
+```
+
+This should give you an first idea on how to use the package. Feel free to try out and read the API reference section to get more information.
+

docs/source/setup/setup.rst

@@ -0,0 +1,12 @@
+setup
+=====
+
+This chapter will give you an introduction on how to install and get started with this module.
+
+.. toctree::
+   :maxdepth: 2
+
+   Installation <Install.md>
+   Configuration <Configuration.md>
+   Quick-start <Quickstart.md>
+   Hosting <Hosting.md>

weatherdb/__init__.py

@@ -0,0 +1,24 @@
+__author__ = "Max Schmit"
+__email__ = "max.schmit@hydrology.uni-freiburg.de"
+__copyright__ = "Copyright 2024, Max Schmit"
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = "0.0.0" # not set when running from source
+
+from .utils.logging import remove_old_logs, setup_logging_handlers
+from . import station, stations, broker
+from .station import StationP, StationPD, StationT, StationET, GroupStation
+from .stations import StationsP, StationsPD, StationsT, StationsET, GroupStations
+from .config import config
+from .broker import Broker
+
+
+remove_old_logs()
+setup_logging_handlers()
+
+__all__ = ["StationP", "StationPD", "StationT", "StationET", "GroupStation",
+           "StationsP", "StationsPD", "StationsT", "StationsET", "GroupStations",
+           "Broker",
+           "station", "stations", "broker", "config"]

weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.1.0"

weatherdb/alembic/README.md

@@ -0,0 +1,8 @@
+# database migrations
+
+To migrate the database alembic is used. Alembic is a lightweight database migration tool for SQLAlchemy. It is used to generate and apply migrations to the database. The migrations are generated for each version of the WeatherDB module.
+
+To use alembic, you have to point to the `alembic.ini` file in the `weatherdb/alembic` directory, like:
+```bash
+alembic -c weatherdb_module_path/alembic/alembic.ini {command}
+```

weatherdb/alembic/alembic.ini

@@ -0,0 +1,80 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = %(here)s/../../
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+exclude_tables = spatial_ref_sys,layer,topology
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME

weatherdb/alembic/config.py

@@ -0,0 +1,9 @@
+from alembic.config import Config
+from pathlib import Path
+
+from ..config import config
+from ..db.connections import db_engine
+
+config = Config(
+    Path(config.get("main", "module_path"))/"alembic"/"alembic.ini",
+    attributes={"engine": db_engine.get_engine()})