toolsos 0.1.4__tar.gz → 0.2.6__tar.gz

{toolsos-0.1.4 → toolsos-0.2.6}/PKG-INFO

@@ -1,13 +1,13 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: toolsos
-Version: 0.1.4
+Version: 0.2.6
 Summary: OS tools
 Author-email: OS <d.schmitz@amsterdam.nl>
 Keywords: tools,Onderzoek & Statistiek
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
@@ -63,5 +63,12 @@ Instructions on building a package can be found [here](https://packaging.python.
 
 - make a pypi account
 - ask to be added as collaborator to toolsos
-- first update twine: py -m pip install --upgrade twin
-- upload to pypi: twine upload dist/*
+- first update twine: py -m pip install --upgrade twine
+- upload to pypi: twine upload dist/* --skip-existing
+
+## Install to local enviroment for testing
+
+- python -m venv local (maak een lokale venv aan)
+- local\Scripts\activate (activeer de venv)
+- pip install -e . (installer toolsos)
+- pip install -r local_requirements.txt (installeer de benodigde dependencies)

{toolsos-0.1.4 → toolsos-0.2.6}/README.md

@@ -34,5 +34,12 @@ Instructions on building a package can be found [here](https://packaging.python.
 
 - make a pypi account
 - ask to be added as collaborator to toolsos
-- first update twine: py -m pip install --upgrade twin
-- upload to pypi: twine upload dist/*
+- first update twine: py -m pip install --upgrade twine
+- upload to pypi: twine upload dist/* --skip-existing
+
+## Install to local enviroment for testing
+
+- python -m venv local (maak een lokale venv aan)
+- local\Scripts\activate (activeer de venv)
+- pip install -e . (installer toolsos)
+- pip install -r local_requirements.txt (installeer de benodigde dependencies)

{toolsos-0.1.4 → toolsos-0.2.6}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "toolsos"
-version = "0.1.4"
+version = "0.2.6"
 description = "OS tools"
 readme = "README.md"
 authors = [{ name = "OS", email = "d.schmitz@amsterdam.nl" }]
@@ -25,7 +25,7 @@ keywords = ["tools", "Onderzoek & Statistiek"]
 # use pip install "toolsos[all]" to pip install with al dependencies
 
 dependencies = []
-requires-python = ">=3.11"
+requires-python = ">=3.10"
 
 [project.optional-dependencies]
 dev = [
@@ -49,5 +49,5 @@ all = [
     "sqlalchemy",
     ]
 
-#[project.urls]
-#Homepage = "https://github.com/realpython/reader"
+# [project.urls]
+#Homepage = ""

{toolsos-0.1.4 → toolsos-0.2.6}/src/toolsos/cbs_tools.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import json
 import pickle
+from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Iterator, Optional, Any
+from typing import TYPE_CHECKING, Any, Iterator, Optional
 
 import pandas as pd
 import pyarrow as pa
@@ -14,18 +15,27 @@ if TYPE_CHECKING:
     import pyreadstat
 
 
+def get_batch_size(path, memory_limit):
+    df, _ = prs.read_sav(path, row_limit=1000)
+
+    # memory in megabytes
+    mem_size = df.memory_usage().sum() / 1_000_000
+
+    # The amount of blocks (of a thousand rows fit in the memory_limit)
+    n_blocks = memory_limit / mem_size
+
+    # Calculate the number of rows that fit within the memory limit
+    return round(n_blocks * 1000)
+
+
 class SavToParquet:
     def __init__(
-        self,
-        file: str,
-        folder_out: str,
-        chunksize: Optional[int] = None,
-        verbose: bool = False,
+        self, file: str, folder_out: str, verbose: bool = False, memory_limit=10_000
     ) -> None:
         self.file = file
         self.folder_out = folder_out
         self.verbose = verbose
-        self.chunksize = 5_000_000 if not chunksize else chunksize
+        self.memory_limit = memory_limit
 
     @property
     def path_out(self) -> str:
@@ -33,20 +43,27 @@ class SavToParquet:
 
     @property
     def chunks(self) -> Iterator[tuple["pyreadstat.metadata_container", pd.DataFrame]]:
-        return prs.read_file_in_chunks(
-            prs.read_sav, self.file, chunksize=self.chunksize
-        )
 
-    def get_meta(self) -> Iterator:
-        return prs.read_sav(self.file, row_limit=10)
+        chunksize = get_batch_size(self.file, self.memory_limit)
+
+        if self.verbose:
+            print(f"Reading file in blocks of {chunksize} rows")
+            print("One such block should fit within the memory limit")
+
+        return prs.read_file_in_chunks(prs.read_sav, self.file, chunksize=chunksize)
 
     def write_meta_to_json(self) -> None:
         json_path = self.path_out.replace(".parquet", "_meta.json")
 
         meta_dict = {}
-        for attr in dir(self.meta):
-            if not attr.startswith("__"):
-                meta_dict[attr] = getattr(self.meta, attr)
+        for attr_name in dir(self.meta):
+            if not attr_name.startswith("__"):
+                attr = getattr(self.meta, attr_name)
+
+                if isinstance(attr, datetime):
+                    attr = attr.strftime("%Y-%m-%d %H:%M:%S")
+
+                meta_dict[attr_name] = attr
 
         with open(json_path, "w") as file:
             json.dump(meta_dict, file)
@@ -58,10 +75,12 @@ class SavToParquet:
             pickle.dump(self.meta, file)
 
     def write_to_parquet(self) -> None:
-        meta_df, self.meta = self.get_meta()
-        schema = table = pa.Table.from_pandas(meta_df).schema
 
         print("Writing table")
+
+        line1, self.meta = prs.read_sav(self.file, row_limit=1)
+        schema = pa.Table.from_pandas(line1).schema
+
         with pq.ParquetWriter(self.path_out, schema) as writer:
             for idx, (df, _) in enumerate(self.chunks):
                 if self.verbose:

{toolsos-0.1.4/src/toolsos → toolsos-0.2.6/src/toolsos/database}/database_connection.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 import getpass
 import json
+import os
 import subprocess
 from json import JSONDecodeError
+from pathlib import Path
 from typing import Optional
 
 import keyring
@@ -33,7 +35,7 @@ def get_db_connection_strings(
     for dbname, params in db_info.items():
         flush = dbname in reset_pw if reset_pw else False
 
-        if params["pw"] == "acces_token":
+        if params["password"] == "access_token":
             pw = get_azure_access_token()
         else:
             pw = get_pw_from_keyring(dbname=dbname, user=params["user"], reset_pw=flush)
@@ -94,21 +96,60 @@ def get_azure_access_token():
     result = subprocess.run(command, capture_output=True, shell=True, text=True)
 
     try:
-        json.loads(result.stdout)["accessToken"]
+        return json.loads(result.stdout)["accessToken"]
     except JSONDecodeError:
         subprocess.run("az login", shell=True)
 
 
-if __name__ == "__main__":
-    ...
-    # Examples
+def get_token_from_pgpass() -> None:
+    p = Path(os.getenv("APPDATA")) / "postgresql" / "pgpass.conf"
+    with open(p) as f:
+        token = f.readline().split(":")[4]
 
-    # Get database connection settings from yaml
-    # engine_strings = get_db_connection_strings("python/database_config.yml")
-    # print(engine_strings.ruimte_analyse222)
+    return token
 
-    # Get database connection settings from yaml and reset password
-    # engine_strings = get_db_connection_strings(
-    #     "python/database_config.yml", reset_pw=["ruimte_analyse222"]
-    # )
-    # print(engine_strings.ruimte_analyse222)
+
+def write_pgpass(
+    host: str, port: str, database: str, user: str, path: str | None = None
+) -> None:
+    password = get_azure_access_token()
+    conn_string = f"{host}:{port}:{database}:{user}:{password}"
+
+    if not path:
+        if os.name == "nt":
+            path = Path(os.getenv("APPDATA")) / "postgresql" / "pgpass.conf"
+        else:
+            path = Path("$home/.pgpass")
+
+    if not path.parent.exists():
+        path.parent.mkdir()
+
+    with open(path, "w") as f:
+        f.write(conn_string)
+
+    if os.name != "nt":
+        path.chmod("0600")
+
+
+def write_multiple_pgpass(conn_details, path: str | None = None):
+    password = get_azure_access_token()
+
+    conn_strings = []
+    for c in conn_details:
+        c_string = f'{c["host"]}:{c["port"]}:{c["database"]}:{c["user"]}:{password}'
+        conn_strings.append(c_string)
+
+    if not path:
+        if os.name == "nt":
+            path = Path(os.getenv("APPDATA")) / "postgresql" / "pgpass.conf"
+        else:
+            path = Path("$home/.pgpass")
+
+    if not path.parent.exists():
+        path.parent.mkdir()
+
+    with open(path, "w") as f:
+        f.writelines(line + "\n" for line in conn_strings)
+
+    if os.name != "nt":
+        path.chmod("0600")

{toolsos-0.1.4 → toolsos-0.2.6}/src/toolsos/geo.py

@@ -17,7 +17,7 @@ def get_geo_json(
     Returns:
         dict[str, str]: geo json containg of the desired level and year
     """
-    base_url = "https://gitlab.com/os-amsterdam/datavisualisatie-onderzoek-en-statistiek/-/raw/main/geo/"
+    base_url = "https://gitlab.com/os-amsterdam/datavisualisatie-onderzoek-en-statistiek/-/raw/main/public/geo/"
 
     if mra:
         level = f"{level}-mra"

{toolsos-0.1.4 → toolsos-0.2.6}/src/toolsos/huisstijl/graphs/bargraph.py

@@ -1,4 +1,5 @@
 import plotly.express as px
+
 from .styler import BaseStyle
 
 basestyle = BaseStyle()
@@ -14,6 +15,7 @@ def bar(
     barmode=None,
     width=750,
     height=490,
+    font="Amsterdam Sans",
     **kwargs,
 ):
     fig = px.bar(
@@ -21,7 +23,7 @@ def bar(
         x=x,
         y=y,
         color=color,
-        template=basestyle.get_base_template("bar", orientation=orientation),
+        template=basestyle.get_base_template("bar", orientation=orientation, font=font),
         width=width,
         color_discrete_sequence=color_discrete_sequence,
         height=height,
@@ -43,6 +45,7 @@ def stacked_single(
     color: str = None,
     color_discrete_sequence: list = None,
     orientation="v",
+    font="Amsterdam Sans",
     **kwargs,
 ):
     fig = bar(
@@ -53,6 +56,7 @@ def stacked_single(
         color_discrete_sequence=color_discrete_sequence,
         barmode="relative",
         orientation=orientation,
+        font=font,
         **kwargs,
     )
 
@@ -71,6 +75,7 @@ def stacked_multiple(
     color: str = None,
     color_discrete_sequence: list = None,
     orientation="v",
+    font="Amsterdam Sans",
     **kwargs,
 ):
     fig = bar(
@@ -81,6 +86,7 @@ def stacked_multiple(
         color_discrete_sequence=color_discrete_sequence,
         barmode="stack",
         orientation=orientation,
+        font=font,
         **kwargs,
     )
 
@@ -94,6 +100,7 @@ def grouped(
     color: str = None,
     color_discrete_sequence: list = None,
     orientation="v",
+    font="Amsterdam Sans",
     **kwargs,
 ):
     fig = bar(
@@ -104,6 +111,7 @@ def grouped(
         color_discrete_sequence=color_discrete_sequence,
         barmode="group",
         orientation=orientation,
+        font=font,
         **kwargs,
     )
 
@@ -116,6 +124,7 @@ def single(
     y: str,
     color_discrete_sequence: list = None,
     orientation="v",
+    font="Amsterdam Sans",
     **kwargs,
 ):
     fig = bar(
@@ -124,6 +133,7 @@ def single(
         y=y,
         color_discrete_sequence=color_discrete_sequence,
         orientation=orientation,
+        font=font,
         **kwargs,
     )
 

toolsos-0.2.6/src/toolsos/huisstijl/graphs/graph_styles.py

@@ -0,0 +1,32 @@
+font = "Amsterdam Sans ExtraBold, Corbel"
+# font = "Arial"
+
+STYLE_OLD = {
+    "font": {"family": "Corbel", "size": 15},
+    "plot_bgcolor": "#FFFFFF",
+    "gridline_color": "#E8E8E8",
+    "gridlinewidth": 0.75,
+    "showline": True,
+    "showgrid": False,
+    "zerolinecolor": "#E8E8E8",
+    "gridcolor": "#E8E8E8",
+    "separators": ",",
+}
+
+
+STYLE_NEW = {
+    "font_bold": {"family": "Amsterdam Sans ExtraBold, Corbel", "size": 15},
+    "font_bold_corbel": {"family": "Corbel Bold", "size": 15},
+    "font": {"family": "Amsterdam Sans, Corbel", "size": 15},
+    "font_corbel": {"family": "Corbel", "size": 15},
+    "axis_font": {"family": "Amsterdam Sans ExtraBold, Corbel", "size": 15},
+    "plot_bgcolor": "#FFFFFF",
+    "gridline_color": "#dbdbdb",
+    "gridline_width": 0.75,
+    "showline": True,
+    "showgrid": False,
+    "zerolinecolor": "#dbdbdb",
+    "gridcolor": "#dbdbdb",
+    "gridlinecolor": "#dbdbdb",
+    "separators": ",",
+}

toolsos-0.2.6/src/toolsos/huisstijl/graphs/linegraph.py

@@ -0,0 +1,35 @@
+import plotly.express as px
+
+from .styler import BaseStyle
+
+basestyle = BaseStyle()
+
+
+def line(
+    data,
+    x,
+    y,
+    color: None,
+    width=750,
+    height=490,
+    color_discrete_sequence=None,
+    font="Amsterdam Sans",
+    **kwargs,
+):
+    fig = px.line(
+        data_frame=data,
+        x=x,
+        y=y,
+        color=color,
+        width=width,
+        height=height,
+        color_discrete_sequence=color_discrete_sequence,
+        template=BaseStyle().get_base_template(graph_type="line", font=font),
+        **kwargs,
+    )
+
+    fig.update_layout(
+        dict(xaxis_title_text="", yaxis_title_text="", legend_title_text="")
+    )
+
+    return fig

{toolsos-0.1.4 → toolsos-0.2.6}/src/toolsos/huisstijl/graphs/piegraph.py

@@ -1,4 +1,5 @@
 import plotly.express as px
+
 from .styler import BaseStyle
 
 basestyle = BaseStyle()
@@ -12,6 +13,8 @@ def pie(
     width=750,
     height=490,
     text_format: str = None,
+    color_discrete_sequence=None,
+    font="Amsterdam Sans",
     **kwargs,
 ):
     fig = px.pie(
@@ -21,7 +24,8 @@ def pie(
         width=width,
         height=height,
         hole=hole,
-        template=BaseStyle().get_base_template(),
+        template=BaseStyle().get_base_template(font=font),
+        color_discrete_sequence=color_discrete_sequence,
         **kwargs,
     )
 

toolsos-0.2.6/src/toolsos/huisstijl/graphs/styler.py

@@ -0,0 +1,189 @@
+import json
+
+import plotly.graph_objects as go
+import requests
+from requests import ConnectionError
+
+from .graph_styles import STYLE_NEW
+
+# class BaseStyle:
+#     style_url = (
+#         "https://raw.githubusercontent.com/jbosga-ams/oistyle/main/base_style.json"
+#     )
+
+#     def __init__(self):
+#         self.grab_styling()
+
+#     def grab_styling(self, style_path: str = None):
+#         if not style_path:
+#             try:
+#                 res = requests.get(self.style_url).json()
+#             except ConnectionError:
+#                 print("Failed grabbing basestyle from the interwebs")
+#                 # Add option to manually provide json file
+#         else:
+#             res = json.loads()
+
+#         for k, v in res.items():
+#             setattr(self, k, v)
+
+#         self.font = "Amsterdam Sans ExtraBold, Corbel"
+
+#     def _get_axis_format(self):
+#         self.gridline_color = "#dbdbdb"  # Jorren vragen om deze aan te passen
+
+#         return {
+#             "zerolinecolor": self.gridline_color,
+#             "gridcolor": self.gridline_color,
+#             "gridwidth": self.gridline_width,
+#             "showline": True,
+#             "linewidth": self.gridline_width,
+#             "linecolor": self.gridline_color,
+#             #             "mirror": True,
+#             "showgrid": False,
+#         }
+
+#     def _get_base_template_layout(self):
+#         return go.layout.Template(
+#             layout={
+#                 "font": {"family": self.font, "size": self.font_size},
+#                 "plot_bgcolor": self.plot_bgcolor,
+#                 "colorway": self.colors["darkblue_lightblue_gradient_5"],
+#                 "separators": ",",  # Jorren vragen om deze toe te voegen
+#             }
+#         )
+
+#     def get_base_template(
+#         self, graph_type: str = None, orientation: str = None, colors: str = None
+#     ):
+#         """[summary]
+
+#         Args:
+#             graph_type (str, optional): Pick 'bar', 'line' or 'bar'. Defaults to None.
+#             orientation (str, optional): [description]. Pick horizontal ('h') or vertical 'v'. Defaults to None.
+#             colors (str, optional): Set basecolors. Defaults to None.
+
+#         Raises:
+#             ValueError: [description]
+
+#         Returns:
+#             [type]: [description]
+#         """
+#         base_template = self._get_base_template_layout()
+#         axis_format = self._get_axis_format()
+
+#         if graph_type == "bar":
+#             if orientation in ["v", "vertical"]:
+#                 base_template.layout.xaxis.update(axis_format)
+#                 base_template.layout.yaxis.update(zeroline=False)
+#             elif orientation in ["h", "horizontal"]:
+#                 base_template.layout.yaxis.update(axis_format)
+#                 base_template.layout.xaxis.update(zeroline=False)
+#             else:
+#                 raise ValueError(
+#                     "Orientation ('v'/'vertical' or 'h'/'horizontal') should be supplied with graph_type=='bar'"
+#                 )
+
+#         elif graph_type == "line":
+#             base_template.layout.xaxis.update(axis_format)
+
+#         if colors:
+#             base_template.layout.update({"colorway": colors})
+
+#         return base_template
+
+#     def get_ois_colors(self, colorscale):
+#         colorscale = self.colors.get(colorscale, [])
+#         if not colorscale:
+#             raise Exception(f"Kies uit {self.colors.keys()}")
+
+#         return colorscale
+
+
+class BaseStyle:
+    def __init__(self, style_path=None):
+        if style_path is None:
+            self.style = STYLE_NEW
+        else:
+            with open(style_path) as file:
+                self.style = json.load(file)
+
+    def _get_axis_format(self):
+
+        return {
+            "zerolinecolor": self.style["gridline_color"],
+            "gridcolor": self.style["gridline_color"],
+            "gridwidth": self.style["gridline_width"],
+            "showline": True,
+            "linewidth": self.style["gridline_width"],
+            "linecolor": self.style["gridline_color"],
+            "showgrid": self.style["showgrid"],
+        }
+
+    def _get_base_template_layout(self, font):
+        if font == "Amsterdam Sans":
+            font_ = self.style["font"]
+            font_bold_ = self.style["font_bold"]
+        elif font == "Corbel":
+            font_ = self.style["font_corbel"]
+            font_bold_ = self.style["font_bold_corbel"]
+        else:
+            raise ValueError("Font should be 'Amsterdam Sans' or 'Corbel'")
+
+        return go.layout.Template(
+            layout={
+                "xaxis": {
+                    "tickfont": font_bold_,
+                },
+                "yaxis": {
+                    "tickfont": font_bold_,
+                },
+                "legend": {"font": font_},
+                "plot_bgcolor": self.style["plot_bgcolor"],
+                "separators": ",",
+                "font": font_bold_,
+            }
+        )
+
+    def get_base_template(
+        self,
+        graph_type: str = None,
+        orientation: str = None,
+        colors: str = None,
+        font: str = "Amsterdam Sans",
+    ):
+        """[summary]
+
+        Args:
+            graph_type (str, optional): Pick 'bar', 'line' or 'bar'. Defaults to None.
+            orientation (str, optional): [description]. Pick horizontal ('h') or vertical 'v'. Defaults to None.
+            colors (str, optional): Set basecolors. Defaults to None.
+
+        Raises:
+            ValueError: [description]
+
+        Returns:
+            [type]: [description]
+        """
+        base_template = self._get_base_template_layout(font)
+        axis_format = self._get_axis_format()
+
+        if graph_type == "bar":
+            if orientation in ["v", "vertical"]:
+                base_template.layout.xaxis.update(axis_format)
+                base_template.layout.yaxis.update(zeroline=False)
+            elif orientation in ["h", "horizontal"]:
+                base_template.layout.yaxis.update(axis_format)
+                base_template.layout.xaxis.update(zeroline=False)
+            else:
+                raise ValueError(
+                    "Orientation ('v'/'vertical' or 'h'/'horizontal') should be supplied with graph_type=='bar'"
+                )
+
+        elif graph_type == "line":
+            base_template.layout.xaxis.update(axis_format)
+
+        if colors:
+            base_template.layout.update({"colorway": colors})
+
+        return base_template