toolsos 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

toolsos/cbs_tools.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import json
 import pickle
+from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Iterator, Optional, Any
+from typing import TYPE_CHECKING, Any, Iterator, Optional
 
 import pandas as pd
 import pyarrow as pa
@@ -14,18 +15,27 @@ if TYPE_CHECKING:
     import pyreadstat
 
 
+def get_batch_size(path, memory_limit):
+    df, _ = prs.read_sav(path, row_limit=1000)
+
+    # memory in megabytes
+    mem_size = df.memory_usage().sum() / 1_000_000
+
+    # The amount of blocks (of a thousand rows fit in the memory_limit)
+    n_blocks = memory_limit / mem_size
+
+    # Calculate the number of rows that fit within the memory limit
+    return round(n_blocks * 1000)
+
+
 class SavToParquet:
     def __init__(
-        self,
-        file: str,
-        folder_out: str,
-        chunksize: Optional[int] = None,
-        verbose: bool = False,
+        self, file: str, folder_out: str, verbose: bool = False, memory_limit=10_000
     ) -> None:
         self.file = file
         self.folder_out = folder_out
         self.verbose = verbose
-        self.chunksize = 5_000_000 if not chunksize else chunksize
+        self.memory_limit = memory_limit
 
     @property
     def path_out(self) -> str:
@@ -33,20 +43,27 @@ class SavToParquet:
 
     @property
     def chunks(self) -> Iterator[tuple["pyreadstat.metadata_container", pd.DataFrame]]:
-        return prs.read_file_in_chunks(
-            prs.read_sav, self.file, chunksize=self.chunksize
-        )
 
-    def get_meta(self) -> Iterator:
-        return prs.read_sav(self.file, row_limit=10)
+        chunksize = get_batch_size(self.file, self.memory_limit)
+
+        if self.verbose:
+            print(f"Reading file in blocks of {chunksize} rows")
+            print("One such block should fit within the memory limit")
+
+        return prs.read_file_in_chunks(prs.read_sav, self.file, chunksize=chunksize)
 
     def write_meta_to_json(self) -> None:
         json_path = self.path_out.replace(".parquet", "_meta.json")
 
         meta_dict = {}
-        for attr in dir(self.meta):
-            if not attr.startswith("__"):
-                meta_dict[attr] = getattr(self.meta, attr)
+        for attr_name in dir(self.meta):
+            if not attr_name.startswith("__"):
+                attr = getattr(self.meta, attr_name)
+
+                if isinstance(attr, datetime):
+                    attr = attr.strftime("%Y-%m-%d %H:%M:%S")
+
+                meta_dict[attr_name] = attr
 
         with open(json_path, "w") as file:
             json.dump(meta_dict, file)
@@ -58,10 +75,12 @@ class SavToParquet:
             pickle.dump(self.meta, file)
 
     def write_to_parquet(self) -> None:
-        meta_df, self.meta = self.get_meta()
-        schema = table = pa.Table.from_pandas(meta_df).schema
 
         print("Writing table")
+
+        line1, self.meta = prs.read_sav(self.file, row_limit=1)
+        schema = pa.Table.from_pandas(line1).schema
+
         with pq.ParquetWriter(self.path_out, schema) as writer:
             for idx, (df, _) in enumerate(self.chunks):
                 if self.verbose:

toolsos/database/database_connection.py

@@ -153,23 +153,3 @@ def write_multiple_pgpass(conn_details, path: str | None = None):
 
     if os.name != "nt":
         path.chmod("0600")
-
-
-# Writing connection settings to pgpass.conf
-
-
-if __name__ == "__main__":
-    ...
-    # Examples
-
-    # Get database connection settings from yaml
-    engine_strings = get_db_connection_strings(
-        "src/toolsos/database/database_config.yml"
-    )
-    print(engine_strings.ruimte_analyse222)
-
-    # Get database connection settings from yaml and reset password
-    engine_strings = get_db_connection_strings(
-        "src/toolsos/database/database_config.yml", reset_pw=["ruimte_analyse222"]
-    )
-    print(engine_strings.ruimte_analyse222)

toolsos/geo.py

@@ -17,7 +17,7 @@ def get_geo_json(
     Returns:
         dict[str, str]: geo json containg of the desired level and year
     """
-    base_url = "https://gitlab.com/os-amsterdam/datavisualisatie-onderzoek-en-statistiek/-/raw/main/geo/"
+    base_url = "https://gitlab.com/os-amsterdam/datavisualisatie-onderzoek-en-statistiek/-/raw/main/public/geo/"
 
     if mra:
         level = f"{level}-mra"

toolsos/huisstijl/graphs/piegraph.py

@@ -24,9 +24,8 @@ def pie(
         width=width,
         height=height,
         hole=hole,
-        template=BaseStyle().get_base_template(),
+        template=BaseStyle().get_base_template(font=font),
         color_discrete_sequence=color_discrete_sequence,
-        font=font,
         **kwargs,
     )
 

toolsos/huisstijl/tables/tables.py

@@ -56,6 +56,8 @@ def cols_to_str(df: pd.DataFrame) -> pd.DataFrame:
     Returns:
         pd.DataFrame: Dataframe with column names as strings
     """
+
+    # Multiindex columns are always strings and therefore can't be casted as string
     if df.columns.nlevels == 1:
         df.columns = df.columns.astype(str)
 

{toolsos-0.2.4.dist-info → toolsos-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: toolsos
-Version: 0.2.4
+Version: 0.2.5
 Summary: OS tools
 Author-email: OS <d.schmitz@amsterdam.nl>
 Keywords: tools,Onderzoek & Statistiek
@@ -10,22 +10,22 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Provides-Extra: all
-Requires-Dist: keyring ; extra == 'all'
-Requires-Dist: openpyxl ; extra == 'all'
-Requires-Dist: pandas ; extra == 'all'
-Requires-Dist: plotly ; extra == 'all'
-Requires-Dist: polars ; extra == 'all'
-Requires-Dist: pyarrow ; extra == 'all'
-Requires-Dist: pyreadstat ; extra == 'all'
-Requires-Dist: pyyaml ; extra == 'all'
-Requires-Dist: requests ; extra == 'all'
-Requires-Dist: sqlalchemy ; extra == 'all'
+Requires-Dist: keyring; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: plotly; extra == "all"
+Requires-Dist: polars; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: pyreadstat; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: requests; extra == "all"
+Requires-Dist: sqlalchemy; extra == "all"
 Provides-Extra: dev
-Requires-Dist: black ; extra == 'dev'
-Requires-Dist: bumpver ; extra == 'dev'
-Requires-Dist: isort ; extra == 'dev'
-Requires-Dist: pip-tools ; extra == 'dev'
-Requires-Dist: pytest ; extra == 'dev'
+Requires-Dist: black; extra == "dev"
+Requires-Dist: bumpver; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pip-tools; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 
 # Tools Onderzoek & Statistiek
 
@@ -63,5 +63,12 @@ Instructions on building a package can be found [here](https://packaging.python.
 
 - make a pypi account
 - ask to be added as collaborator to toolsos
-- first update twine: py -m pip install --upgrade twin
+- first update twine: py -m pip install --upgrade twine
 - upload to pypi: twine upload dist/* --skip-existing
+
+## Install to local enviroment for testing
+
+- python -m venv local (maak een lokale venv aan)
+- local\Scripts\activate (activeer de venv)
+- pip install -e . (installer toolsos)
+- pip install -r local_requirements.txt (installeer de benodigde dependencies)

{toolsos-0.2.4.dist-info → toolsos-0.2.5.dist-info}/RECORD

@@ -1,11 +1,11 @@
 toolsos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-toolsos/cbs_tools.py,sha256=VLhptzy7m5EaET07s6VeAfDz79g1Hs38qeWPyA14wkw,2823
+toolsos/cbs_tools.py,sha256=361cogk0aIU4D4BKHaa7YSOBh64t5C3zrHlqtWx0iIc,3465
 toolsos/create_tables.py,sha256=43FHK3EERjumBtnGhngIdtthZzcc_Qi37lJ1MgATzBg,908
 toolsos/download.py,sha256=88hehmPL5m5d1nrcJjltuh4xrCItF5EYHaZdHOcSt-g,2652
-toolsos/geo.py,sha256=_OexkeUgXcnPW1mw27VN6fMcX2PMUSljLwIg48Xkv3M,2412
+toolsos/geo.py,sha256=arapy_ol6_so8KZ5gJk9ywXysSz4W8ah-cjrJ3DuxAo,2419
 toolsos/helpers.py,sha256=VeOl-fLgePCbjEmAQdVmYe7z8OE1pISeDDuP1t5QSxM,997
 toolsos/polars_helpers.py,sha256=P3RHLQFeDL7-9U_Q1n4ma_NSkdYAiker4pnc57uluHw,770
-toolsos/database/database_connection.py,sha256=NTgwXLJ7LFrTlMr74W9Yge8hLyQG0DCW4t5UuwtoP18,4920
+toolsos/database/database_connection.py,sha256=_CW84UMElCI4ix0LqDWRCL6igMjuJilJYXxxWdMUcbA,4352
 toolsos/database/database_transfer.py,sha256=1ghq5VEtKyOdCKdM45uOyrZSoXMuWsdC35R3WNuFvdU,1827
 toolsos/huisstijl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toolsos/huisstijl/colors.py,sha256=lSCHCdSjge5cGfLfAObd6mV6TaXq3QGImLOmoGJpGkw,1484
@@ -13,13 +13,13 @@ toolsos/huisstijl/graphs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 toolsos/huisstijl/graphs/bargraph.py,sha256=HYl01_euh23iDYSUhnAzYAXS0DhDpg9eLRjJEpeR6iU,2815
 toolsos/huisstijl/graphs/graph_styles.py,sha256=Z9LLH7j8ODTsYMYK0rslacphuiRDcq5_IpSjEEiK2VY,975
 toolsos/huisstijl/graphs/linegraph.py,sha256=dMUarRe31SXaY78OCXLy-PgnU8LlVJ9KkzKaHhDtuuI,698
-toolsos/huisstijl/graphs/piegraph.py,sha256=tHWQlM5BP03Pq044-pONoyYlDRNyG-hC1itwsoZ7DDA,714
+toolsos/huisstijl/graphs/piegraph.py,sha256=aEFiEM-9QuhBOjKHSXVuE5bTh-8uucq4FP6O8Vk1vZI,703
 toolsos/huisstijl/graphs/styler.py,sha256=-uZ7pjY1G39XvmaGHQd31gPRxjxmJGhYZk8xhy2JUWc,6623
 toolsos/huisstijl/tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toolsos/huisstijl/tables/table_helpers.py,sha256=jsQ6lw93sxtGJGrUn8X2_LyA2vYYnytngpUI5A_wpWQ,2037
 toolsos/huisstijl/tables/table_styles.py,sha256=oYU6GJcfqlKpZof5PUjPsA7woJ3Tew78CHPyT0_jY6w,1343
-toolsos/huisstijl/tables/tables.py,sha256=67vYOe4DAOyZDPCdc4BOEcPJ80IJLCGilHyQNSMrIvo,23967
-toolsos-0.2.4.dist-info/METADATA,sha256=TX31wvxxA7reLc6bKCtkzAuq20DELKfSkDQfvFSJAO8,2433
-toolsos-0.2.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-toolsos-0.2.4.dist-info/top_level.txt,sha256=2ClEjUBbtfDQ8oPwvWRy1Sz2nrkLCXlg0mHaMdCWia0,8
-toolsos-0.2.4.dist-info/RECORD,,
+toolsos/huisstijl/tables/tables.py,sha256=2FO-ByLjgs-DbNgem3cDfYJbLbIDzRDqXtjL75WN7kY,24054
+toolsos-0.2.5.dist-info/METADATA,sha256=rTUT5FhyCYenHMdaFTeU6v5LQymQDfMjdJOM1lTsTdM,2683
+toolsos-0.2.5.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+toolsos-0.2.5.dist-info/top_level.txt,sha256=2ClEjUBbtfDQ8oPwvWRy1Sz2nrkLCXlg0mHaMdCWia0,8
+toolsos-0.2.5.dist-info/RECORD,,

{toolsos-0.2.4.dist-info → toolsos-0.2.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any