toolsos 0.1__tar.gz

toolsos-0.1/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: toolsos
+Version: 0.1
+Summary: OS tools
+Author-email: OS <d.schmitz@amsterdam.nl>
+Keywords: feed,reader,tutorial
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: bumpver; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pip-tools; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Provides-Extra: all
+Requires-Dist: keyring; extra == "all"
+Requires-Dist: plotly; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: sqlalchemy; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: requests; extra == "all"
+
+# Tools Onderzoek & Statistiek
+
+This repository contains the tools used by the data scientist/researchers working at Onderzoek & Statistiek
+
+## Installation instructions
+
+The package can be installed using:
+    - pip
+      - Use pip install toolsos[all]
+    - conda.
+      - Use pip install toolsos. The user has to download the dependencies themselves
+
+## Building the package
+
+Instructions on building a package can be found [here](https://packaging.python.org/en/latest/tutorials/packaging-projects/)
+
+- py -m pip install --upgrade build
+- py -m build
+
+## Uploading the package to PyPi

toolsos-0.1/README.md

@@ -0,0 +1,20 @@
+# Tools Onderzoek & Statistiek
+
+This repository contains the tools used by the data scientist/researchers working at Onderzoek & Statistiek
+
+## Installation instructions
+
+The package can be installed using:
+    - pip
+      - Use pip install toolsos[all]
+    - conda.
+      - Use pip install toolsos. The user has to download the dependencies themselves
+
+## Building the package
+
+Instructions on building a package can be found [here](https://packaging.python.org/en/latest/tutorials/packaging-projects/)
+
+- py -m pip install --upgrade build
+- py -m build
+
+## Uploading the package to PyPi

toolsos-0.1/pyproject.toml

@@ -0,0 +1,37 @@
+# pyproject.toml
+
+[build-system]
+requires      = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "toolsos"
+version = "0.1"
+description = "OS tools"
+readme = "README.md"
+authors = [{ name = "OS", email = "d.schmitz@amsterdam.nl" }]
+# license = { file = "LICENSE" }
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+]
+keywords = ["feed", "reader", "tutorial"]
+# dependencies are kept empty until to be able to install in conda enviroment
+# use pip install toolsos[all] to pip install with al dependencies
+dependencies = []
+requires-python = ">=3.11"
+
+[project.optional-dependencies]
+dev = ["black", "bumpver", "isort", "pip-tools", "pytest"]
+all = [
+    "keyring",
+    "plotly",
+    "openpyxl", 
+    "sqlalchemy",
+    "pyyaml",
+    "requests"
+    ]
+
+#[project.urls]
+#Homepage = "https://github.com/realpython/reader"

toolsos-0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

toolsos-0.1/src/toolsos/cbs_tools.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+import pickle
+from pathlib import Path
+from typing import TYPE_CHECKING, Iterator, Optional, Any
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+from pyreadstat import pyreadstat as prs
+
+if TYPE_CHECKING:
+    import pyreadstat
+
+
+class SavToParquet:
+    def __init__(
+        self,
+        file: str,
+        folder_out: str,
+        chunksize: Optional[int] = None,
+        verbose: bool = False,
+    ) -> None:
+        self.file = file
+        self.folder_out = folder_out
+        self.verbose = verbose
+        self.chunksize = 5_000_000 if not chunksize else chunksize
+
+    @property
+    def path_out(self) -> str:
+        return str(Path(self.file)).replace(".sav", ".parquet")
+
+    @property
+    def chunks(self) -> Iterator[tuple["pyreadstat.metadata_container", pd.DataFrame]]:
+        return prs.read_file_in_chunks(
+            prs.read_sav, self.file, chunksize=self.chunksize
+        )
+
+    def get_meta(self) -> Iterator:
+        return prs.read_sav(self.file, row_limit=10)
+
+    def write_meta_to_json(self) -> None:
+        json_path = self.path_out.replace(".parquet", "_meta.json")
+
+        meta_dict = {}
+        for attr in dir(self.meta):
+            if not attr.startswith("__"):
+                meta_dict[attr] = getattr(self.meta, attr)
+
+        with open(json_path, "w") as file:
+            json.dump(meta_dict, file)
+
+    def write_meta_to_pickle(self) -> None:
+        pickle_path = self.path_out.replace(".parquet", "_meta.pickle")
+
+        with open(pickle_path, "wb") as file:
+            pickle.dump(self.meta, file)
+
+    def write_to_parquet(self) -> None:
+        meta_df, self.meta = self.get_meta()
+        schema = table = pa.Table.from_pandas(meta_df).schema
+
+        print("Writing table")
+        with pq.ParquetWriter(self.path_out, schema) as writer:
+            for idx, (df, _) in enumerate(self.chunks):
+                if self.verbose:
+                    print(f"Writing chunk: {idx: >4}")
+
+                table = pa.Table.from_pandas(df)
+                writer.write_table(table)
+
+        print("Writing metadata")
+        self.write_meta_to_json()
+        self.write_meta_to_pickle()
+        print("Done")
+
+
+def read_parquet_in_chunks(
+    path: str, columns: Optional[list[str]] = None
+) -> Iterator[pd.DataFrame]:
+    parquet_file = pq.ParquetFile(path)
+    for table in parquet_file.iter_batches(columns=columns):
+        df = table.to_pandas()
+        yield df
+
+
+def read_metadata_container(path: str) -> dict[str, Any]:
+    with open(path, "rb") as file:
+        return pickle.load(file)
+
+
+def read_meta_from_json(path: str) -> dict[str, Any]:
+    with open(path) as file:
+        return json.load(file)

toolsos-0.1/src/toolsos/database_connection.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import getpass
+import json
+import subprocess
+from json import JSONDecodeError
+from typing import Optional
+
+import keyring
+import yaml
+
+
+def get_db_connection_strings(
+    path: str, reset_pw: Optional[list[str]] = None
+) -> DbStringCollection:
+    """Creates object containing all database connection strings based on yaml
+    file containg the database connection settings. Password for the specific
+    database will be prompted and stored in the keyring of the device
+
+    Args:
+        path (str): _description_
+        flush_pw (Optional[list[str]], optional): List with passwords to be reset. Use the
+            name of the database connection in the config file. Defaults to None.
+
+    Returns:
+        DbStringCollection: Simple class with an attribute for each connection string
+    """
+    with open(path) as f:
+        db_info = yaml.safe_load(f)
+
+    dsc = DbStringCollection()
+
+    for dbname, params in db_info.items():
+        flush = dbname in reset_pw if reset_pw else False
+
+        if params["pw"] == "acces_token":
+            pw = get_azure_access_token()
+        else:
+            pw = get_pw_from_keyring(dbname=dbname, user=params["user"], reset_pw=flush)
+
+        engine = build_conn_string(pw=pw, **params)
+        dsc.add_conn_string(dbname, engine)
+
+    return dsc
+
+
+def build_conn_string(user: str, pw: str, host: str, port: str, dbname: str) -> str:
+    """Builds the connection string for the database
+
+    @@TODO
+    Add possibility to use different database types
+
+    Args:
+        user (str): username
+        pw (str): database password
+        host (str): database host
+        port (str): database port
+        dbname (str): database name
+
+    Returns:
+        str: engine string
+    """
+    return f"postgresql://{user}:{pw}@{host}:{port}/{dbname}"
+
+
+def get_pw_from_keyring(dbname: str, user: str, reset_pw: Optional[bool] = None) -> str:
+    """_summary_
+
+    Args:
+        db_name (str): database name
+        user (str): username
+
+    Returns:
+        str: password
+    """
+    pw = keyring.get_password(dbname, user)
+
+    if not pw or reset_pw:
+        pw = getpass.getpass(f"Input password for {dbname}: ")
+        keyring.set_password(dbname, user, pw)
+
+    return pw
+
+
+class DbStringCollection:
+    """_summary_"""
+
+    def add_conn_string(self, db_name: str, connection_str: str) -> None:
+        setattr(self, db_name, connection_str)
+
+
+def get_azure_access_token():
+    command = "az account get-access-token --resource-type oss-rdbms"
+    result = subprocess.run(command, capture_output=True, shell=True, text=True)
+
+    try:
+        json.loads(result.stdout)["accessToken"]
+    except JSONDecodeError:
+        subprocess.run("az login", shell=True)
+
+
+if __name__ == "__main__":
+    ...
+    # Examples
+
+    # Get database connection settings from yaml
+    # engine_strings = get_db_connection_strings("python/database_config.yml")
+    # print(engine_strings.ruimte_analyse222)
+
+    # Get database connection settings from yaml and reset password
+    # engine_strings = get_db_connection_strings(
+    #     "python/database_config.yml", reset_pw=["ruimte_analyse222"]
+    # )
+    # print(engine_strings.ruimte_analyse222)

toolsos-0.1/src/toolsos/database_transfer.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from sqlalchemy import MetaData, create_engine
+from sqlalchemy.exc import ProgrammingError
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+
+
+def query_as_dict(rs):
+    result = []
+    for idx, row in enumerate(rs):
+        try:
+            result.append(row._as_dict())
+        except AttributeError:
+            print(idx)
+
+
+def table_from_db_to_db(
+    conn_string_db_from: str,
+    conn_string_db_to: str,
+    table: str,
+    schema_from: Optional[str] = None,
+    schema_to: Optional[str] = None,
+    rename_table: Optional[str] = None,
+    if_exist: Optional[str] = None,
+):
+    engine_from = create_engine(conn_string_db_from)
+    engine_to = create_engine(conn_string_db_to)
+
+    print("Reflecting table")
+    metadata_from = MetaData()
+    metadata_from.reflect(engine_from, schema=schema_from, only=[table])
+    Base = automap_base(metadata=metadata_from)
+    table_meta = metadata_from.tables[table]
+    Base_to = automap_base(metadata=Base.metadata)
+
+    print("Querying table")
+    with Session(engine_from) as s:
+        rs = s.query(table_meta).all()
+        rs = [row._asdict() for row in rs]
+
+    print("Setting schema")
+    if rename_table:
+        Base_to.metadata.tables[table].name = rename_table
+
+    Base_to.metadata.tables[table].schema = schema_to
+
+    if if_exist == "drop":
+        print("Dropping table")
+        try:
+            table_meta.drop(engine_to)
+        except ProgrammingError as pe:
+            print(f"Exception Caught: {pe}")
+
+    print("Creating table")
+    Base_to.metadata.create_all(engine_to)
+
+    print("Writing table")
+    with Session(engine_to) as s:
+        s.execute(table_meta.insert(), rs)
+        s.commit()

toolsos-0.1/src/toolsos/download.py

@@ -0,0 +1,98 @@
+import pathlib
+import shutil
+import zipfile
+
+import requests
+
+
+def download(url: str, dest_path: pathlib.Path):
+    r = requests.get(url, stream=True)
+    if not r.ok:
+        raise ValueError("Download failed: check repo and language")
+
+    with open(dest_path, "wb") as f:
+        for chunk in r.iter_content():
+            if chunk:
+                f.write(chunk)
+
+
+def unzip(zip_store: pathlib.Path, dest_folder: pathlib.Path):
+    with zipfile.ZipFile(zip_store) as zipfile_:
+        for filename in zipfile_.namelist():
+            zipfile_.extract(filename, path=dest_folder)
+
+
+class FileLocation:
+    GIT_PROVIDER = "https://gitlab.com/os-amsterdam"
+
+    def __init__(
+        self,
+        dest_folder: str,
+        dest_folder_name: str,
+        repo: str,
+        branch: str,
+        subfolder: str,
+    ):
+        self.dest_folder = pathlib.Path(dest_folder)
+        self.dest_folder_name = dest_folder_name
+        self.repo = repo
+        self.branch = branch
+        self.subfolder = subfolder
+
+    @property
+    def url(self):
+        return f"{self.GIT_PROVIDER}/{self.repo}/-/archive/main/{self.repo}-{self.branch}.zip"
+
+    @property
+    def zipfile(self):
+        return self.dest_folder / "_temp.zip"
+
+    @property
+    def move_folder(self):
+        return self.dest_folder / f"{self.repo}-{self.branch}" / self.subfolder
+
+    @property
+    def unzipped_folder(self):
+        return self.dest_folder / f"{self.repo}-{self.branch}"
+
+    @property
+    def os_tools_folder(self):
+        return self.dest_folder / self.dest_folder_name
+
+
+def copy_repo(
+    repo: str,
+    dest_folder: str,
+    dest_folder_name,
+    branch,
+    subfolder,
+):
+    fl = FileLocation(
+        dest_folder=dest_folder,
+        dest_folder_name=dest_folder_name,
+        repo=repo,
+        branch=branch,
+        subfolder=subfolder,
+    )
+    download(url=fl.url, dest_path=fl.zipfile)
+    unzip(zip_store=fl.zipfile, dest_folder=fl.dest_folder)
+
+    if fl.os_tools_folder.exists():
+        shutil.rmtree(fl.os_tools_folder)
+    shutil.move(fl.move_folder, fl.os_tools_folder)
+
+    # Remove downloaded zip file and unzipped folder
+    fl.zipfile.unlink()
+    shutil.rmtree(fl.unzipped_folder)
+
+
+def copy_os_tools(dest_folder: str, branch="main", subfolder="python"):
+    REPO = "tools-onderzoek-en-statistiek"
+    DEST_FOLDER_NAME = "ostools"
+    if not branch:
+        branch = "main"
+    copy_repo(REPO, dest_folder, DEST_FOLDER_NAME, branch, subfolder)
+
+
+if __name__ == "__main__":
+    copy_os_tools("C:/python_projects/_uitproberen/test_project/src")

toolsos-0.1/src/toolsos/geo.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any, Union
+
+import requests
+
+
+def get_geo_json(
+    level: str, year: Union[int, Any], with_water: bool = False, mra: bool = False
+) -> dict[str, str]:
+    """_summary_
+
+    Args:
+        level (str): 'stadsdelen'/'gebieden'/'wijken'/'buurten'
+        year (int): jaar
+
+    Returns:
+        dict[str, str]: geo json containg of the desired level and year
+    """
+    base_url = "https://gitlab.com/os-amsterdam/datavisualisatie-onderzoek-en-statistiek/-/raw/main/geo/"
+
+    if mra:
+        level = f"{level}-mra"
+        base_url = f"{base_url}mra/"
+    else:
+        base_url = f"{base_url}amsterdam/"
+
+    if (year <= 2020) & ~mra:
+        year = "2015-2020"
+
+    if with_water:
+        url = f"{base_url}/{year}/{level}-{year}-geo.json"
+    else:
+        url = f"{base_url}/{year}/{level}-{year}-zw-geo.json"
+
+    print(url)
+    json = requests.get(url).json()
+    return json
+
+
+def extract_name_code_table(geo_json: dict[str, str]) -> dict[str, str]:
+    """_summary_
+
+    Args:
+        geo_json (dict[str, str]): geo_json of a specific level and year
+
+    Returns:
+        dict[str, str]: dictionary containing the mapping 'naam': 'year'
+    """
+    naam_code = {}
+    f: Any  # Add explicit type hint for complex dict structure
+    for f in geo_json["features"]:
+        properties = f.get("properties")
+        naam_code[properties["naam"]] = properties["code"]
+    return naam_code
+
+
+def get_geo_name_code(level: str, year: int, mra: bool = False) -> dict[str, str]:
+    """_summary_
+
+    Args:
+        level (str): 'stadsdelen'/'gebieden'/'wijken'/'buurten'
+        year (int): jaar
+
+    Returns:
+        dict[str, str]: _description_
+    """
+    json = get_geo_json(level=level, year=year)
+    name_code = extract_name_code_table(json)
+    return name_code
+
+
+if __name__ == "__main__":
+    ...
+    # print(get_geo_json("buurten", 2021, mra=False))
+    # print(get_geo_json("buurten", 2018, mra=False))
+
+    # print(get_geo_json("buurten", 2021, mra=True))
+    # print(get_geo_json("buurten", 2018, mra=True))
+
+    print(get_geo_name_code("wijken", 2020, mra=False))
+    print(get_geo_name_code("wijken", 2020, mra=True))
+
+
+# https://gitlab.com/os-amsterdam/datavisualisatie-onderzoek-en-statistiek/-/raw/main/geo/mra//2015-2020/buurten-mra-2015-2020-zw-geo.json

toolsos-0.1/src/toolsos/helpers.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import functools
+import time
+from typing import Optional, Union
+
+import pandas as pd
+
+
+def time_it(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        value = func(*args, **kwargs)
+        print(time.perf_counter() - start)
+        return value
+
+    return wrapper
+
+
+def os_cut(
+    x: Union[list[Union[int, float]], pd.Series],
+    bins: list,
+    start_label: str = "lager dan",
+    end_label: str = "en hoger",
+    add_edge: Optional[int] = None,
+    sep: str = " - ",
+) -> pd.Series:
+    # Add non_overlap to left edge/boundary
+    if not add_edge:
+        add_edge = 0
+
+    start_l = [f"{start_label} {bins[1]}"]
+    inbetween_labels = [
+        f"{bins[i] + add_edge}{sep}{bins[i+1]}" for i in range(1, len(bins) - 2)
+    ]
+    end_l = [f"{bins[-2]} {end_label}"]
+
+    return pd.cut(x, bins=bins, labels=start_l + inbetween_labels + end_l)  # type: ignore

toolsos-0.1/src/toolsos/huisstijl/colors.py

@@ -0,0 +1,48 @@
+import requests
+
+
+def get_os_colors(
+    type: str, kleur: str, aantal: str | int, invert: bool = False
+) -> list[str]:
+    """_summary_
+
+    Args:
+        type (str): type of (oplopend, uiteenlopend, discreet)
+        kleur (str):
+            oplopend:
+                'blauw' |
+                'paars' |
+                'groen' |
+                'roze' |
+                'lichtblauw' |
+                'oranje' |
+                'lichtgroen' |
+                'grijs'
+            uiteenlopend:
+                'stoplicht (1-7)' |
+                'blauw - grijs - groen (1-9)' |
+                'paars - grijs - lichtblauw (1-9)' |
+                'blauw - geel - groen (1-9)' |
+                'rood - geel - lichtblauw (1-9)'
+            discreet:
+                'discreet (1-9)' |
+                'fruitig (1-9)' |
+                'fruitig (1-9, anders gesorteerd)' |
+                'waterkant (1-9)' |
+                'waterkant (1-9, anders gesorteerd)' |
+                'zonsondergang (1-9)'
+        aantal (str): number of colors returned
+        invert (bool, optional): invert colors. Defaults to False.
+
+    Returns:
+        list[str]: list with colors
+    """
+    url = "https://gitlab.com/os-amsterdam/tools-onderzoek-en-statistiek/-/raw/main/references/OS_colors.json"
+    colors = requests.get(url).json()
+
+    colors = colors[type][kleur][str(aantal)]
+
+    if invert:
+        colors = colors[::-1]
+
+    return colors