beanhub-cli 1.4.2__tar.gz → 1.5.0b2__tar.gz

{beanhub_cli-1.4.2 → beanhub_cli-1.5.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: beanhub-cli
-Version: 1.4.2
+Version: 1.5.0b2
 Summary: Command line tools for BeanHub
 License: MIT
 Author: Fang-Pen Lin
@@ -11,19 +11,26 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: connect
+Provides-Extra: login
 Requires-Dist: beancount-black (>=1.0.1,<2.0.0)
 Requires-Dist: beancount-parser (>=1.2.3,<1.3.0)
 Requires-Dist: beanhub-extract (>=0.1.0,<1.0.0)
 Requires-Dist: beanhub-forms (>=0.1.3,<0.2.0)
 Requires-Dist: beanhub-import (>=1.0.0,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: cryptography (>=44.0.0,<45.0.0) ; extra == "connect"
 Requires-Dist: fastapi (>=0.98.0,<1.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: pydantic (>=2.4.0,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.0.3,<3.0.0)
+Requires-Dist: pynacl (>=1.5.0,<2.0.0) ; extra == "connect"
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0) ; extra == "login" or extra == "connect"
 Requires-Dist: rich (>=13.6.0,<14.0.0)
 Requires-Dist: starlette-wtf (>=0.4.3,<0.5.0)
+Requires-Dist: tomli (>=2.2.1,<3.0.0) ; extra == "login" or extra == "connect"
+Requires-Dist: tomli-w (>=1.1.0,<2.0.0) ; extra == "login" or extra == "connect"
 Requires-Dist: uvicorn (>=0.23.2,<1.0.0)
 Project-URL: Documentation, https://beanhub-cli-docs.beanhub.io
 Description-Content-Type: text/markdown

beanhub_cli-1.5.0b2/beanhub_cli/config.py

@@ -0,0 +1,49 @@
+import pathlib
+
+from pydantic import BaseModel
+
+
+class AccessToken(BaseModel):
+    token: str
+
+
+class Repository(BaseModel):
+    default: str | None = None
+
+
+class Config(BaseModel):
+    access_token: AccessToken | None = None
+    repo: Repository | None = None
+
+
+def get_config_path() -> pathlib.Path:
+    return pathlib.Path.home() / ".beanhub" / "config.toml"
+
+
+def load_config(file_path: pathlib.Path | None = None) -> Config | None:
+    import tomli
+
+    if file_path is None:
+        file_path = get_config_path()
+    if not file_path.exists():
+        return
+
+    with file_path.open("rb") as fo:
+        obj = tomli.load(fo)
+        return Config.model_validate(obj)
+
+
+def save_config(config: Config, file_path: pathlib.Path | None = None):
+    import tomli_w
+
+    if file_path is None:
+        file_path = get_config_path()
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    file_path.write_text("")
+    file_path.chmod(0o600)
+    # TODO: for now we save the config as plaintext. We should consider to use OS keychain service in the future
+    #       for better security
+    with file_path.open("wb") as fo:
+        obj = config.model_dump(mode="json")
+        tomli_w.dump(obj, fo)

beanhub_cli-1.5.0b2/beanhub_cli/connect/cli.py

@@ -0,0 +1,11 @@
+from ..aliase import AliasedGroup
+from ..cli import cli as root_cli
+
+
+@root_cli.group(
+    name="connect",
+    help="BeanHub Connect features, such as sync or dump (login required).",
+    cls=AliasedGroup,
+)
+def cli():
+    pass

beanhub_cli-1.5.0b2/beanhub_cli/connect/config.py

@@ -0,0 +1,30 @@
+import dataclasses
+import sys
+
+from ..config import load_config
+from ..environment import Environment
+
+
+@dataclasses.dataclass
+class ConnectConfig:
+    token: str
+    repo: str
+
+
+# TODO: maybe extract this part to a shared env for connect command?
+def ensure_config(env: Environment, repo: str | None) -> ConnectConfig:
+    config = load_config()
+    if config is None or config.access_token is None:
+        env.logger.error(
+            'You need to login into your BeanHub account with "bh login" command first'
+        )
+        sys.exit(-1)
+    if repo is None and (config.repo is None or config.repo.default is None):
+        env.logger.error(
+            'You need to provide a repo by -r argument, such as "myuser/myrepo" or define a default repo in your config file'
+        )
+        sys.exit(-1)
+    return ConnectConfig(
+        token=config.access_token.token,
+        repo=repo if repo is not None else config.repo.default,
+    )

beanhub_cli-1.5.0b2/beanhub_cli/connect/encryption.py

@@ -0,0 +1,25 @@
+import io
+
+
+def decrypt_file(
+    input_file: io.BytesIO, output_file: io.BytesIO, iv: bytes, key: bytes
+):
+    from cryptography.hazmat.primitives import padding
+    from cryptography.hazmat.primitives.ciphers import algorithms
+    from cryptography.hazmat.primitives.ciphers import Cipher
+    from cryptography.hazmat.primitives.ciphers import modes
+
+    cipher = Cipher(algorithms.AES256(key), modes.CBC(iv))
+    decryptor = cipher.decryptor()
+    padder = padding.PKCS7(128).unpadder()
+    while True:
+        chunk = input_file.read(4096)
+        if not chunk:
+            break
+        decrypted = decryptor.update(chunk)
+        unpadded_chunk = padder.update(decrypted)
+        output_file.write(unpadded_chunk)
+    output_file.write(padder.update(decryptor.finalize()))
+    output_file.write(padder.finalize())
+    output_file.flush()
+    output_file.seek(0)

beanhub_cli-1.5.0b2/beanhub_cli/connect/main.py

@@ -0,0 +1,301 @@
+import enum
+import json
+import sys
+import tarfile
+import tempfile
+import time
+import urllib.parse
+
+import click
+import rich
+from rich import box
+from rich.markup import escape
+from rich.padding import Padding
+from rich.table import Table
+
+from ..environment import Environment
+from ..environment import pass_env
+from ..utils import check_imports
+from .cli import cli
+from .config import ConnectConfig
+from .config import ensure_config
+from .encryption import decrypt_file
+
+TABLE_HEADER_STYLE = "yellow"
+TABLE_COLUMN_STYLE = "cyan"
+SPOOLED_FILE_MAX_SIZE = 1024 * 1024 * 5
+
+
+@enum.unique
+class PlaidItemSyncState(enum.Enum):
+    PENDING = "PENDING"
+    PROCESSING = "PROCESSING"
+    SYNC_COMPLETE = "SYNC_COMPLETE"
+    SYNC_FAILED = "SYNC_FAILED"
+    IMPORT_COMPLETE = "IMPORT_COMPLETE"
+    IMPORT_COMPLETE_NO_CHANGES = "IMPORT_COMPLETE_NO_CHANGES"
+    IMPORT_FAILED = "IMPORT_FAILED"
+
+
+@enum.unique
+class DumpRequestState(enum.Enum):
+    PENDING = "PENDING"
+    PROCESSING = "PROCESSING"
+    COMPLETE = "COMPLETE"
+    FAILED = "FAILED"
+
+
+GOOD_TERMINAL_SYNC_STATES = frozenset(
+    [
+        PlaidItemSyncState.IMPORT_COMPLETE,
+        PlaidItemSyncState.IMPORT_COMPLETE_NO_CHANGES,
+    ]
+)
+BAD_TERMINAL_SYNC_STATES = frozenset(
+    [
+        PlaidItemSyncState.IMPORT_FAILED,
+        PlaidItemSyncState.SYNC_FAILED,
+    ]
+)
+
+
+def run_sync(env: Environment, config: ConnectConfig):
+    import requests
+
+    env.logger.info(
+        "Running sync batch for repo [green]%s[/]",
+        config.repo,
+        extra={"markup": True, "highlighter": None},
+    )
+    # TODO: generate API client from OpenAPI spec instead
+    url = urllib.parse.urljoin(
+        env.api_base_url, f"v1/repos/{config.repo}/connect/sync_batches"
+    )
+    resp = requests.post(url, headers={"access-token": config.token})
+    if resp.status_code == 422:
+        env.logger.error("Failed to sync with error: %s", resp.json())
+        sys.exit(-1)
+    elif resp.status_code == 401:
+        env.logger.error(
+            "Failed to sync permission error: %s, please ensure your Access Token has API_CONNECT_SYNC permission",
+            resp.json(),
+        )
+        sys.exit(-1)
+    resp.raise_for_status()
+
+    batch_id = resp.json()["id"]
+    env.logger.info(
+        "Created sync batch [green]%s[/], waiting for updates ...",
+        batch_id,
+        extra={"markup": True, "highlighter": None},
+    )
+
+    url = urllib.parse.urljoin(
+        env.api_base_url, f"v1/repos/{config.repo}/connect/sync_batches/{batch_id}"
+    )
+    while True:
+        time.sleep(5)
+        resp = requests.get(url, headers={"access-token": config.token})
+        # TODO: provide friendly error messages here
+        resp.raise_for_status()
+        payload = resp.json()
+        total = len(payload["syncs"])
+        good_terms = list(
+            sync
+            for sync in payload["syncs"]
+            if PlaidItemSyncState[sync["state"]] in GOOD_TERMINAL_SYNC_STATES
+        )
+        bad_terms = list(
+            sync
+            for sync in payload["syncs"]
+            if PlaidItemSyncState[sync["state"]] in BAD_TERMINAL_SYNC_STATES
+        )
+        progress = len(good_terms) + len(bad_terms)
+        if progress >= total:
+            break
+        else:
+            env.logger.info(
+                "Still processing, [green]%s[/] out of [green]%s[/]",
+                progress,
+                total,
+                extra={"markup": True, "highlighter": None},
+            )
+    table = Table(
+        title="Sync finished successfully",
+        box=box.SIMPLE,
+        header_style=TABLE_HEADER_STYLE,
+        expand=True,
+    )
+    table.add_column("Id", style=TABLE_COLUMN_STYLE)
+    table.add_column("Institution", style=TABLE_COLUMN_STYLE)
+    table.add_column("State", style=TABLE_COLUMN_STYLE)
+    for sync in good_terms:
+        table.add_row(
+            escape(sync["id"]),
+            escape(sync["item"]["institution_name"]),
+            escape(sync["state"]),
+        )
+    rich.print(Padding(table, (1, 0, 0, 4)))
+
+    table = Table(
+        title="Sync finished with error",
+        box=box.SIMPLE,
+        header_style=TABLE_HEADER_STYLE,
+        expand=True,
+    )
+    table.add_column("Id", style=TABLE_COLUMN_STYLE)
+    table.add_column("Institution", style=TABLE_COLUMN_STYLE)
+    table.add_column("State", style=TABLE_COLUMN_STYLE)
+    table.add_column("Error", style=TABLE_COLUMN_STYLE)
+    for sync in bad_terms:
+        table.add_row(
+            escape(sync["id"]),
+            escape(sync["item"]["institution_name"]),
+            escape(sync["state"]),
+            escape(sync["error_message"]),
+        )
+    rich.print(Padding(table, (1, 0, 0, 4)))
+
+
+@cli.command(help="Sync transactions for all BeanHub Connect banks")
+@click.option(
+    "-r",
+    "--repo",
+    type=str,
+    help='Which repository to run sync on, in "<username>/<repo_name>" format',
+)
+@pass_env
+def sync(env: Environment, repo: str | None):
+    check_imports(
+        logger=env.logger,
+        module_names=["requests", "tomli", "tomli_w"],
+        required_extras=["login"],
+    )
+    config = ensure_config(env, repo=repo)
+    run_sync(env, config)
+    env.logger.info("done")
+
+
+@cli.command(
+    help="Dump transaction CSV files from BeanHub Connect to your local environment"
+)
+@click.option(
+    "-r",
+    "--repo",
+    type=str,
+    help='Which repository to run sync on, in "<username>/<repo_name>" format',
+)
+@click.option(
+    "-s",
+    "--sync",
+    type=bool,
+    is_flag=True,
+    help="Run sync first before running dump",
+)
+@click.option(
+    "--unsafe-tar-extract",
+    type=bool,
+    is_flag=True,
+    help="Allow unsafe tar extraction, mostly for Python < 3.11",
+)
+@pass_env
+def dump(env: Environment, repo: str | None, sync: bool, unsafe_tar_extract: bool):
+    check_imports(
+        logger=env.logger,
+        module_names=["requests", "tomli", "tomli_w", "nacl", "cryptography"],
+        required_extras=["connect"],
+    )
+
+    import requests
+    from nacl.encoding import URLSafeBase64Encoder
+    from nacl.public import PrivateKey
+    from nacl.public import SealedBox
+
+    if not hasattr(tarfile, "data_filter") and not unsafe_tar_extract:
+        env.logger.error(
+            "You need to use Python >= 3.11 in order to safely unpack the downloaded tar file, or you need to pass "
+            "in --unsafe-tar-extract argument to allow unsafe tar file extracting"
+        )
+        sys.exit(-1)
+    config = ensure_config(env, repo=repo)
+    if sync:
+        run_sync(env, config)
+
+    private_key = PrivateKey.generate()
+    public_key = private_key.public_key.encode(URLSafeBase64Encoder).decode("ascii")
+
+    # TODO: generate API client from OpenAPI spec instead
+    url = urllib.parse.urljoin(
+        env.api_base_url, f"v1/repos/{config.repo}/connect/dumps"
+    )
+    resp = requests.post(
+        url,
+        json=dict(public_key=public_key),
+        headers={"access-token": config.token},
+    )
+    if resp.status_code == 422:
+        env.logger.error("Failed to dump with error: %s", resp.json())
+        sys.exit(-1)
+    elif resp.status_code == 401:
+        env.logger.error(
+            "Failed to dump permission error: %s, please ensure your Access Token has API_CONNECT_DUMP permission",
+            resp.json(),
+        )
+        sys.exit(-1)
+    resp.raise_for_status()
+
+    dump_id = resp.json()["id"]
+    env.logger.info(
+        "Created dump [green]%s[/] with public_key [green]%s[/], waiting for updates ...",
+        dump_id,
+        public_key,
+        extra={"markup": True, "highlighter": None},
+    )
+
+    url = urllib.parse.urljoin(
+        env.api_base_url, f"v1/repos/{config.repo}/connect/dumps/{dump_id}"
+    )
+    while True:
+        time.sleep(5)
+        resp = requests.get(url, headers={"access-token": config.token})
+        # TODO: provide friendly error messages here
+        resp.raise_for_status()
+        payload = resp.json()
+        state = DumpRequestState[payload["state"]]
+        if state == DumpRequestState.FAILED:
+            env.logger.error("Failed to dump with error: %s", payload["error_message"])
+            sys.exit(-1)
+        elif state == DumpRequestState.COMPLETE:
+            break
+        else:
+            env.logger.debug("State is %s, keep polling...", state)
+
+    download_url = payload["download_url"]
+    sealed_box = SealedBox(private_key)
+    encryption_key = json.loads(
+        sealed_box.decrypt(URLSafeBase64Encoder.decode(payload["encryption_key"]))
+    )
+    key = URLSafeBase64Encoder.decode(encryption_key["key"])
+    iv = URLSafeBase64Encoder.decode(encryption_key["iv"])
+
+    with (
+        tempfile.SpooledTemporaryFile(SPOOLED_FILE_MAX_SIZE) as encrypted_file,
+        tempfile.SpooledTemporaryFile(SPOOLED_FILE_MAX_SIZE) as decrypted_file,
+    ):
+        with (requests.get(download_url, stream=True) as req,):
+            for chunk in req.iter_content(4096):
+                encrypted_file.write(chunk)
+        encrypted_file.flush()
+        encrypted_file.seek(0)
+        env.logger.info("Decrypting downloaded file ...")
+        decrypt_file(
+            input_file=encrypted_file, output_file=decrypted_file, key=key, iv=iv
+        )
+        with tarfile.open(fileobj=decrypted_file, mode="r:gz") as tar_file:
+            if hasattr(tarfile, "data_filter"):
+                tar_file.extractall(filter="data")
+            else:
+                env.logger.warning("Performing unsafe tar file extracting")
+                tar_file.extractall()
+
+    env.logger.info("done")

{beanhub_cli-1.4.2 → beanhub_cli-1.5.0b2}/beanhub_cli/environment.py

@@ -1,6 +1,7 @@
 import dataclasses
 import enum
 import logging
+import os
 
 import click
 from beancount_black.formatter import VERBOSE_LOG_LEVEL
@@ -30,6 +31,7 @@ LOG_LEVEL_MAP = {
 class Environment:
     log_level: LogLevel = LogLevel.INFO
     logger: logging.Logger = logging.getLogger("beanhub_cli")
+    api_base_url: str = os.getenv("BEANHUB_API_URL", "https://api.beanhub.io")
 
 
 pass_env = click.make_pass_decorator(Environment, ensure=True)

beanhub_cli-1.5.0b2/beanhub_cli/login.py

@@ -0,0 +1,88 @@
+import platform
+import time
+import urllib.parse
+import webbrowser
+
+from .cli import cli
+from .config import AccessToken
+from .config import Config
+from .config import get_config_path
+from .config import load_config
+from .config import Repository
+from .config import save_config
+from .environment import Environment
+from .environment import pass_env
+from .utils import check_imports
+
+
+@cli.command(name="login", help="Login your BeanHub account")
+@pass_env
+def main(env: Environment):
+    check_imports(
+        logger=env.logger,
+        module_names=["requests", "tomli", "tomli_w"],
+        required_extras=["login"],
+    )
+
+    import requests
+
+    config_path = get_config_path()
+    config = load_config()
+    if config is not None and config.access_token is not None:
+        # TODO: ask the user if they want to log out the original session first
+        env.logger.error(
+            "Already logged in, if you want to login again, please delete the config file at %s first",
+            config_path,
+        )
+        return
+
+    env.logger.info("Creating auth session ...")
+    url = urllib.parse.urljoin(env.api_base_url, "v1/auth/sessions")
+    resp = requests.post(url, json=dict(hostname=platform.node()))
+    # TODO: check status and provide more user friendly error message?
+    resp.raise_for_status()
+
+    data = resp.json()
+    session_id = data["id"]
+    code = data["code"]
+    poll_url = data["poll_url"]
+    auth_url = data["auth_url"]
+
+    env.logger.info(
+        "Auth Code: %s",
+        code,
+    )
+    if not webbrowser.open(data["auth_url"], new=2):
+        env.logger.info(
+            "Cannot open auth url, please open it manually in your browser: %s",
+            auth_url,
+        )
+
+    env.logger.info(
+        "Waiting granting access for current auth session: %s ...", session_id
+    )
+    while True:
+        time.sleep(5)
+        resp = requests.get(poll_url)
+        if resp.status_code == 200:
+            payload = resp.json()
+            # TODO: ideally this should be a different API call, make it simple for now
+            repositories = payload["repositories"]
+            save_config(
+                Config(
+                    access_token=AccessToken(token=payload["token"]),
+                    repo=Repository(default=repositories[0]) if repositories else None,
+                )
+            )
+            env.logger.info("Session access granted, saved config to %s", config_path)
+            break
+        elif resp.status_code == 202:
+            env.logger.debug("Session access not granted yet, try again later")
+        else:
+            env.logger.error(
+                "Failed to fetch token, encountered unexpected status code %s",
+                resp.status_code,
+            )
+            return
+
+    env.logger.info("done")

{beanhub_cli-1.4.2 → beanhub_cli-1.5.0b2}/beanhub_cli/main.py

@@ -1,7 +1,9 @@
 from . import format  # noqa
 from .cli import cli
+from .connect import main as connect  # noqa
 from .forms import main as forms  # noqa
 from .import_cli import main as import_cli  # noqa
+from .login import main as login  # noqa
 
 __ALL__ = [cli]
 

beanhub_cli-1.5.0b2/beanhub_cli/utils.py

@@ -0,0 +1,24 @@
+import logging
+import sys
+
+
+def check_imports(
+    logger: logging.Logger, module_names: list[str], required_extras: list[str]
+):
+    missing_modules = []
+    for name in module_names:
+        try:
+            __import__(name)
+        except ImportError:
+            missing_modules.append(name)
+    if not missing_modules:
+        return
+    extras = ",".join(required_extras)
+    logger.error(
+        "Cannot import module %s, please ensure that you install beanhub-cli with optional deps [%s]. "
+        'Like `pip install "beanhub-cli[%s]"`',
+        missing_modules,
+        extras,
+        extras,
+    )
+    sys.exit(-1)

{beanhub_cli-1.4.2 → beanhub_cli-1.5.0b2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "beanhub-cli"
-version = "1.4.2"
+version = "1.5.0b2"
 description = "Command line tools for BeanHub"
 authors = ["Fang-Pen Lin <fangpen@launchplatform.com>"]
 license = "MIT"
@@ -26,8 +26,15 @@ rich = "^13.6.0"
 beanhub-import = ">= 1.0.0, <2.0.0"
 beanhub-extract = ">= 0.1.0, <1.0.0"
 beancount-parser = ">= 1.2.3, <1.3.0"
+requests = { version ="^2.32.3", optional = true }
+tomli = { version = "^2.2.1", optional = true }
+tomli-w = { version = "^1.1.0", optional = true }
+pynacl = { version = "^1.5.0", optional = true }
+cryptography = { version = "^44.0.0", optional = true}
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.extras]
+login = ["requests", "tomli", "tomli-w"]
+connect = ["requests", "tomli", "tomli-w", "pynacl", "cryptography"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.1"