calkit-python 0.12.0__tar.gz → 0.13.0__tar.gz

{calkit_python-0.12.0 → calkit_python-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.12.0
+Version: 0.13.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.12.0 → calkit_python-0.13.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.12.0"
+__version__ = "0.13.0"
 
 from .core import *
 from . import git

{calkit_python-0.12.0 → calkit_python-0.13.0}/calkit/core.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import base64
 import glob
 import json
 import logging
@@ -10,6 +11,8 @@ import pickle
 import re
 import subprocess
 
+import requests
+
 try:
     from datetime import UTC
 except ImportError:
@@ -276,3 +279,73 @@ def check_system_deps(wdir: str | None = None) -> None:
             dep_name = re.split("[=<>]", dep)[0]
         if not check_dep_exists(dep_name):
             raise ValueError(f"{dep_name} not found")
+
+
+def project_and_path_from_path(path: str) -> tuple:
+    """Split a path into project and path, respecting the ``CALKIT_PROJECT``
+    environmental variable if set.
+
+    For example, a path like
+
+        someone/some-project:some/path/to/file.png
+
+    will return
+
+        (someone/some-project, some/path/to/file.png)
+    """
+    path_split = path.split(":")
+    if len(path_split) == 2:
+        project = path_split[0]
+        path = path_split[1]
+    elif len(path_split) == 1:
+        project = None
+    else:
+        raise ValueError("Path has too many colons in it")
+    if project is None:
+        project = os.getenv("CALKIT_PROJECT")
+    return project, path
+
+
+def read_file(path: str, as_bytes: bool = None) -> str | bytes:
+    """Read file content from path, which can optionally include a project
+    identifier, which if specified will indicate we should read from the API.
+    """
+    project, path = project_and_path_from_path(path)
+    if as_bytes is None:
+        _, ext = os.path.splitext(path)
+        as_bytes = ext in [
+            ".png",
+            ".jpg",
+            ".gif",
+            ".jpeg",
+            ".pdf",
+            ".xlsx",
+            ".docx",
+        ]
+    if project is not None:
+        import calkit.cloud
+
+        if len(project.split("/")) != 2:
+            raise ValueError("Invalid project identifier (too many slashes)")
+        resp = calkit.cloud.get(f"/projects/{project}/contents/{path}")
+        # If the response has a content key, that is a base64 encoded string
+        if (content := resp.get("content")) is not None:
+            # Load the content appropriately
+            content_bytes = base64.b64decode(content)
+            if as_bytes:
+                return content_bytes
+            else:
+                return content_bytes.decode()
+        # If the response has a URL, we can fetch from that directly
+        elif (url := resp.get("url")) is not None:
+            resp2 = requests.get(url)
+            resp2.raise_for_status()
+            if as_bytes:
+                return resp2.content
+            else:
+                return resp2.text
+        else:
+            raise ValueError("No content or URL returned from API")
+    # Project is None, so let's just read a local file
+    with open(path, mode="rb" if as_bytes else "r") as f:
+        return f.read()

calkit_python-0.13.0/calkit/datasets.py

@@ -0,0 +1,71 @@
+"""Functionality for working with datasets."""
+
+from __future__ import annotations
+
+import base64
+import io
+from typing import Literal
+
+import calkit
+import calkit.config
+
+DEFAULT_ENGINE = calkit.config.read().dataframe_engine
+
+
+def _get_df_lib(engine: str):
+    if engine == "pandas":
+        import pandas
+
+        return pandas
+    elif engine == "polars":
+        import polars
+
+        return polars
+    else:
+        raise ValueError("Unknown engine")
+
+
+def list_datasets() -> list[dict]:
+    """Read the Calkit metadata file and list out our datasets."""
+    ck_info = calkit.load_calkit_info(as_pydantic=False, process_includes=True)
+    return ck_info.get("datasets", [])
+
+
+def read_dataset(
+    path: str,
+    engine: Literal["pandas", "polars"] = DEFAULT_ENGINE,
+):
+    """Read a dataset from a path.
+
+    Path can include the project owner/name like
+
+        someone/some-project:my-data-folder/data.csv
+
+    When a project is set via the ``CALKIT_PROJECT`` environmental variable,
+    we will use the API to fetch the data.
+    """
+
+    def load_from_fobj(fobj, path: str):
+        """Read from a filelike object or path."""
+        if path.endswith(".csv"):
+            return _get_df_lib(engine).read_csv(fobj)
+        elif path.endswith(".parquet"):
+            return _get_df_lib(engine).read_parquet(fobj)
+
+    project, path = calkit.project_and_path_from_path(path)
+    if project is not None:
+        if len(project.split("/")) != 2:
+            raise ValueError("Invalid project identifier (too many slashes)")
+        resp = calkit.cloud.get(f"/projects/{project}/contents/{path}")
+        # If the response has a content key, that is a base64 encoded string
+        if (content := resp.get("content")) is not None:
+            # Load the content appropriately
+            content_bytes = base64.b64decode(content)
+            return load_from_fobj(io.BytesIO(content_bytes), path=path)
+        # If the response has a URL, we can fetch from that directly
+        elif (url := resp.get("url")) is not None:
+            return load_from_fobj(url, path=path)
+        else:
+            raise ValueError("No content or URL returned from API")
+    # Project is None, so let's just read a local file
+    return load_from_fobj(path, path)

calkit_python-0.12.0/calkit/data.py

@@ -1,60 +0,0 @@
-"""Functionality for working with datasets.
-
-Since the dependencies here are optional, we need to ensure this isn't imported
-by default, or otherwise ensure ``import calkit`` works when the data
-dependencies are not installed.
-"""
-
-from __future__ import annotations
-
-from typing import Literal, Union
-
-import pandas as pd
-import polars as pl
-
-import calkit.config
-
-DEFAULT_ENGINE = calkit.config.read().dataframe_engine
-
-
-def list_data():
-    """Read the Calkit metadata file and list out our datasets."""
-    pass
-
-
-def read_data(
-    path: str, engine: Literal["pandas", "polars"] = DEFAULT_ENGINE
-) -> Union[pd.DataFrame, pl.DataFrame]:
-    """Read (tabular) data from dataset with path ``path`` and return a
-    DataFrame.
-
-    If the dataset doesn't exist locally, but is a DVC object, download it
-    first.
-
-    If the dataset path includes a user and project name, we add it to the
-    project as an imported dataset, and therefore DVC import it?
-
-    For example: someuser/someproject:data/somefile.parquet
-
-    We can run a DVC import command if it needs to be imported. We will need to
-    find the Git repo and path within it? Maybe we should require an explicit
-    import of the data.
-    """
-    pass
-
-
-def write_data(
-    data: Union[pd.DataFrame, pl.DataFrame],
-    path: str,
-    filename: str | None = None,
-    commit=False,
-):
-    """Write ``data`` to the dataset with path ``path``.
-
-    If the dataset path is a directory, the filename must be specified.
-
-    If the path is not a Calkit dataset, it will be created.
-
-    If ``commit`` is specified, create a commit for the dataset update.
-    """
-    pass