squirrels 0.1.1.post1__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl

squirrels/_utils.py

@@ -1,56 +1,87 @@
-from typing import List, Dict, Optional, Union, Any
-from types import ModuleType
+from typing import Sequence, Optional, Union, Any, TypeVar, Callable
 from pathlib import Path
-from importlib.machinery import SourceFileLoader
-import json
-
-from squirrels._timed_imports import jinja2 as j2, pandas as pd, pd_types
+from pandas.api import types as pd_types
+import json, jinja2 as j2, pandas as pd
 
 FilePath = Union[str, Path]
 
 
-# Custom Exceptions
+## Custom Exceptions
+
 class InvalidInputError(Exception):
+    """
+    Use this exception when the error is due to providing invalid inputs to the REST API
+    """
     pass
 
 class ConfigurationError(Exception):
+    """
+    Use this exception when the server error is due to errors in the squirrels project instead of the squirrels framework/library
+    """
     pass
 
-class AbstractMethodCallError(NotImplementedError):
-    def __init__(self, cls, method, more_message = ""):
-        message = f"Abstract method {method}() not implemented in {cls.__name__}."
-        super().__init__(message + more_message)
+class FileExecutionError(ConfigurationError):
+    def __init__(self, message: str, error: Exception, *args) -> None:
+        new_message = message + f"\n... Produced error message `{error}` (see above for more details)"
+        super().__init__(new_message, *args)
 
 
-# Utility functions/variables
-j2_env = j2.Environment(loader=j2.FileSystemLoader('.'))
+## Utility functions/variables
 
+def join_paths(*paths: FilePath) -> Path:
+    """
+    Joins paths together.
+
+    Parameters:
+        paths (str | pathlib.Path): The paths to join.
 
-def import_file_as_module(filepath: Optional[FilePath]) -> ModuleType:
+    Returns:
+        (pathlib.Path) The joined path.
+    """
+    return Path(*paths)
+
+
+_j2_env = j2.Environment(loader=j2.FileSystemLoader('.'))
+
+def render_string(raw_str: str, kwargs: dict) -> str:
     """
-    Imports a python file as a module.
+    Given a template string, render it with the given keyword arguments
 
     Parameters:
-        filepath: The path to the file to import.
+        raw_str: The template string
+        kwargs: The keyword arguments
 
     Returns:
-        The imported module.
+        The rendered string
     """
-    filepath = str(filepath) if filepath is not None else None
-    return SourceFileLoader(filepath, filepath).load_module() if filepath is not None else None
+    template = _j2_env.from_string(raw_str)
+    return template.render(kwargs)
 
 
-def join_paths(*paths: FilePath) -> Path:
+T = TypeVar('T')
+def __process_file_handler(file_handler: Callable[[FilePath], T], filepath: FilePath, is_required: bool) -> Optional[T]:
+    try:
+        return file_handler(filepath)
+    except FileNotFoundError as e:
+        if is_required:
+            raise ConfigurationError(f"Required file not found: '{str(filepath)}'") from e
+
+
+def read_file(filepath: FilePath, *, is_required: bool = True) -> Optional[str]:
     """
-    Joins paths together.
+    Reads a file and return its content if required
 
     Parameters:
-        paths: The paths to join.
+        filepath (str | pathlib.Path): The path to the file to read
+        is_required: If true, throw error if file doesn't exist
 
     Returns:
-        The joined path.
+        Content of the file, or None if doesn't exist and not required
     """
-    return Path(*paths)
+    def file_handler(filepath: FilePath):
+        with open(filepath, 'r') as f:
+            return f.read()
+    return __process_file_handler(file_handler, filepath, is_required)
 
 
 def normalize_name(name: str) -> str:
@@ -79,28 +110,7 @@ def normalize_name_for_api(name: str) -> str:
     return name.replace('_', '-')
 
 
-def get_row_value(row: pd.Series, value: str) -> Any:
-    """
-    Gets the value of a row from a pandas Series.
-
-    Parameters:
-        row: The row to get the value from.
-        value: The name of the column to get the value from.
-
-    Returns:
-        The value of the column.
-
-    Raises:
-        ConfigurationError: If the column does not exist.
-    """
-    try:
-        result = row[value]
-    except KeyError as e:
-        raise ConfigurationError(f'Column name "{value}" does not exist') from e
-    return result
-
-
-def df_to_json(df: pd.DataFrame, dimensions: List[str] = None) -> Dict[str, Any]:
+def df_to_json0(df: pd.DataFrame, dimensions: list[str] = None) -> dict[str, Any]:
     """
     Convert a pandas DataFrame to the same JSON format that the dataset result API of Squirrels outputs.
 
@@ -124,10 +134,10 @@ def df_to_json(df: pd.DataFrame, dimensions: List[str] = None) -> Dict[str, Any]
     
     out_dimensions = non_numeric_fields if dimensions is None else dimensions
     out_schema = {"fields": out_fields, "dimensions": out_dimensions}
-    return {"response_version": 0, "schema": out_schema, "data": in_df_json["data"]}
+    return {"schema": out_schema, "data": in_df_json["data"]}
 
 
-def load_json_or_comma_delimited_str_as_list(input_str: str) -> List[str]:
+def load_json_or_comma_delimited_str_as_list(input_str: Union[str, Sequence]) -> Sequence[str]:
     """
     Given a string, load it as a list either by json string or comma delimited value
 
@@ -137,6 +147,9 @@ def load_json_or_comma_delimited_str_as_list(input_str: str) -> List[str]:
     Returns:
         The list representation of the input string
     """
+    if not isinstance(input_str, str):
+        return (input_str)
+    
     output = None
     try:
         output = json.loads(input_str)
@@ -145,5 +158,24 @@ def load_json_or_comma_delimited_str_as_list(input_str: str) -> List[str]:
     
     if isinstance(output, list):
         return output
+    elif input_str == "":
+        return []
     else:
-        return [] if input_str == "" else input_str.split(",")
+        return [x.strip() for x in input_str.split(",")]
+
+
+X, Y = TypeVar('X'), TypeVar('Y')
+def process_if_not_none(input_val: Optional[X], processor: Callable[[X], Y]) -> Optional[Y]:
+    """
+    Given a input value and a function that processes the value, return the output of the function unless input is None
+
+    Parameters:
+        input_val: The input value
+        processor: The function that processes the input value
+    
+    Returns:
+        The output type of "processor" or None if input value if None
+    """
+    if input_val is None:
+        return None
+    return processor(input_val)

squirrels/_version.py

@@ -1,3 +1,3 @@
-__version__ = '0.1.1'
+from . import __version__
 
-major_version, minor_version, patch_version = __version__.split('.')
+sq_major_version, sq_minor_version, sq_patch_version = __version__.split('.')[:3]

squirrels/arguments/init_time_args.py

@@ -0,0 +1,32 @@
+from typing import Callable, Any
+from dataclasses import dataclass
+
+
+@dataclass
+class BaseArguments:
+    proj_vars: dict[str, Any]
+    env_vars: dict[str, Any]
+
+
+@dataclass
+class ConnectionsArgs(BaseArguments):
+    _get_credential: Callable[[str], tuple[str, str]]
+
+    def __post_init__(self):
+        self.get_credential = self._get_credential
+
+    def get_credential(self, key: str) -> tuple[str, str]:
+        """
+        Return (username, password) tuple configured for credentials key in environcfg.yaml
+
+        Parameters:
+            key: The credentials key
+        
+        Returns:
+            A tuple of strings of size 2
+        """
+
+
+@dataclass
+class ParametersArgs(BaseArguments):
+    pass

squirrels/arguments/run_time_args.py

@@ -0,0 +1,82 @@
+from typing import Union, Callable, Any
+from dataclasses import dataclass
+from sqlalchemy import Engine, Pool
+import pandas as pd, sqlite3
+
+from .init_time_args import ParametersArgs
+from ..user_base import User
+from ..parameters import Parameter
+from .._connection_set import ConnectionSetIO
+
+
+@dataclass
+class ContextArgs(ParametersArgs):
+    user: User
+    prms: dict[str, Parameter]
+    args: dict[str, Any]
+
+
+@dataclass
+class ModelDepsArgs(ContextArgs):
+    ctx: dict[str, Any]
+
+
+@dataclass
+class ModelArgs(ModelDepsArgs):
+    connection_name: str
+    connections: dict[str, Union[Engine, Pool]]
+    _ref: Callable[[str], pd.DataFrame]
+    dependencies: set[str]
+
+    def __post_init__(self):
+        self.ref = self._ref
+    
+    def ref(self, model: str) -> pd.DataFrame:
+        """
+        Returns the result (as pandas DataFrame) of a dependent model (predefined in "dependencies" function)
+
+        Note: This is different behaviour than the "ref" function for SQL models, which figures out the dependent models for you, 
+        and returns a string for the table/view name in SQLite instead of a pandas DataFrame.
+
+        Parameters:
+            model: The model name
+        
+        Returns:
+            A pandas DataFrame
+        """
+
+    def run_external_sql(self, sql: str, *, connection_name: str = None, **kwargs) -> pd.DataFrame:
+        """
+        Runs a SQL query against an external database, with option to specify the connection name
+
+        Parameters:
+            sql: The SQL query
+            connection_name: The connection name for the database. If None, uses the one configured for the model
+        
+        Returns:
+            The query result as a pandas DataFrame
+        """
+        connection_name = self.connection_name if connection_name is None else connection_name
+        return ConnectionSetIO.obj.run_sql_query_from_conn_name(sql, connection_name)
+
+    def run_sql_on_dataframes(self, query: str, *, dataframes: dict[str, pd.DataFrame] = None, **kwargs) -> pd.DataFrame:
+        """
+        Uses a dictionary of dataframes to execute a SQL query in an in-memory sqlite database
+
+        Parameters:
+            query: The SQL query to run using sqlite
+            dataframes: A dictionary of table names to their pandas Dataframe
+        
+        Returns:
+            The result as a pandas Dataframe from running the query
+        """
+        if dataframes is None:
+            dataframes = {x: self.ref(x) for x in self.dependencies}
+
+        conn = sqlite3.connect(":memory:")
+        try:
+            for name, df in dataframes.items():
+                df.to_sql(name, conn, index=False)
+            return pd.read_sql(query, conn)
+        finally:
+            conn.close()