gamsapi 52.4.0__cp314-cp314-win_amd64.whl → 53.0.0rc1__cp314-cp314-win_amd64.whl

gams/_version.py

@@ -1 +1 @@
-__version__ = "52.4.0"
+__version__ = "53.0.0rc1"

gams/connect/agents/_excel/excelagent.py

@@ -186,8 +186,8 @@ class ExcelAgent(ConnectAgent):
     def parse_range(self, sym_range, wb, clear_sheet=False, create_missing=False):
         sheet, rng = self._split_range(sym_range, wb)
         toc_range = f"'{sheet}'!{rng.split(':')[0]}"
-        sheet = self.sheet_by_name(sheet, wb, clear_sheet, create_missing)
         nw_col, nw_row, se_col, se_row = self._range_to_coords(rng)
+        sheet = self.sheet_by_name(sheet, wb, clear_sheet, create_missing, nw_col, nw_row, se_col, se_row)
         return sheet, nw_col, nw_row, se_col, se_row, toc_range
 
     def parse_index(self, index, wb, index_parameter_map):
@@ -240,12 +240,14 @@ class ExcelAgent(ConnectAgent):
                 symbols.append(sym)
         return symbols
 
-    def sheet_by_name(self, sheet, wb, clear_sheet=False, create_missing=False):
+    def sheet_by_name(self, sheet, wb, clear_sheet=False, create_missing=False, nw_col=None, nw_row=None, se_col=None, se_row=None):
         for idx, s in enumerate(wb.sheetnames):
             if sheet.lower() == s.lower():
-                if clear_sheet:
+                if clear_sheet is True:
                     wb.delete_sheet(s)
                     return wb.create_sheet(s, idx)
+                if clear_sheet == "range":
+                    wb.delete_range(s, nw_col, nw_row, se_col, se_row)
                 return wb.get_sheet(s)
         if create_missing:
             return wb.create_sheet(sheet)

gams/connect/agents/_excel/workbook.py

@@ -111,6 +111,21 @@ class Workbook:
         else:
             raise Exception("Invalid engine.")
 
+    def delete_range(self, sheet_name, nw_col, nw_row, se_col, se_row):
+        if self._engine == "xlwings":
+            raise Exception("Not imlemented.")
+        elif self._engine == "openpyxl":
+            sheet = self._wb[sheet_name]
+            if se_row is None:
+                se_row = sheet.max_row
+            if se_col is None:
+                se_col = sheet.max_column
+            for row in range(nw_row+1, se_row + 1):
+                for col in range(nw_col+1, se_col + 1):
+                    sheet.cell(row=row, column=col).value = None
+        else:
+            raise Exception("Invalid engine.")
+
     def delete_sheet(self, sheet_name):
         if self._engine == "xlwings":
             self._wb.sheets[sheet_name].delete()

gams/connect/agents/_sqlconnectors/_accesshandler.py

@@ -30,6 +30,7 @@ from gams.connect.agents._sqlconnectors._databasehandler import DatabaseConnecto
 class AccessConnector(DatabaseConnector):
     SUPPORTED_INSERT_METHODS = ["default", "bulkInsert"]
     QUOTE_CHAR = ["[]", '""', "``"]
+    SCHEMA_SUPPORT = False
 
     @staticmethod
     def _create_accdb(dbpath):

gams/connect/agents/_sqlconnectors/_databasehandler.py

@@ -45,10 +45,20 @@ class ConnectionType(Enum):
     SQLSERVER = "sqlserver"
     ACCESS = "access"
 
+CONNECTOR = {
+    "SQLiteConnector": ConnectionType.SQLITE.value,
+    "SQLAlchemyConnector": ConnectionType.SQLALCHEMY.value,
+    "PyodbcConnector": ConnectionType.PYODBC.value,
+    "MySQLConnector": ConnectionType.MYSQL.value,
+    "PostgresConnector": ConnectionType.POSTGRES.value,
+    "SQLServerConnector": ConnectionType.SQLSERVER.value,
+    "AccessConnector": ConnectionType.ACCESS.value,
+}
 
 class DatabaseConnector(ABC):
     SUPPORTED_INSERT_METHODS = []
     QUOTE_CHAR = []
+    SCHEMA_SUPPORT = True
 
     def __init__(
         self,
@@ -70,6 +80,13 @@ class DatabaseConnector(ABC):
                 f"Valid methods are >{self.SUPPORTED_INSERT_METHODS}<"
             )
 
+    def is_schema_available(self, schema: str | None):
+        """Checks if the connectionType allows schema"""
+        if (not self.SCHEMA_SUPPORT) and (schema is not None):
+            self._raise_error(
+                f"connectionType >{CONNECTOR[type(self).__name__]}< does not support schema but schemaName is set to >{schema}<."
+            )
+
     def read_table(self, sql_query: str, read_sql_args: dict) -> pd.DataFrame:
         """
         Read data from select DBMS using the provided SQL Query. Returns a pandas.DataFrame

gams/connect/agents/_sqlconnectors/_sqlitehandler.py

@@ -31,6 +31,7 @@ from gams.connect.agents._sqlconnectors._databasehandler import DatabaseConnecto
 class SQLiteConnector(DatabaseConnector):
     SUPPORTED_INSERT_METHODS = ["default"]
     QUOTE_CHAR = ["[]", '""', "``"]
+    SCHEMA_SUPPORT = False
 
     def connect(self, connection_details, connection_args, **kwargs) -> None:
 

gams/connect/agents/connectagent.py

@@ -381,14 +381,21 @@ class ConnectAgent(ABC):
                 # pandas-version-check
                 if self._pandas_version_before(pd.__version__, "2.2"):  # pandas < 2.2.0
                     df.iloc[:, -1] = df.iloc[:, -1].replace(vs)
-                else:  # pandas >= 2.2.0
+                elif self._pandas_version_before(pd.__version__, "3.0"):  # 2.2.0 <= pandas < 3.0.0
                     with pd.option_context("future.no_silent_downcasting", True):
                         df.iloc[:, -1] = df.iloc[:, -1].replace(vs).infer_objects()
+                else:  # pandas >= 3.0.0
+                    df.iloc[:, -1] = df.iloc[:, -1].replace(vs).infer_objects()
+
             else:
                 # pandas-version-check
                 if self._pandas_version_before(pd.__version__, "2.2"):  # pandas < 2.2.0
                     df.iloc[~mask, -1] = df.iloc[~mask, -1].replace(vs)
-                else:  # pandas >= 2.2.0
+                elif self._pandas_version_before(pd.__version__, "3.0"):  # 2.2.0 <= pandas < 3.0.0
+                    df.iloc[~mask, -1] = (
+                        df.iloc[~mask, -1].replace(vs).infer_objects()
+                    )
+                else:  # pandas >= 3.0.0
                     with pd.option_context("future.no_silent_downcasting", True):
                         df.iloc[~mask, -1] = (
                             df.iloc[~mask, -1].replace(vs).infer_objects()

gams/connect/agents/csvreader.py

@@ -325,9 +325,11 @@ class CSVReader(ConnectAgent):
         # pandas-version-check
         if self._pandas_version_before(pd.__version__, "2.2"):  # pandas < 2.2.0
             df.isetitem(-1, df.iloc[:, -1].replace(self._value_sub))
-        else:  # pandas >= 2.2.0
+        elif self._pandas_version_before(pd.__version__, "3.0"):  # 2.2.0 <= pandas < 3.0.0
             with pd.option_context("future.no_silent_downcasting", True):
                 df.isetitem(-1, df.iloc[:, -1].replace(self._value_sub).infer_objects())
+        else:  # pandas >= 3.0.0
+            df.isetitem(-1, df.iloc[:, -1].replace(self._value_sub).infer_objects())
 
         if self._trace > 2:
             self._cdb.print_log(f"DataFrame after value substitution:\n{df}")
@@ -426,9 +428,10 @@ class CSVReader(ConnectAgent):
         # pandas-version-check
         if self._pandas_version_before(pd.__version__, "2.2"):  # pandas < 2.2.0
             df = df.stack(level=df.columns.names, dropna=False)
-        else:  # pandas >= 2.2.0
+        elif self._pandas_version_before(pd.__version__, "3.0"):  # 2.2.0 <= pandas < 3.0.0
             df = df.stack(level=df.columns.names, future_stack=True)
-
+        else:  # pandas >= 3.0.0
+            df = df.stack(level=df.columns.names)
         return df
 
     def _sort_value_columns(self, df: pd.DataFrame) -> pd.DataFrame:
@@ -495,6 +498,36 @@ class CSVReader(ConnectAgent):
                 )
 
         return df
+    
+    def _create_sym(self, df, domain):
+        """Creates symbol (parameter or set) in the Connect container
+
+        Parameters
+        ----------
+        df : pd.DataFrame | None
+            Symbol records
+        domain : list
+            Symbol domains
+
+        Returns
+        -------
+        Symbol object
+            Created symbol
+        """
+        if self._sym_type == "par":
+            sym = self._cdb.container.addParameter(
+                self._name,
+                domain,
+                records=df,
+            )
+        else:
+            sym = self._cdb.container.addSet(
+                self._name,
+                domain,
+                records=df,
+            )
+
+        return sym
 
     def execute(self):
         if self._trace > 0:
@@ -543,7 +576,37 @@ class CSVReader(ConnectAgent):
                 f"Arguments for reading the CSV file:\n{self._read_csv_args}"
             )
 
-        df = pd.read_csv(self._file, **self._read_csv_args)
+        try:
+            df = pd.read_csv(self._file, **self._read_csv_args)
+
+        except pd.errors.EmptyDataError:
+            if self._trace > 1:
+                self._cdb.print_log(
+                    "Empty data after reading CSV file."
+                )
+                
+            # set dimensionality
+            dim = len(self._index_cols)
+            if self._stack:
+                if self._multiheader:
+                    dim += len(self._header)
+                else:
+                    dim += 1
+
+            sym = self._create_sym(None, ["*"] * dim)
+
+            # For symbols with None records, empty df is assigned
+            self._transform_sym_none_to_empty(sym)
+
+            if self._trace > 2:
+                self._cdb.print_log(
+                    f"Connect Container symbol={self._name}:\n {sym.records}\n"
+                )
+
+            if self._trace > 0:
+                self._describe_container(self._cdb.container, "Connect Container (after):")
+
+            return
 
         if self._trace > 2:
             self._cdb.print_log(
@@ -569,11 +632,7 @@ class CSVReader(ConnectAgent):
         if self._auto_col is not None and not df.columns.empty:
             self._generate_column_labels(df)
 
-            if self._stack:
-                dim += 1
-                domain.append("*")
-
-        elif self._stack:
+        if self._stack:
             if self._multiheader:
                 dim += len(self._header)
                 domain.extend(
@@ -596,8 +655,12 @@ class CSVReader(ConnectAgent):
                         pd.__version__, "2.2"
                     ):  # pandas < 2.2.0
                         df = df.stack(dropna=False)
-                    else:  # pandas >= 2.2.0
+                    elif self._pandas_version_before(
+                        pd.__version__, "3.0"
+                    ):  # 2.2.0 <= pandas < 3.0.0
                         df = df.stack(future_stack=True)
+                    else:  # pandas >= 3.0.0
+                        df = df.stack()
 
                 if dim == 1 or (self._multiheader and dim == columns.nlevels):
                     # drop pandas default index level
@@ -638,11 +701,6 @@ class CSVReader(ConnectAgent):
             self._substitute_values(df)
         df.dropna(inplace=True)
 
-        if self._sym_type == "par":
-            sym = gt.Parameter(self._cdb.container, self._name, domain=domain)
-        else:
-            sym = gt.Set(self._cdb.container, self._name, domain=domain)
-
         # reset the index to the default integer index
         df = df.reset_index(drop=True)
 
@@ -651,7 +709,7 @@ class CSVReader(ConnectAgent):
                 "Final DataFrame that will be processed by" f" GAMSTransfer:\n{df}"
             )
 
-        sym.setRecords(df)
+        sym = self._create_sym(df, domain)
 
         if dim > 0 and self._stack:
             if self._multiheader:

gams/connect/agents/excelreader.py

@@ -276,9 +276,11 @@ class ExcelReader(ExcelAgent):
             # pandas-version-check
             if self._pandas_version_before(pd.__version__, "2.2"):  # pandas < 2.2.0
                 df.replace(value_sub, inplace=True)
-            else:  # pandas >= 2.2.0
+            elif self._pandas_version_before(pd.__version__, "3.0"):  # 2.2.0 <= pandas < 3.0.0
                 with pd.option_context("future.no_silent_downcasting", True):
                     df = df.replace(value_sub).infer_objects()
+            else:  # pandas >= 3.0.0
+                df = df.replace(value_sub).infer_objects()
         return df
 
     def _write(self, df, sym_name, sym_type, rdim, cdim):
@@ -654,9 +656,11 @@ class ExcelReader(ExcelAgent):
             # pandas-version-check
             if self._pandas_version_before(pd.__version__, "2.2"):  # pandas < 2.2.0
                 df.replace(regex=pattern, value=GMS_SV_UNDEF, inplace=True)
-            else:  # pandas >= 2.2.0
+            elif self._pandas_version_before(pd.__version__, "3.0"):  # 2.2.0 <= pandas < 3.0.0
                 with pd.option_context("future.no_silent_downcasting", True):
                     df = df.replace(regex=pattern, value=GMS_SV_UNDEF).infer_objects()
+            else:  # pandas >= 3.0.0
+                df = df.replace(regex=pattern, value=GMS_SV_UNDEF).infer_objects()
         self._write(df, sym_name, sym_type, rdim, cdim)
 
     def _open(self):

gams/connect/agents/schema/ExcelWriter.yaml

@@ -10,8 +10,11 @@ columnDimension:
     min: 0
     max: 20
 clearSheet:
-  type: boolean
   default: false
+  oneof:
+  - type: string
+    allowed: [range]
+  - type: boolean
 mergedCells:
   type: boolean
   default: false
@@ -65,9 +68,12 @@ symbols:
           - type: string
             allowed: [infer]
         clearSheet:
-          type: boolean
           default: null
           nullable: true
+          oneof:
+          - type: string
+            allowed: [range]
+          - type: boolean
         mergedCells:
           type: boolean
           default: null

gams/connect/agents/sqlreader.py

@@ -216,8 +216,12 @@ class SQLReader(ConnectAgent):
                             pd.__version__, "2.2"
                         ):  # pandas < 2.2.0
                             df = df.stack(dropna=False)
-                        else:  # pandas >= 2.2.0
+                        elif self._pandas_version_before(
+                            pd.__version__, "3.0"
+                        ):  # 2.2.0 <= pandas < 3.0.0
                             df = df.stack(future_stack=True)
+                        else:  # pandas >= 3.0.0
+                            df = df.stack()
                         if dim == 1:
                             df = df.droplevel(
                                 level=0

gams/connect/agents/sqlwriter.py

@@ -117,6 +117,7 @@ class SQLWriter(ConnectAgent):
             self._describe_container(self._cdb.container, "Connect Container:")
 
         self._handler.validate_insert_method(method=self._insertMethod)
+        self._handler.is_schema_available(schema=self._schema_name)
         self._open()
 
         try:
@@ -170,6 +171,7 @@ class SQLWriter(ConnectAgent):
                 insertMethod = sym["insertMethod"]
                 skip_text = sym["skipText"]
                 self._handler.validate_insert_method(method=insertMethod)
+                self._handler.is_schema_available(schema=schema)
 
                 if self._small and table_name == "UEL$":
                     self._connect_error(

gams/tools/toolcollection/data/sqlitewrite.py

@@ -24,6 +24,17 @@
 #
 
 from gams.tools.toolcollection.tooltemplate import ToolTemplate
+from gams.connect import ConnectDatabase
+from gams import transfer as gt
+from collections import defaultdict
+
+
+def scalar_symbol_template():
+    """
+    Template for maintaining scalar symbols within a dictionary.
+    Intended to support potential future implementations of SpecialSymbols, subclasses of base symbols
+    """
+    return {"sym_name": [], "sym_recs": []}
 
 
 class Sqlitewrite(ToolTemplate):
@@ -95,6 +106,16 @@ class Sqlitewrite(ToolTemplate):
 
         return False
 
+    @staticmethod
+    def combine_scalars(m: gt.Container, scalar_dict, scalar_name, pd_concat):
+        df = pd_concat(scalar_dict["sym_recs"], ignore_index=True)
+        df.insert(0, "name", scalar_dict["sym_name"])
+        dom = ["name"]
+        if scalar_name in ["scalarvariables", "scalarequations"]:
+            dom.append("attribute")
+            df = df.melt(id_vars="name", var_name="attribute", value_name="value")
+        m.addParameter(name=scalar_name, domain=dom, records=df)
+
     def execute(self):
         if self.dohelp():
             return
@@ -137,9 +158,6 @@ class Sqlitewrite(ToolTemplate):
         unstack = self.check_bool_args("unstack")
         fast = self.check_bool_args("fast")
 
-        from gams.connect import ConnectDatabase
-        from gams import transfer as gt
-
         id_list = None  # reads all if ids is not set
         if "ids" in self.namedargs:
             id_list = self.namedargs_val("ids").split(",")
@@ -148,117 +166,92 @@ class Sqlitewrite(ToolTemplate):
         m: gt.Container = cdb.container
         self.read_id_inputs(m, inputs=id_list)
 
-        scalars = False
-        scalar_parameter, scalar_variable, scalar_equation, symbols = [], [], [], []
+        symbols = []
+        scalar_data = defaultdict(scalar_symbol_template)
         cc = m.data.copy()  # the following loop adds new symbols
 
-        for sym_name, data in cc.items():
-            if data.dimension == 0:
-                scalars = True
-                if isinstance(data, gt.Parameter):  # scalar parameter
-                    scalar_parameter.append(sym_name)
-                elif isinstance(data, gt.Variable):  # scalar variable
-                    scalar_variable.append(sym_name)
-                elif isinstance(data, gt.Equation):  # scalar equation
-                    scalar_equation.append(sym_name)
-            elif isinstance(data, gt.Variable) or isinstance(data, gt.Equation):
-                dom = ",".join(f"d{i}" for i in range(data.dimension))
-                cdb.execute(
-                    {
-                        "Projection": {
-                            "name": f"{sym_name}.all({dom})",
-                            "newName": f"{sym_name}_all({dom})",
-                        }
-                    }
+        for name, sym in cc.items():
+            if sym.dimension == 0:
+                if isinstance(sym, gt.Parameter):
+                    sym_type = "Parameter"
+                elif isinstance(sym, gt.Equation):
+                    sym_type = "Equation"
+                elif isinstance(sym, gt.Variable):
+                    sym_type = "Variable"
+                else:
+                    continue
+                scalar_data[sym_type]["sym_name"].append(sym.name)
+                scalar_data[sym_type]["sym_recs"].append(sym.records)
+            elif isinstance(sym, (gt.Variable, gt.Equation)):
+                if sym.records is not None:
+                    dom = list(sym.records.columns[: sym.dimension])
+                    df = sym.records.melt(
+                        id_vars=dom, var_name="attribute", value_name="value"
+                    )
+                else:
+                    ### The old tool creates a blank table for symbols with no records
+                    df = None
+                m.addParameter(
+                    name=f"{name}_all",
+                    # NOTE: using sym.domain causes validity issues later on in SQLWriter
+                    domain=[dom if isinstance(dom, str) else dom.name for dom in sym.domain] + ["attribute"],
+                    records=df,
                 )
                 symbols.append(
                     {
-                        "name": f"{sym_name}_all",
-                        "tableName": sym_name if small else f"[{sym_name}]",
+                        "name": f"{name}_all",
+                        "tableName": name if small else f"[{name}]",
                         "unstack": True,
                     }
                 )
 
-            elif isinstance(data, gt.Alias):
+            elif isinstance(sym, gt.Alias):
                 pass
             else:
                 symbols.append(
                     {
-                        "name": sym_name,
-                        "tableName": sym_name if small else f"[{sym_name}]",
+                        "name": name,
+                        "tableName": name if small else f"[{name}]",
                     }
                 )
-
-        if scalars:
-            if scalar_parameter:
-                cdb.execute(
-                    {"Projection": {"name": scalar_parameter, "newName": "scalars"}}
-                )
-                m["scalars"].records = m["scalars"].records.rename(
-                    columns={"uni_0": "name"}
-                )
-                symbols.append({"name": "scalars", "tableName": "scalars"})
-            if scalar_variable:
-                cdb.execute(
-                    [
-                        # combine all scalar variables into one
-                        {
-                            "Projection": {
-                                "name": scalar_variable,
-                                "newName": "scalarvariables_dummy",
-                            }
-                        },
-                        # convert the combined variable to parameter with variable attributes
-                        {
-                            "Projection": {
-                                "name": f"scalarvariables_dummy.all(i)",
-                                "newName": f"scalarvariables(i)",
-                            }
-                        },
-                    ]
-                )
-                m["scalarvariables"].records = m["scalarvariables"].records.rename(
-                    columns={"uni_0": "name"}
-                )
-                symbols.append(
-                    {
-                        "name": "scalarvariables",
-                        "tableName": "scalarvariables",
-                        "unstack": True,
-                    }
-                )
-                m.removeSymbols(symbols="scalarvariables_dummy")
-            if scalar_equation:
-                cdb.execute(
-                    [
-                        # combine all scalar equations into one
-                        {
-                            "Projection": {
-                                "name": scalar_equation,
-                                "newName": "scalarequations_dummy",
-                            }
-                        },
-                        # convert the combined equation to parameter with equation attributes
-                        {
-                            "Projection": {
-                                "name": f"scalarequations_dummy.all(i)",
-                                "newName": f"scalarequations(i)",
-                            }
-                        },
-                    ]
-                )
-
-                m["scalarequations"].records = m["scalarequations"].records.rename(
-                    columns={"uni_0": "name"}
-                )
-                symbols.append(
-                    {
-                        "name": "scalarequations",
-                        "tableName": "scalarequations",
-                        "unstack": True,
-                    }
-                )
-                m.removeSymbols(symbols="scalarequations_dummy")
+        if any(data["sym_name"] for data in scalar_data.values()):
+            from pandas import concat as pd_concat
+        if scalar_data["Parameter"]["sym_name"]:
+            self.combine_scalars(
+                m,
+                scalar_dict=scalar_data["Parameter"],
+                scalar_name="scalars",
+                pd_concat=pd_concat,
+            )
+            symbols.append({"name": "scalars", "tableName": "scalars"})
+        if scalar_data["Variable"]["sym_name"]:
+            self.combine_scalars(
+                m,
+                scalar_dict=scalar_data["Variable"],
+                scalar_name="scalarvariables",
+                pd_concat=pd_concat,
+            )
+            symbols.append(
+                {
+                    "name": "scalarvariables",
+                    "tableName": "scalarvariables",
+                    "unstack": True,
+                }
+            )
+        if scalar_data["Equation"]["sym_name"]:
+            self.combine_scalars(
+                m,
+                scalar_dict=scalar_data["Equation"],
+                scalar_name="scalarequations",
+                pd_concat=pd_concat,
+            )
+            symbols.append(
+                {
+                    "name": "scalarequations",
+                    "tableName": "scalarequations",
+                    "unstack": True,
+                }
+            )
         sqlite_params = {
             "connection": {"database": sqlite_file},
             "connectionArguments": {"__globalCommit__": True},

gams/transfer/_internals/algorithms.py

@@ -212,7 +212,7 @@ def convert_to_categoricals_str(
         dfs.append(pd.DataFrame(arrvals))
 
     if has_domains and has_values:
-        df = pd.concat(dfs, axis=1, copy=False)
+        df = pd.concat(dfs, axis=1)
         df.columns = pd.RangeIndex(start=0, stop=len(df.columns))
     elif has_domains or has_values:
         df = dfs[0]
@@ -245,7 +245,7 @@ def convert_to_categoricals_cat(
         dfs.append(pd.DataFrame(arrvals))
 
     if has_domains and has_values:
-        df = pd.concat(dfs, axis=1, copy=False)
+        df = pd.concat(dfs, axis=1)
         df.columns = pd.RangeIndex(start=0, stop=len(df.columns))
     elif has_domains or has_values:
         df = dfs[0]