gamsapi 52.5.0__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gams/__init__.py +27 -0
- gams/_version.py +1 -0
- gams/connect/__init__.py +28 -0
- gams/connect/agents/__init__.py +24 -0
- gams/connect/agents/_excel/__init__.py +32 -0
- gams/connect/agents/_excel/excelagent.py +312 -0
- gams/connect/agents/_excel/workbook.py +155 -0
- gams/connect/agents/_sqlconnectors/__init__.py +42 -0
- gams/connect/agents/_sqlconnectors/_accesshandler.py +211 -0
- gams/connect/agents/_sqlconnectors/_databasehandler.py +250 -0
- gams/connect/agents/_sqlconnectors/_mysqlhandler.py +168 -0
- gams/connect/agents/_sqlconnectors/_postgreshandler.py +131 -0
- gams/connect/agents/_sqlconnectors/_pyodbchandler.py +112 -0
- gams/connect/agents/_sqlconnectors/_sqlalchemyhandler.py +74 -0
- gams/connect/agents/_sqlconnectors/_sqlitehandler.py +262 -0
- gams/connect/agents/_sqlconnectors/_sqlserverhandler.py +179 -0
- gams/connect/agents/concatenate.py +440 -0
- gams/connect/agents/connectagent.py +743 -0
- gams/connect/agents/csvreader.py +675 -0
- gams/connect/agents/csvwriter.py +151 -0
- gams/connect/agents/domainwriter.py +143 -0
- gams/connect/agents/excelreader.py +756 -0
- gams/connect/agents/excelwriter.py +467 -0
- gams/connect/agents/filter.py +223 -0
- gams/connect/agents/gamsreader.py +112 -0
- gams/connect/agents/gamswriter.py +239 -0
- gams/connect/agents/gdxreader.py +109 -0
- gams/connect/agents/gdxwriter.py +146 -0
- gams/connect/agents/labelmanipulator.py +303 -0
- gams/connect/agents/projection.py +539 -0
- gams/connect/agents/pythoncode.py +71 -0
- gams/connect/agents/rawcsvreader.py +248 -0
- gams/connect/agents/rawexcelreader.py +312 -0
- gams/connect/agents/schema/CSVReader.yaml +92 -0
- gams/connect/agents/schema/CSVWriter.yaml +44 -0
- gams/connect/agents/schema/Concatenate.yaml +52 -0
- gams/connect/agents/schema/DomainWriter.yaml +25 -0
- gams/connect/agents/schema/ExcelReader.yaml +121 -0
- gams/connect/agents/schema/ExcelWriter.yaml +78 -0
- gams/connect/agents/schema/Filter.yaml +74 -0
- gams/connect/agents/schema/GAMSReader.yaml +20 -0
- gams/connect/agents/schema/GAMSWriter.yaml +47 -0
- gams/connect/agents/schema/GDXReader.yaml +23 -0
- gams/connect/agents/schema/GDXWriter.yaml +32 -0
- gams/connect/agents/schema/LabelManipulator.yaml +99 -0
- gams/connect/agents/schema/Projection.yaml +24 -0
- gams/connect/agents/schema/PythonCode.yaml +6 -0
- gams/connect/agents/schema/RawCSVReader.yaml +34 -0
- gams/connect/agents/schema/RawExcelReader.yaml +42 -0
- gams/connect/agents/schema/SQLReader.yaml +75 -0
- gams/connect/agents/schema/SQLWriter.yaml +103 -0
- gams/connect/agents/sqlreader.py +301 -0
- gams/connect/agents/sqlwriter.py +276 -0
- gams/connect/connectdatabase.py +275 -0
- gams/connect/connectvalidator.py +93 -0
- gams/connect/errors.py +34 -0
- gams/control/__init__.py +136 -0
- gams/control/database.py +2231 -0
- gams/control/execution.py +1900 -0
- gams/control/options.py +2792 -0
- gams/control/workspace.py +1198 -0
- gams/core/__init__.py +24 -0
- gams/core/cfg/__init__.py +26 -0
- gams/core/cfg/_cfgmcc.cp312-win_amd64.pyd +0 -0
- gams/core/cfg/cfgmcc.py +519 -0
- gams/core/dct/__init__.py +26 -0
- gams/core/dct/_dctmcc.cp312-win_amd64.pyd +0 -0
- gams/core/dct/dctmcc.py +574 -0
- gams/core/embedded/__init__.py +26 -0
- gams/core/embedded/gamsemb.py +1024 -0
- gams/core/emp/__init__.py +24 -0
- gams/core/emp/emplexer.py +89 -0
- gams/core/emp/empyacc.py +281 -0
- gams/core/gdx/__init__.py +26 -0
- gams/core/gdx/_gdxcc.cp312-win_amd64.pyd +0 -0
- gams/core/gdx/gdxcc.py +866 -0
- gams/core/gev/__init__.py +26 -0
- gams/core/gev/_gevmcc.cp312-win_amd64.pyd +0 -0
- gams/core/gev/gevmcc.py +855 -0
- gams/core/gmd/__init__.py +26 -0
- gams/core/gmd/_gmdcc.cp312-win_amd64.pyd +0 -0
- gams/core/gmd/gmdcc.py +917 -0
- gams/core/gmo/__init__.py +26 -0
- gams/core/gmo/_gmomcc.cp312-win_amd64.pyd +0 -0
- gams/core/gmo/gmomcc.py +2046 -0
- gams/core/idx/__init__.py +26 -0
- gams/core/idx/_idxcc.cp312-win_amd64.pyd +0 -0
- gams/core/idx/idxcc.py +510 -0
- gams/core/numpy/__init__.py +29 -0
- gams/core/numpy/_gams2numpy.cp312-win_amd64.pyd +0 -0
- gams/core/numpy/gams2numpy.py +1048 -0
- gams/core/opt/__init__.py +26 -0
- gams/core/opt/_optcc.cp312-win_amd64.pyd +0 -0
- gams/core/opt/optcc.py +840 -0
- gams/engine/__init__.py +204 -0
- gams/engine/api/__init__.py +13 -0
- gams/engine/api/auth_api.py +7653 -0
- gams/engine/api/cleanup_api.py +751 -0
- gams/engine/api/default_api.py +887 -0
- gams/engine/api/hypercube_api.py +2629 -0
- gams/engine/api/jobs_api.py +5229 -0
- gams/engine/api/licenses_api.py +2220 -0
- gams/engine/api/namespaces_api.py +7783 -0
- gams/engine/api/usage_api.py +5627 -0
- gams/engine/api/users_api.py +5931 -0
- gams/engine/api_client.py +804 -0
- gams/engine/api_response.py +21 -0
- gams/engine/configuration.py +601 -0
- gams/engine/exceptions.py +216 -0
- gams/engine/models/__init__.py +86 -0
- gams/engine/models/bad_input.py +89 -0
- gams/engine/models/cleanable_job_result.py +104 -0
- gams/engine/models/cleanable_job_result_page.py +113 -0
- gams/engine/models/engine_license.py +107 -0
- gams/engine/models/files_not_found.py +93 -0
- gams/engine/models/forwarded_token_response.py +112 -0
- gams/engine/models/generic_key_value_pair.py +89 -0
- gams/engine/models/hypercube.py +160 -0
- gams/engine/models/hypercube_page.py +111 -0
- gams/engine/models/hypercube_summary.py +91 -0
- gams/engine/models/hypercube_token.py +97 -0
- gams/engine/models/identity_provider.py +107 -0
- gams/engine/models/identity_provider_ldap.py +121 -0
- gams/engine/models/identity_provider_oauth2.py +146 -0
- gams/engine/models/identity_provider_oauth2_scope.py +89 -0
- gams/engine/models/identity_provider_oauth2_with_secret.py +152 -0
- gams/engine/models/identity_provider_oidc.py +133 -0
- gams/engine/models/identity_provider_oidc_with_secret.py +143 -0
- gams/engine/models/inex.py +91 -0
- gams/engine/models/invitation.py +136 -0
- gams/engine/models/invitation_quota.py +106 -0
- gams/engine/models/invitation_token.py +87 -0
- gams/engine/models/job.py +165 -0
- gams/engine/models/job_no_text_entry.py +138 -0
- gams/engine/models/job_no_text_entry_page.py +111 -0
- gams/engine/models/license.py +91 -0
- gams/engine/models/log_piece.py +96 -0
- gams/engine/models/message.py +87 -0
- gams/engine/models/message_and_token.py +99 -0
- gams/engine/models/message_with_webhook_id.py +89 -0
- gams/engine/models/model_auth_token.py +87 -0
- gams/engine/models/model_configuration.py +125 -0
- gams/engine/models/model_default_instance.py +99 -0
- gams/engine/models/model_default_user_instance.py +98 -0
- gams/engine/models/model_hypercube_job.py +106 -0
- gams/engine/models/model_hypercube_usage.py +130 -0
- gams/engine/models/model_instance_info.py +116 -0
- gams/engine/models/model_instance_info_full.py +123 -0
- gams/engine/models/model_instance_pool_info.py +112 -0
- gams/engine/models/model_job_labels.py +179 -0
- gams/engine/models/model_job_usage.py +133 -0
- gams/engine/models/model_pool_usage.py +124 -0
- gams/engine/models/model_usage.py +115 -0
- gams/engine/models/model_user.py +96 -0
- gams/engine/models/model_userinstance_info.py +119 -0
- gams/engine/models/model_userinstancepool_info.py +95 -0
- gams/engine/models/model_version.py +91 -0
- gams/engine/models/models.py +120 -0
- gams/engine/models/namespace.py +104 -0
- gams/engine/models/namespace_quota.py +96 -0
- gams/engine/models/namespace_with_permission.py +96 -0
- gams/engine/models/not_found.py +91 -0
- gams/engine/models/password_policy.py +97 -0
- gams/engine/models/perm_and_username.py +89 -0
- gams/engine/models/quota.py +117 -0
- gams/engine/models/quota_exceeded.py +97 -0
- gams/engine/models/status_code_meaning.py +89 -0
- gams/engine/models/stream_entry.py +89 -0
- gams/engine/models/system_wide_license.py +92 -0
- gams/engine/models/text_entries.py +87 -0
- gams/engine/models/text_entry.py +101 -0
- gams/engine/models/time_span.py +95 -0
- gams/engine/models/time_span_pool_worker.py +99 -0
- gams/engine/models/token_forward_error.py +87 -0
- gams/engine/models/user.py +127 -0
- gams/engine/models/user_group_member.py +96 -0
- gams/engine/models/user_groups.py +108 -0
- gams/engine/models/vapid_info.py +87 -0
- gams/engine/models/webhook.py +138 -0
- gams/engine/models/webhook_parameterized_event.py +99 -0
- gams/engine/py.typed +0 -0
- gams/engine/rest.py +258 -0
- gams/magic/__init__.py +32 -0
- gams/magic/gams_magic.py +142 -0
- gams/magic/interactive.py +402 -0
- gams/tools/__init__.py +30 -0
- gams/tools/errors.py +34 -0
- gams/tools/toolcollection/__init__.py +24 -0
- gams/tools/toolcollection/alg/__init__.py +24 -0
- gams/tools/toolcollection/alg/rank.py +51 -0
- gams/tools/toolcollection/data/__init__.py +24 -0
- gams/tools/toolcollection/data/csvread.py +444 -0
- gams/tools/toolcollection/data/csvwrite.py +311 -0
- gams/tools/toolcollection/data/exceldump.py +47 -0
- gams/tools/toolcollection/data/sqlitewrite.py +276 -0
- gams/tools/toolcollection/gdxservice/__init__.py +24 -0
- gams/tools/toolcollection/gdxservice/gdxencoding.py +104 -0
- gams/tools/toolcollection/gdxservice/gdxrename.py +94 -0
- gams/tools/toolcollection/linalg/__init__.py +24 -0
- gams/tools/toolcollection/linalg/cholesky.py +57 -0
- gams/tools/toolcollection/linalg/eigenvalue.py +56 -0
- gams/tools/toolcollection/linalg/eigenvector.py +58 -0
- gams/tools/toolcollection/linalg/invert.py +55 -0
- gams/tools/toolcollection/linalg/ols.py +138 -0
- gams/tools/toolcollection/tooltemplate.py +321 -0
- gams/tools/toolcollection/win32/__init__.py +24 -0
- gams/tools/toolcollection/win32/excelmerge.py +93 -0
- gams/tools/toolcollection/win32/exceltalk.py +76 -0
- gams/tools/toolcollection/win32/msappavail.py +49 -0
- gams/tools/toolcollection/win32/shellexecute.py +54 -0
- gams/tools/tools.py +116 -0
- gams/transfer/__init__.py +35 -0
- gams/transfer/_abcs/__init__.py +37 -0
- gams/transfer/_abcs/container_abcs.py +433 -0
- gams/transfer/_internals/__init__.py +63 -0
- gams/transfer/_internals/algorithms.py +436 -0
- gams/transfer/_internals/casepreservingdict.py +124 -0
- gams/transfer/_internals/constants.py +270 -0
- gams/transfer/_internals/domainviolation.py +103 -0
- gams/transfer/_internals/specialvalues.py +172 -0
- gams/transfer/containers/__init__.py +26 -0
- gams/transfer/containers/_container.py +1794 -0
- gams/transfer/containers/_io/__init__.py +28 -0
- gams/transfer/containers/_io/containers.py +164 -0
- gams/transfer/containers/_io/gdx.py +1029 -0
- gams/transfer/containers/_io/gmd.py +872 -0
- gams/transfer/containers/_mixins/__init__.py +26 -0
- gams/transfer/containers/_mixins/ccc.py +1274 -0
- gams/transfer/syms/__init__.py +33 -0
- gams/transfer/syms/_methods/__init__.py +24 -0
- gams/transfer/syms/_methods/tables.py +120 -0
- gams/transfer/syms/_methods/toDict.py +115 -0
- gams/transfer/syms/_methods/toList.py +83 -0
- gams/transfer/syms/_methods/toValue.py +60 -0
- gams/transfer/syms/_mixins/__init__.py +32 -0
- gams/transfer/syms/_mixins/equals.py +626 -0
- gams/transfer/syms/_mixins/generateRecords.py +499 -0
- gams/transfer/syms/_mixins/pivot.py +313 -0
- gams/transfer/syms/_mixins/pve.py +627 -0
- gams/transfer/syms/_mixins/sa.py +27 -0
- gams/transfer/syms/_mixins/sapve.py +27 -0
- gams/transfer/syms/_mixins/saua.py +27 -0
- gams/transfer/syms/_mixins/sauapve.py +199 -0
- gams/transfer/syms/_mixins/spve.py +1528 -0
- gams/transfer/syms/_mixins/ve.py +936 -0
- gams/transfer/syms/container_syms/__init__.py +31 -0
- gams/transfer/syms/container_syms/_alias.py +984 -0
- gams/transfer/syms/container_syms/_equation.py +333 -0
- gams/transfer/syms/container_syms/_parameter.py +973 -0
- gams/transfer/syms/container_syms/_set.py +604 -0
- gams/transfer/syms/container_syms/_universe_alias.py +461 -0
- gams/transfer/syms/container_syms/_variable.py +321 -0
- gamsapi-52.5.0.dist-info/METADATA +150 -0
- gamsapi-52.5.0.dist-info/RECORD +257 -0
- gamsapi-52.5.0.dist-info/WHEEL +5 -0
- gamsapi-52.5.0.dist-info/licenses/LICENSE +22 -0
- gamsapi-52.5.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
#
|
|
2
|
+
# GAMS - General Algebraic Modeling System Python API
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2017-2026 GAMS Development Corp. <support@gams.com>
|
|
5
|
+
# Copyright (c) 2017-2026 GAMS Software GmbH <support@gams.com>
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
8
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
9
|
+
# in the Software without restriction, including without limitation the rights
|
|
10
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
12
|
+
# furnished to do so, subject to the following conditions:
|
|
13
|
+
#
|
|
14
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
15
|
+
# copies or substantial portions of the Software.
|
|
16
|
+
#
|
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
19
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
20
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
21
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
22
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
23
|
+
# SOFTWARE.
|
|
24
|
+
#
|
|
25
|
+
|
|
26
|
+
import pandas as pd
|
|
27
|
+
from gams.connect.agents._sqlconnectors._databasehandler import DatabaseConnector
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AccessConnector(DatabaseConnector):
|
|
31
|
+
SUPPORTED_INSERT_METHODS = ["default", "bulkInsert"]
|
|
32
|
+
QUOTE_CHAR = ["[]", '""', "``"]
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def _create_accdb(dbpath):
|
|
36
|
+
"""
|
|
37
|
+
Creates an MS-Access (.accdb) file/database at provided dbpath.
|
|
38
|
+
"""
|
|
39
|
+
import win32com.client as win32
|
|
40
|
+
|
|
41
|
+
Access = win32.Dispatch("Access.Application")
|
|
42
|
+
Access.NewCurrentDataBase(dbpath)
|
|
43
|
+
Access.CloseCurrentDataBase()
|
|
44
|
+
Access.Quit() # required in order to remove access application from python memory
|
|
45
|
+
del Access
|
|
46
|
+
|
|
47
|
+
def connect(self, connection_details, connection_args, **kwargs) -> None:
|
|
48
|
+
import pyodbc as sql
|
|
49
|
+
|
|
50
|
+
isWrite: bool = kwargs.get("isWrite", True)
|
|
51
|
+
|
|
52
|
+
if isWrite:
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
|
|
55
|
+
if not Path(
|
|
56
|
+
connection_details["DBQ"]
|
|
57
|
+
).is_file(): # if .accdb file does not exist at the provided loc in DBQ, then create a new .accdb file
|
|
58
|
+
self._create_accdb(dbpath=connection_details["DBQ"])
|
|
59
|
+
if self._traceValue > 1:
|
|
60
|
+
self._traceLog(
|
|
61
|
+
f'Created a new .accdb file: >{connection_details["DBQ"]}<'
|
|
62
|
+
)
|
|
63
|
+
self._engine = sql.connect(**connection_details, **connection_args)
|
|
64
|
+
self._conn = self._engine.cursor()
|
|
65
|
+
|
|
66
|
+
def create_transaction(self):
|
|
67
|
+
"""AccessConnector utilizes PyODBC to connect to the database file.
|
|
68
|
+
Therefore, autocommit can be enabled via the connection dictionary."""
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
def _check_table_exists(self, tableName: str, schema: str | None) -> bool:
|
|
72
|
+
tableExists = False
|
|
73
|
+
rawTableName = self._strip_escape_chars(
|
|
74
|
+
tableName=tableName, quote_chars=self.QUOTE_CHAR
|
|
75
|
+
)
|
|
76
|
+
for ele in self._conn.tables().fetchall():
|
|
77
|
+
if ele[2].lower() == rawTableName.lower():
|
|
78
|
+
tableExists = True
|
|
79
|
+
|
|
80
|
+
return tableExists
|
|
81
|
+
|
|
82
|
+
def _create_table(
|
|
83
|
+
self,
|
|
84
|
+
df: pd.DataFrame,
|
|
85
|
+
tableName: str,
|
|
86
|
+
schema: str | None,
|
|
87
|
+
ifExists: str,
|
|
88
|
+
**kwargs,
|
|
89
|
+
) -> None:
|
|
90
|
+
"""
|
|
91
|
+
Drops an exisiting table and creates a new table with the same name. Uses specific SQL queries for each DBMS flavour.
|
|
92
|
+
"""
|
|
93
|
+
tableCols = ""
|
|
94
|
+
for col, dtype in df.dtypes.items():
|
|
95
|
+
if dtype == "float64":
|
|
96
|
+
tableCols += f"[{col}] FLOAT,"
|
|
97
|
+
elif dtype == "int64":
|
|
98
|
+
tableCols += f"[{col}] BIGINT,"
|
|
99
|
+
elif dtype in ["object", "category"]:
|
|
100
|
+
tableCols += f"[{col}] VARCHAR(255),"
|
|
101
|
+
|
|
102
|
+
tableCols = tableCols[:-1]
|
|
103
|
+
|
|
104
|
+
if schema:
|
|
105
|
+
tableName = schema + "." + tableName
|
|
106
|
+
|
|
107
|
+
if ifExists == "replace":
|
|
108
|
+
try:
|
|
109
|
+
if self._check_table_exists(tableName, schema=None):
|
|
110
|
+
self._conn.execute(f"""DROP TABLE {tableName};""")
|
|
111
|
+
except Exception as e:
|
|
112
|
+
self._raise_error(
|
|
113
|
+
f"Cannot drop table >{tableName}<.\nException from {type(e).__module__}: {type(e).__name__}> {e}"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
self._conn.execute(f"""CREATE TABLE {tableName}({tableCols});""")
|
|
117
|
+
if self._traceValue > 1:
|
|
118
|
+
self._traceLog(
|
|
119
|
+
f"Created new table: >{tableName}< with columns: >{tableCols}<"
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def _execute_write(self, df: pd.DataFrame, writeFunction_args: dict):
|
|
123
|
+
insertMethod = writeFunction_args["insertMethod"]
|
|
124
|
+
|
|
125
|
+
if insertMethod == "default":
|
|
126
|
+
return super()._execute_write(df, writeFunction_args)
|
|
127
|
+
elif insertMethod == "bulkInsert":
|
|
128
|
+
self._write_file_to_access(
|
|
129
|
+
df=df,
|
|
130
|
+
tableName=writeFunction_args["name"],
|
|
131
|
+
ifExists=writeFunction_args["if_exists"],
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
def _insert_data(self, df: pd.DataFrame, writeFunction_args: dict):
|
|
135
|
+
tableName = writeFunction_args["name"]
|
|
136
|
+
if writeFunction_args["schema"]:
|
|
137
|
+
tableName = writeFunction_args["schema"] + "." + tableName
|
|
138
|
+
|
|
139
|
+
placeHolder = "?," * (len(df.columns) - 1)
|
|
140
|
+
if df.isnull().values.any(): # replace NaN with None, for SQL NULL
|
|
141
|
+
df = df.astype(object).where(pd.notnull(df), None)
|
|
142
|
+
df_list = list(
|
|
143
|
+
df.itertuples(index=False, name=None)
|
|
144
|
+
) # sql server does not accept nested lists, it has to be tuples
|
|
145
|
+
|
|
146
|
+
query = f"INSERT INTO {tableName} VALUES(" + placeHolder + "?)"
|
|
147
|
+
if len(df_list) > 0:
|
|
148
|
+
self._conn.executemany(query, df_list)
|
|
149
|
+
|
|
150
|
+
elif self._traceValue > 1:
|
|
151
|
+
self._traceLog(
|
|
152
|
+
f"Empty symbol. No rows were inserted in table >{tableName}<."
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def _write_file_to_access(self, df: pd.DataFrame, tableName: str, ifExists: str):
|
|
156
|
+
"""
|
|
157
|
+
Uses MS-Access' make-table query to create a new table from csv.
|
|
158
|
+
This does not require the table to be present in the database file.
|
|
159
|
+
Thus, `ifExists` behavior changes accordingly.
|
|
160
|
+
"""
|
|
161
|
+
import tempfile
|
|
162
|
+
|
|
163
|
+
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
164
|
+
with tempfile.NamedTemporaryFile(
|
|
165
|
+
mode="w", dir=tmpdirname, delete=False, suffix=".csv"
|
|
166
|
+
) as fp:
|
|
167
|
+
df.to_csv(fp.name, index=False)
|
|
168
|
+
fp.flush()
|
|
169
|
+
fp.seek(0)
|
|
170
|
+
fp.close()
|
|
171
|
+
filename = fp.name.split("\\")[-1]
|
|
172
|
+
if ifExists == "replace":
|
|
173
|
+
try:
|
|
174
|
+
# DROP TABLE IF EXISTS ELSE PASS
|
|
175
|
+
self._conn.execute(f"""DROP TABLE {tableName};""")
|
|
176
|
+
except:
|
|
177
|
+
pass
|
|
178
|
+
self._conn.execute(
|
|
179
|
+
f"SELECT * INTO [{tableName}] FROM [text;HDR=Yes;FMT=Delimited(,);"
|
|
180
|
+
+ f"Database={tmpdirname}].{filename};"
|
|
181
|
+
)
|
|
182
|
+
elif (
|
|
183
|
+
ifExists == "append"
|
|
184
|
+
): # creates a temp table `randomTemp_<tableName>` in the same db file, inserts the result of newly created temp table into the existing table
|
|
185
|
+
if self._check_table_exists(tableName, schema=None):
|
|
186
|
+
self._conn.execute(
|
|
187
|
+
f"SELECT * INTO [randomTemp_{tableName}] FROM [text;HDR=Yes;FMT=Delimited(,);"
|
|
188
|
+
+ f"Database={tmpdirname}].{filename};"
|
|
189
|
+
)
|
|
190
|
+
self._conn.execute(
|
|
191
|
+
f"INSERT INTO {tableName} SELECT * FROM [randomTemp_{tableName}];"
|
|
192
|
+
)
|
|
193
|
+
self._conn.execute(f"DROP TABLE [randomTemp_{tableName}];")
|
|
194
|
+
else:
|
|
195
|
+
self._raise_error(
|
|
196
|
+
f"Table >{tableName}< does not exists and ifExists is set to `append`."
|
|
197
|
+
)
|
|
198
|
+
elif ifExists == "fail":
|
|
199
|
+
if not self._check_table_exists(tableName, schema=None):
|
|
200
|
+
self._conn.execute(
|
|
201
|
+
f"SELECT * INTO [{tableName}] FROM [text;HDR=Yes;FMT=Delimited(,);"
|
|
202
|
+
+ f"Database={tmpdirname}].{filename};"
|
|
203
|
+
)
|
|
204
|
+
else:
|
|
205
|
+
self._raise_error(
|
|
206
|
+
f"Table >{tableName}< already exist and ifExists is set to `fail`."
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
def close(self):
|
|
210
|
+
self._conn.close()
|
|
211
|
+
self._engine.close()
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
#
|
|
2
|
+
# GAMS - General Algebraic Modeling System Python API
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2017-2026 GAMS Development Corp. <support@gams.com>
|
|
5
|
+
# Copyright (c) 2017-2026 GAMS Software GmbH <support@gams.com>
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
8
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
9
|
+
# in the Software without restriction, including without limitation the rights
|
|
10
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
12
|
+
# furnished to do so, subject to the following conditions:
|
|
13
|
+
#
|
|
14
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
15
|
+
# copies or substantial portions of the Software.
|
|
16
|
+
#
|
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
19
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
20
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
21
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
22
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
23
|
+
# SOFTWARE.
|
|
24
|
+
#
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from typing import TYPE_CHECKING, Callable
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from gams import transfer as gt
|
|
32
|
+
|
|
33
|
+
from abc import ABC, abstractmethod
|
|
34
|
+
from enum import Enum
|
|
35
|
+
|
|
36
|
+
import pandas as pd
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ConnectionType(Enum):
|
|
40
|
+
SQLITE = "sqlite"
|
|
41
|
+
PYODBC = "pyodbc"
|
|
42
|
+
SQLALCHEMY = "sqlalchemy"
|
|
43
|
+
POSTGRES = "postgres"
|
|
44
|
+
MYSQL = "mysql"
|
|
45
|
+
SQLSERVER = "sqlserver"
|
|
46
|
+
ACCESS = "access"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class DatabaseConnector(ABC):
|
|
50
|
+
SUPPORTED_INSERT_METHODS = []
|
|
51
|
+
QUOTE_CHAR = []
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
error_callback: Callable[[str], None],
|
|
56
|
+
printLog_callback: Callable[[str], None],
|
|
57
|
+
trace: int = 0,
|
|
58
|
+
):
|
|
59
|
+
self._raise_error: Callable[[str], None] = error_callback
|
|
60
|
+
self._traceLog: Callable[[str], None] = printLog_callback
|
|
61
|
+
self._traceValue = trace
|
|
62
|
+
self._engine = None
|
|
63
|
+
self._conn = None
|
|
64
|
+
|
|
65
|
+
def validate_insert_method(self, method: str):
|
|
66
|
+
"""Checks if the insert method is supported by this handler."""
|
|
67
|
+
if method not in self.SUPPORTED_INSERT_METHODS:
|
|
68
|
+
self._raise_error(
|
|
69
|
+
f"insertMethod >{method}< is not valid for this connection type. "
|
|
70
|
+
f"Valid methods are >{self.SUPPORTED_INSERT_METHODS}<"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def read_table(self, sql_query: str, read_sql_args: dict) -> pd.DataFrame:
|
|
74
|
+
"""
|
|
75
|
+
Read data from select DBMS using the provided SQL Query. Returns a pandas.DataFrame
|
|
76
|
+
|
|
77
|
+
Note: All except SQLAlchemy use the same method
|
|
78
|
+
"""
|
|
79
|
+
if len(read_sql_args) > 0:
|
|
80
|
+
self._conn.execute(sql_query, read_sql_args) # type: ignore
|
|
81
|
+
else:
|
|
82
|
+
self._conn.execute(sql_query) # type: ignore
|
|
83
|
+
|
|
84
|
+
return pd.DataFrame.from_records(
|
|
85
|
+
self._conn.fetchall(), # type: ignore
|
|
86
|
+
columns=[col[0] for col in self._conn.description], # type: ignore
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def pre_write_procedures(self, **kwargs) -> gt.Container:
|
|
90
|
+
assert (
|
|
91
|
+
"container" in kwargs
|
|
92
|
+
), ">pre_write_procedures< args must include the container."
|
|
93
|
+
return kwargs["container"]
|
|
94
|
+
|
|
95
|
+
def post_write_procedures(self, **kwargs):
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
def write_dataframe(self, df: pd.DataFrame, writeFunction_args: dict):
|
|
99
|
+
"""Delegates the write operation."""
|
|
100
|
+
if self._traceValue > 2:
|
|
101
|
+
self._traceLog(f"DataFrame before writing:\n{df}")
|
|
102
|
+
if self._traceValue > 1:
|
|
103
|
+
self._traceLog(f"writeFunction_args: >{writeFunction_args}<")
|
|
104
|
+
|
|
105
|
+
self._execute_write(df, writeFunction_args)
|
|
106
|
+
|
|
107
|
+
def _execute_write(self, df: pd.DataFrame, writeFunction_args: dict):
|
|
108
|
+
"""
|
|
109
|
+
Main function to process the incoming write request.
|
|
110
|
+
Since `SQLAlchemy` and `Access` handle >ifExists< differently,
|
|
111
|
+
this method gets re-implemented in their respective subclasses.
|
|
112
|
+
|
|
113
|
+
This method covers the following DBMS: ["PyODBC", "SQLite", "MySQL", "Postgres", "SQLServer"]
|
|
114
|
+
"""
|
|
115
|
+
tableName = writeFunction_args["name"]
|
|
116
|
+
schema = writeFunction_args["schema"]
|
|
117
|
+
pyodbc_options = {
|
|
118
|
+
"dtype_map": writeFunction_args["dtype_map"],
|
|
119
|
+
"columnEncloser": writeFunction_args["columnEncloser"],
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if writeFunction_args["if_exists"] == "replace":
|
|
123
|
+
self._create_table(
|
|
124
|
+
df=df,
|
|
125
|
+
tableName=tableName,
|
|
126
|
+
schema=schema,
|
|
127
|
+
ifExists="replace",
|
|
128
|
+
**pyodbc_options,
|
|
129
|
+
)
|
|
130
|
+
self._insert_data(df, writeFunction_args)
|
|
131
|
+
|
|
132
|
+
elif writeFunction_args["if_exists"] == "append":
|
|
133
|
+
if self._check_table_exists(tableName=tableName, schema=schema):
|
|
134
|
+
self._insert_data(df, writeFunction_args)
|
|
135
|
+
else:
|
|
136
|
+
self._raise_error(
|
|
137
|
+
f"Table >{tableName}< does not exist in the database and ifExists is set to >append<."
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
elif writeFunction_args["if_exists"] == "fail":
|
|
141
|
+
if not self._check_table_exists(tableName=tableName, schema=schema):
|
|
142
|
+
self._create_table(
|
|
143
|
+
df=df,
|
|
144
|
+
tableName=tableName,
|
|
145
|
+
schema=schema,
|
|
146
|
+
ifExists="fail",
|
|
147
|
+
**pyodbc_options,
|
|
148
|
+
)
|
|
149
|
+
self._insert_data(df, writeFunction_args)
|
|
150
|
+
else:
|
|
151
|
+
self._raise_error(
|
|
152
|
+
f"Table >{tableName}< already exists in the database and ifExists is set to >fail<."
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def rollback(self):
|
|
156
|
+
"""
|
|
157
|
+
Method to rollback changes made to the database.
|
|
158
|
+
Common for all but sqlalchemy."""
|
|
159
|
+
self._engine.rollback() # type: ignore
|
|
160
|
+
|
|
161
|
+
def close(self):
|
|
162
|
+
"""
|
|
163
|
+
Closes the connection to the database.
|
|
164
|
+
Common for all but sqlalchemy."""
|
|
165
|
+
self._conn.close() # type: ignore
|
|
166
|
+
|
|
167
|
+
def commit(self):
|
|
168
|
+
"""
|
|
169
|
+
Commit all the changes made to the database.
|
|
170
|
+
Common for all but sqlalchemy."""
|
|
171
|
+
self._engine.commit() # type: ignore
|
|
172
|
+
|
|
173
|
+
@abstractmethod
|
|
174
|
+
def connect(self, connection_details: dict, connection_args: dict, **kwargs):
|
|
175
|
+
"""
|
|
176
|
+
Method to connect to each DBMS with its own library.
|
|
177
|
+
Sets the _engine and _conn attribute.
|
|
178
|
+
"""
|
|
179
|
+
pass
|
|
180
|
+
|
|
181
|
+
@abstractmethod
|
|
182
|
+
def create_transaction(self):
|
|
183
|
+
"""
|
|
184
|
+
Method to start a SQL transaction.
|
|
185
|
+
This helps in rolling back the changes made within a transaction in the event of a failure.
|
|
186
|
+
This is useful when we commit globally once instead of committing each symbol.
|
|
187
|
+
Apart from ["SQLAlchemy", "SQLITE", "Postgres"] other connection start a transaction implicitly.
|
|
188
|
+
|
|
189
|
+
connectionType:pythonLibrary
|
|
190
|
+
postgres:pyscopg2, mysql:pymysql, sqlserver:pymssql and pyodbc:pyodbc have autocommit = False by default
|
|
191
|
+
|
|
192
|
+
mysql:pymysql
|
|
193
|
+
DDL is autocommitted and as a result all the changes made till then also get committed.
|
|
194
|
+
|
|
195
|
+
sqlserver:pymssql,postgres:pyscopg2
|
|
196
|
+
postgres and sqlserver are transaction safe.
|
|
197
|
+
Any change within a transaction does not get committed in case of a failure.
|
|
198
|
+
|
|
199
|
+
sqlite:sqlite3
|
|
200
|
+
Creates a blank table for the first symbol. Setting the transaction to begin resolves it.
|
|
201
|
+
This also makes sqlite's behavior different when autocommit is True.
|
|
202
|
+
It does not commit anything even when autocommit is True and failure occurs.
|
|
203
|
+
|
|
204
|
+
sqlalchemy:sqlalchemy
|
|
205
|
+
follows the default behavior of specific database
|
|
206
|
+
"""
|
|
207
|
+
pass
|
|
208
|
+
|
|
209
|
+
@abstractmethod
|
|
210
|
+
def _create_table(
|
|
211
|
+
self,
|
|
212
|
+
df: pd.DataFrame,
|
|
213
|
+
tableName: str,
|
|
214
|
+
schema: str | None,
|
|
215
|
+
ifExists: str,
|
|
216
|
+
**kwargs,
|
|
217
|
+
) -> None:
|
|
218
|
+
"""
|
|
219
|
+
Drops an exisiting table and creates a new table with the same name. Uses specific SQL queries for each DBMS flavour.
|
|
220
|
+
"""
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
@abstractmethod
|
|
224
|
+
def _check_table_exists(self, tableName: str, schema: str | None) -> bool:
|
|
225
|
+
pass
|
|
226
|
+
|
|
227
|
+
@abstractmethod
|
|
228
|
+
def _insert_data(self, df: pd.DataFrame, writeFunction_args: dict) -> None:
|
|
229
|
+
"""Each Database perform their own insert operation"""
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
@staticmethod
|
|
233
|
+
def _strip_escape_chars(tableName: str, quote_chars: list[str]) -> str:
|
|
234
|
+
"""
|
|
235
|
+
Helper function to strip an enclosed `tableName`.
|
|
236
|
+
For example, _strip_escape_chars("sqlite", "[new_table]") ==> new_table
|
|
237
|
+
|
|
238
|
+
["mysql","sqlite","access","pyodbc"]:
|
|
239
|
+
These >db_types< are sensitive to the use of escape characters in their SQL queries.
|
|
240
|
+
That is, the query would not find "[new_table]" even if "new_table" exist in the database.
|
|
241
|
+
|
|
242
|
+
["postgres","sqlserver"]:
|
|
243
|
+
These are insensitive to the use of escape characters in the table name.
|
|
244
|
+
The SQL query for both the DBs handle enclosed tableNames efficiently.
|
|
245
|
+
"""
|
|
246
|
+
for esc in quote_chars:
|
|
247
|
+
if tableName.startswith(esc[0]) and tableName.endswith(esc[-1]):
|
|
248
|
+
return tableName[1:-1]
|
|
249
|
+
|
|
250
|
+
return tableName
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
#
|
|
2
|
+
# GAMS - General Algebraic Modeling System Python API
|
|
3
|
+
#
|
|
4
|
+
# Copyright (c) 2017-2026 GAMS Development Corp. <support@gams.com>
|
|
5
|
+
# Copyright (c) 2017-2026 GAMS Software GmbH <support@gams.com>
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
8
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
9
|
+
# in the Software without restriction, including without limitation the rights
|
|
10
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
12
|
+
# furnished to do so, subject to the following conditions:
|
|
13
|
+
#
|
|
14
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
15
|
+
# copies or substantial portions of the Software.
|
|
16
|
+
#
|
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
19
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
20
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
21
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
22
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
23
|
+
# SOFTWARE.
|
|
24
|
+
#
|
|
25
|
+
|
|
26
|
+
import pandas as pd
|
|
27
|
+
from gams.connect.agents._sqlconnectors._databasehandler import DatabaseConnector
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class MySQLConnector(DatabaseConnector):
|
|
31
|
+
SUPPORTED_INSERT_METHODS = ["default", "bulkInsert"]
|
|
32
|
+
QUOTE_CHAR = ["``"]
|
|
33
|
+
|
|
34
|
+
def connect(self, connection_details, connection_args, **kwargs) -> None:
|
|
35
|
+
import pymysql as sql
|
|
36
|
+
|
|
37
|
+
isWrite: bool = kwargs.get("isWrite", True)
|
|
38
|
+
|
|
39
|
+
if isWrite:
|
|
40
|
+
connection_args.update(
|
|
41
|
+
{"local_infile": True}
|
|
42
|
+
) # set local_infile to true for bulkInsert symbol option when writing
|
|
43
|
+
|
|
44
|
+
self._engine = sql.connect(**connection_details, **connection_args)
|
|
45
|
+
self._conn = self._engine.cursor()
|
|
46
|
+
|
|
47
|
+
def create_transaction(self):
|
|
48
|
+
"""DDL is autocommitted and as a result all the changes made till then also get committed"""
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
def _check_table_exists(self, tableName: str, schema: str | None) -> bool:
|
|
52
|
+
tableExists = False
|
|
53
|
+
rawTableName = self._strip_escape_chars(
|
|
54
|
+
tableName=tableName, quote_chars=self.QUOTE_CHAR
|
|
55
|
+
)
|
|
56
|
+
query = f"""SELECT table_name FROM information_schema.tables WHERE table_name = '{rawTableName}'"""
|
|
57
|
+
query += f"""AND table_schema = '{schema}';""" if schema else ";"
|
|
58
|
+
|
|
59
|
+
self._conn.execute(query)
|
|
60
|
+
|
|
61
|
+
res = self._conn.fetchone()
|
|
62
|
+
|
|
63
|
+
# TODO: check type(res) for mysql
|
|
64
|
+
### res can be = (obj,) | None | (None,)
|
|
65
|
+
if isinstance(res, tuple):
|
|
66
|
+
if res[0]:
|
|
67
|
+
tableExists = True
|
|
68
|
+
|
|
69
|
+
return tableExists
|
|
70
|
+
|
|
71
|
+
def _create_table(
|
|
72
|
+
self,
|
|
73
|
+
df: pd.DataFrame,
|
|
74
|
+
tableName: str,
|
|
75
|
+
schema: str | None,
|
|
76
|
+
ifExists: str,
|
|
77
|
+
**kwargs,
|
|
78
|
+
) -> None:
|
|
79
|
+
"""
|
|
80
|
+
Drops an exisiting table and creates a new table with the same name. Uses specific SQL queries for each DBMS flavour.
|
|
81
|
+
"""
|
|
82
|
+
tableCols = ""
|
|
83
|
+
for col, dtype in df.dtypes.items():
|
|
84
|
+
if dtype == "float64":
|
|
85
|
+
tableCols += f"`{col}` DOUBLE,"
|
|
86
|
+
elif dtype == "int64":
|
|
87
|
+
tableCols += f"`{col}` BIGINT,"
|
|
88
|
+
elif dtype in ["object", "category"]:
|
|
89
|
+
tableCols += f"`{col}` TEXT,"
|
|
90
|
+
|
|
91
|
+
tableCols = tableCols[:-1]
|
|
92
|
+
|
|
93
|
+
if schema:
|
|
94
|
+
tableName = schema + "." + tableName
|
|
95
|
+
|
|
96
|
+
if ifExists == "replace":
|
|
97
|
+
try:
|
|
98
|
+
self._conn.execute(f"""DROP TABLE IF EXISTS {tableName};""")
|
|
99
|
+
except Exception as e:
|
|
100
|
+
self._raise_error(
|
|
101
|
+
f"Cannot drop table >{tableName}<.\nException from {type(e).__module__}: {type(e).__name__}> {e}"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
self._conn.execute(f"""CREATE TABLE {tableName}({tableCols});""")
|
|
105
|
+
|
|
106
|
+
if self._traceValue > 1:
|
|
107
|
+
self._traceLog(
|
|
108
|
+
f"Created new table: >{tableName}< with columns: >{tableCols}<"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
def _write_file_to_mysql(self, df: pd.DataFrame, tableName: str):
|
|
112
|
+
"""
|
|
113
|
+
Function to import data from file to MySQL DBMS
|
|
114
|
+
Uses `LOAD DATA LOCAL INFILE` query to import csv, provided the infile option (OPT_LOCAL_INFILE = 1) is enabled in the DBMS
|
|
115
|
+
"""
|
|
116
|
+
import tempfile
|
|
117
|
+
|
|
118
|
+
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
119
|
+
with tempfile.NamedTemporaryFile(
|
|
120
|
+
mode="w", dir=tmpdirname, delete=False, suffix=".csv"
|
|
121
|
+
) as fp:
|
|
122
|
+
df.to_csv(fp.name, index=False, header=False)
|
|
123
|
+
fp.flush()
|
|
124
|
+
fp.seek(0)
|
|
125
|
+
fp.close()
|
|
126
|
+
|
|
127
|
+
import sys
|
|
128
|
+
|
|
129
|
+
filepath = fp.name.replace("\\", "/")
|
|
130
|
+
setVals = ", ".join(["@" + str(i + 1) for i in range(len(df.columns))])
|
|
131
|
+
linending = "\r\n" if sys.platform == "win32" else "\n"
|
|
132
|
+
query = f"""LOAD DATA LOCAL INFILE "{filepath}" INTO TABLE {tableName}
|
|
133
|
+
FIELDS TERMINATED BY ','
|
|
134
|
+
ENCLOSED BY '"'
|
|
135
|
+
LINES TERMINATED BY '{linending}'
|
|
136
|
+
({setVals})"""
|
|
137
|
+
set_variables = ", ".join(
|
|
138
|
+
[
|
|
139
|
+
"`{0}`=NULLIF(@{1},'')".format(col, i + 1)
|
|
140
|
+
for i, col in enumerate(df.columns)
|
|
141
|
+
]
|
|
142
|
+
)
|
|
143
|
+
query += "SET " + set_variables + ";"
|
|
144
|
+
self._conn.execute(query)
|
|
145
|
+
|
|
146
|
+
def _insert_data(self, df: pd.DataFrame, writeFunction_args: dict):
|
|
147
|
+
tableName = writeFunction_args["name"]
|
|
148
|
+
if writeFunction_args["schema"]:
|
|
149
|
+
tableName = writeFunction_args["schema"] + "." + tableName
|
|
150
|
+
|
|
151
|
+
if writeFunction_args["insertMethod"] == "default":
|
|
152
|
+
placeHolder = "%s," * (len(df.columns) - 1)
|
|
153
|
+
if df.isnull().values.any(): # replace NaN with None, for SQL NULL
|
|
154
|
+
df = df.astype(object).where(pd.notnull(df), None)
|
|
155
|
+
df_list = list(
|
|
156
|
+
df.itertuples(index=False, name=None)
|
|
157
|
+
) # sql server does not accept nested lists, it has to be tuples
|
|
158
|
+
query = f"INSERT INTO {tableName} VALUES(" + placeHolder + "%s)"
|
|
159
|
+
|
|
160
|
+
if len(df_list) > 0:
|
|
161
|
+
self._conn.executemany(query, df_list)
|
|
162
|
+
|
|
163
|
+
elif self._traceValue > 1:
|
|
164
|
+
self._traceLog(
|
|
165
|
+
f"Empty symbol. No rows were inserted in table >{tableName}<."
|
|
166
|
+
)
|
|
167
|
+
elif writeFunction_args["insertMethod"] == "bulkInsert":
|
|
168
|
+
self._write_file_to_mysql(df=df, tableName=tableName)
|