goe-framework 1.0.8.dev0__tar.gz → 1.1.0__tar.gz

{goe_framework-1.0.8.dev0/src/goe_framework.egg-info → goe_framework-1.1.0}/PKG-INFO

@@ -1,17 +1,17 @@
 Metadata-Version: 2.4
 Name: goe-framework
-Version: 1.0.8.dev0
+Version: 1.1.0
 Maintainer-email: Neil Johnson <neiljohnson@google.com>
 Project-URL: Homepage, https://github.com/gluent/goe
 Project-URL: Issues, https://github.com/gluent/goe/issues
 Classifier: Development Status :: 4 - Beta
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS
@@ -33,7 +33,7 @@ Requires-Dist: google-cloud-storage
 Requires-Dist: boto3
 Requires-Dist: azure-common
 Requires-Dist: azure-storage-blob
-Requires-Dist: cx-Oracle
+Requires-Dist: oracledb
 Requires-Dist: google-cloud-bigquery
 Requires-Dist: google-cloud-kms
 Requires-Dist: python-dotenv
@@ -124,12 +124,12 @@ Variables you will want to pay attention to are:
 - OFFLOAD_FS_CONTAINER
 - OFFLOAD_FS_PREFIX
 
-If using Dataproc to provide Spark:
+If using Google Managed Service for Apache Spark:
 - GOOGLE_DATAPROC_CLUSTER
 - GOOGLE_DATAPROC_SERVICE_ACCOUNT
 - GOOGLE_DATAPROC_REGION
 
-If using Dataproc Batches to provide Spark:
+If using Google Managed Service for Apache Spark serverless:
 - GOOGLE_DATAPROC_BATCHES_VERSION
 - GOOGLE_DATAPROC_SERVICE_ACCOUNT
 - GOOGLE_DATAPROC_REGION

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/README.md

@@ -53,12 +53,12 @@ Variables you will want to pay attention to are:
 - OFFLOAD_FS_CONTAINER
 - OFFLOAD_FS_PREFIX
 
-If using Dataproc to provide Spark:
+If using Google Managed Service for Apache Spark:
 - GOOGLE_DATAPROC_CLUSTER
 - GOOGLE_DATAPROC_SERVICE_ACCOUNT
 - GOOGLE_DATAPROC_REGION
 
-If using Dataproc Batches to provide Spark:
+If using Google Managed Service for Apache Spark serverless:
 - GOOGLE_DATAPROC_BATCHES_VERSION
 - GOOGLE_DATAPROC_SERVICE_ACCOUNT
 - GOOGLE_DATAPROC_REGION

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/pyproject.toml

@@ -14,19 +14,19 @@
 
 [project]
 name = "goe-framework"
-version = "1.0.8.dev0"
+version = "1.1.0"
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 maintainers = [
   {name = "Neil Johnson", email = "neiljohnson@google.com"},
 ]
 classifiers = [
   "Development Status :: 4 - Beta",
-  "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: OS Independent"
 ]
 dependencies = [
@@ -53,7 +53,7 @@ dependencies = [
     "azure-storage-blob",
 
     # For Oracle frontend
-    "cx-Oracle",
+    "oracledb",
 
     # For BigQuery backend
     "google-cloud-bigquery",

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/config/config_validation_functions.py

@@ -14,12 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" config_validation_functions: Library of functions used in goe.py and OrchestrationConfig
-"""
+"""config_validation_functions: Library of functions used in goe.py and OrchestrationConfig"""
 
 import json
 import random
 import re
+import oracledb
 
 from goe.config import orchestration_defaults
 from goe.filesystem.goe_dfs import (
@@ -417,6 +417,15 @@ def normalise_rdbms_oracle_options(options, exc_cls=OrchestrationConfigException
     silent is required when this is called before options has been defined, we don't want to try logging anything
     before logging is ready.
     """
+    if options.oracledb_thick_mode:
+        if oracledb.is_thin_mode():
+            try:
+                oracledb.init_oracle_client()
+            except Exception as exc:
+                raise exc_cls(
+                    f"Failed to initialize oracledb thick mode: {exc}"
+                ) from exc
+
     if not options.oracle_dsn:
         raise exc_cls("Oracle connection options required")
     elif not options.use_oracle_wallet and (

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/config/orchestration_config.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" OrchestrationConfig: Class of configuration attributes for Orchestration commands.
-    These are not attributes we expect an end user to change/provide on a command by command basis,
-    it is config coming from configuration files.
+"""OrchestrationConfig: Class of configuration attributes for Orchestration commands.
+These are not attributes we expect an end user to change/provide on a command by command basis,
+it is config coming from configuration files.
 """
 
 import logging
@@ -47,7 +47,6 @@ from goe.offload.offload_messages import OffloadMessages
 from goe.offload.offload_transport_functions import hs2_connection_log_message
 from goe.util.password_tools import PasswordToolsException
 
-
 logger = logging.getLogger(__name__)
 # Disabling logging by default
 logger.addHandler(logging.NullHandler())
@@ -201,6 +200,7 @@ EXPECTED_CONFIG_ARGS = [
     "teradata_repo_user",
     "use_ssl",
     "use_oracle_wallet",
+    "oracledb_thick_mode",
     "webhdfs_host",
     "webhdfs_port",
     "verbose",
@@ -288,6 +288,7 @@ class OrchestrationConfig:
     offload_transport_spark_submit_executable: Optional[str]
     offload_transport_spark_thrift_host: Optional[str]
     use_oracle_wallet: bool
+    oracledb_thick_mode: bool
 
     def __init__(self, do_not_connect=False, **kwargs):
         """Do not expect to construct directly via __init__.
@@ -591,6 +592,13 @@ class OrchestrationConfig:
                     orchestration_defaults.use_oracle_wallet_default(),
                 ),
             ),
+            oracledb_thick_mode=config_dict.get(
+                "oracledb_thick_mode",
+                orchestration_defaults.bool_option_from_string(
+                    "ORACLEDB_THICK_MODE",
+                    orchestration_defaults.oracledb_thick_mode_default(),
+                ),
+            ),
             offload_transport_auth_using_oracle_wallet=config_dict.get(
                 "offload_transport_auth_using_oracle_wallet",
                 orchestration_defaults.bool_option_from_string(

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/config/orchestration_defaults.py

@@ -14,10 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" OrchestrationDefaults: Library of functions providing default values for command line options.
-    This is a temporary measure in order to get the code out of goe.py and into a single location, in the
-    future we expect to refactor all option processing, including defaults, and this module will hopefully become
-    redundant at that time.
+"""OrchestrationDefaults: Library of functions providing default values for command line options.
+This is a temporary measure in order to get the code out of goe.py and into a single location, in the
+future we expect to refactor all option processing, including defaults, and this module will hopefully become
+redundant at that time.
 """
 
 # Standard Library
@@ -738,6 +738,15 @@ def use_oracle_wallet_default() -> bool:
     return bool_option_from_string("USE_ORACLE_WALLET", str_val)
 
 
+def oracledb_thick_mode_default() -> bool:
+    str_val = (
+        os.environ.get("ORACLEDB_THICK_MODE")
+        or os.environ.get("USE_ORACLE_WALLET")
+        or "false"
+    )
+    return bool_option_from_string("ORACLEDB_THICK_MODE", str_val)
+
+
 def offload_transport_auth_using_oracle_wallet_default() -> bool:
     str_val = os.environ.get("OFFLOAD_TRANSPORT_AUTH_USING_ORACLE_WALLET") or "false"
     return bool_option_from_string(

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/connect/connect.py

@@ -69,7 +69,6 @@ from goe.util.goe_log import log_exception
 from goe.util.misc_functions import unsurround
 from goe.util.redis_tools import RedisClient
 
-
 OS_RELEASE_FILE_REDHAT = "/etc/redhat-release"
 OS_RELEASE_FILE_SUSE = "/etc/SuSE-release"
 OS_RELEASE_FILE_DEBIAN = "/etc/os-release"
@@ -377,7 +376,7 @@ def upgrade_environment_file(environment_file, template_file):
             )
             for key in sorted(set(template.keys()) - set(configuration.keys())):
                 detail("Adding %s" % key)
-                f.write(template[key])
+                f.write(f"{key}={template[key]}\n")
 
         log("%s has been updated" % environment_file, ansi_code="green")
     else:

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/connect/connect_frontend.py

@@ -16,7 +16,7 @@ import os
 import sys
 import traceback
 
-from cx_Oracle import DatabaseError
+from oracledb import DatabaseError
 
 from goe.connect.connect_functions import (
     FatalTestFailure,
@@ -25,6 +25,7 @@ from goe.connect.connect_functions import (
     log,
     success,
     test_header,
+    warning,
 )
 from goe.offload.factory.frontend_api_factory import frontend_api_factory
 from goe.offload import offload_constants
@@ -41,7 +42,6 @@ from goe.goe import (
     NLS_LANG_MISSING_CHARACTER_SET_EXCEPTION_TEMPLATE,
 )
 
-
 GOE_MINIMUM_ORACLE_VERSION = "10.2.0.1"
 
 
@@ -141,7 +141,7 @@ def test_oracle(orchestration_config, messages):
             'NLS_LANG not specified in environment, this will be set at offload time to "%s"'
             % os.environ["NLS_LANG"]
         )
-        failure(test_name)
+        warning(test_name)
     else:
         if not nls_lang_has_charset():
             detail(
@@ -224,34 +224,6 @@ def test_oracle(orchestration_config, messages):
             detail(exc)
             failure(test_name)
 
-    test_name = "Oracle Parameters"
-    test_header(test_name)
-
-    db_parameters = [
-        "processes",
-        "sessions",
-        "query_rewrite_enabled",
-        "_optimizer_cartesian_enabled",
-    ]
-    binds = {}
-
-    for count, value in enumerate(db_parameters):
-        binds["b" + str(count)] = value
-
-    sql = (
-        "select name, value from gv$parameter where name in (%s) group by name, value order by 1,2"
-        % ",".join([":" + _ for _ in binds])
-    )
-    db_parameter_values = frontend_api.execute_query_fetch_all(sql, query_params=binds)
-    col1_width = max(
-        max([len(row[0]) for row in db_parameter_values]), len("Parameter")
-    )
-    col2_width = max(max([len(row[1]) for row in db_parameter_values]), len("Value"))
-    parameter_format = "{0:" + str(col1_width) + "}     {1:>" + str(col2_width) + "}"
-    detail(parameter_format.format("Parameter", "Value"))
-    for row in db_parameter_values:
-        detail(parameter_format.format(row[0], row[1]))
-
     cx.close()
 
 

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/connect/connect_transport.py

@@ -287,7 +287,7 @@ def test_spark_gcloud(orchestration_config, messages):
         log("Skipping Spark gcloud tests due to absent config", detail=VVERBOSE)
         return
 
-    test_name = "Spark Dataproc settings"
+    test_name = "Google Managed Spark settings"
     test_header(test_name)
     if spark_submit_executable_exists(
         orchestration_config,
@@ -309,7 +309,7 @@ def test_spark_gcloud(orchestration_config, messages):
             orchestration_config, messages
         )
         verify_offload_transport_rdbms_connectivity(
-            data_transport_client, "Spark Dataproc"
+            data_transport_client, "Google Managed Spark"
         )
 
     if is_spark_gcloud_batches_available(orchestration_config, None, messages=messages):
@@ -317,7 +317,7 @@ def test_spark_gcloud(orchestration_config, messages):
             orchestration_config, messages
         )
         verify_offload_transport_rdbms_connectivity(
-            data_transport_client, "Spark Dataproc Batches"
+            data_transport_client, "Google Managed Spark serverless"
         )
 
 

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/listener/asgi.py

@@ -17,7 +17,7 @@ import multiprocessing
 from pathlib import Path
 
 # Third Party Libraries
-from cx_Oracle import DatabaseError as OracleDatabaseError
+from oracledb import DatabaseError as OracleDatabaseError
 from fastapi import FastAPI
 from fastapi.exceptions import HTTPException, RequestValidationError
 from fastapi.responses import ORJSONResponse

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/listener/exceptions/base.py

@@ -13,11 +13,12 @@
 # limitations under the License.
 
 """HTTP exception and handler for FastAPI."""
+
 # Standard Library
 from typing import Any, Dict, Optional, Union
 
 # Third Party Libraries
-from cx_Oracle import DatabaseError as OracleDatabaseError
+from oracledb import DatabaseError as OracleDatabaseError
 from fastapi import Request, status
 from fastapi.exceptions import HTTPException, RequestValidationError
 from fastapi.responses import ORJSONResponse
@@ -93,11 +94,13 @@ class ApplicationError(BaseApplicationError):
         BaseApplicationError.__init__(  # noqa: WPS609
             self,
             status_code,
-            content=schemas.ErrorMessage(code=status_code, message=message).dict(
-                exclude_none=True,
-            )
-            if isinstance(message, str)
-            else message,
+            content=(
+                schemas.ErrorMessage(code=status_code, message=message).dict(
+                    exclude_none=True,
+                )
+                if isinstance(message, str)
+                else message
+            ),
         )
 
 

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/offload/factory/offload_transport_factory.py

@@ -195,7 +195,7 @@ def spark_livy_jdbc_connectivity_checker(offload_options, messages):
 
 
 def spark_dataproc_jdbc_connectivity_checker(offload_options, messages):
-    """Connect needs a cut down client to simply check RDBMS connectivity from Dataproc
+    """Connect needs a cut down client to simply check RDBMS connectivity from Managed Spark
     back to the source RDBMS is correctly configured
     """
     from goe.offload.spark.dataproc_offload_transport import (
@@ -207,8 +207,8 @@ def spark_dataproc_jdbc_connectivity_checker(offload_options, messages):
 
 
 def spark_dataproc_batches_jdbc_connectivity_checker(offload_options, messages):
-    """Connect needs a cut down client to simply check RDBMS connectivity from Dataproc Batches
-    back to the source RDBMS is correctly configured
+    """Connect needs a cut down client to simply check RDBMS connectivity from Managed
+    Spark serverless back to the source RDBMS is correctly configured
     """
     from goe.offload.spark.dataproc_offload_transport import (
         OffloadTransportSparkBatchesGcloudCanary,

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/offload/offload_source_data.py

@@ -1042,12 +1042,12 @@ class OffloadSourceDataInterface(metaclass=ABCMeta):
         return source_partitions[hv_partition_index + 1]
 
     def _select_target_table_max(self):
-        """Get highest point of data stored in target table and store in state
-        Impyla/cx_Oracle both truncate nanoseconds hence TIMESTAMP columns are returned as string
-        optimistic_prune_clause is used to limit the partitions scanned when looking for existing rows
+        """Get highest point of data stored in target table and store in state.
+        Impyla and oracledb both truncate nanoseconds hence TIMESTAMP columns are returned as string.
+        optimistic_prune_clause is used to limit the partitions scanned when looking for existing rows:
             If no data is returned then we resort to scanning without the clause
             optimistic_prune_clause is only applicable for single column partition schemes
-        Also returns the max value for convenience
+        Also returns the max value for convenience.
         """
         if not self._target_table_exists():
             return None

{goe_framework-1.0.8.dev0 → goe_framework-1.1.0}/src/goe/offload/offload_transport_functions.py

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" OffloadTransportFunctions: Library of functions used in goe.py and data transport modules
-"""
+"""OffloadTransportFunctions: Library of functions used in goe.py and data transport modules"""
 
 import decimal
 import logging
@@ -26,7 +25,7 @@ from subprocess import PIPE, STDOUT
 import sys
 from typing import Optional, TYPE_CHECKING
 
-import cx_Oracle as cxo
+import oracledb as cxo
 
 from goe.offload.frontend_api import FRONTEND_TRACE_MODULE
 from goe.offload.offload_constants import (
@@ -146,7 +145,7 @@ def get_rdbms_connection_for_oracle(
     if use_oracle_wallet:
         ora_conn = cxo.connect(dsn=ora_dsn)
     else:
-        ora_conn = cxo.connect(ora_user, ora_pass, ora_dsn)
+        ora_conn = cxo.connect(user=ora_user, password=ora_pass, dsn=ora_dsn)
     session_cursor = ora_conn.cursor()
     try:
         session_cursor.execute(