goe-framework 1.0.7__tar.gz → 1.0.8.dev0__tar.gz

{goe_framework-1.0.7/src/goe_framework.egg-info → goe_framework-1.0.8.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: goe-framework
-Version: 1.0.7
+Version: 1.0.8.dev0
 Maintainer-email: Neil Johnson <neiljohnson@google.com>
 Project-URL: Homepage, https://github.com/gluent/goe
 Project-URL: Issues, https://github.com/gluent/goe/issues
@@ -67,6 +67,7 @@ Provides-Extra: test
 Requires-Dist: nox; extra == "test"
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-xdist; extra == "test"
+Dynamic: license-file
 
 # GOE
 A simple and flexible way to copy data from an Oracle Database to Google BigQuery.

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/pyproject.toml

@@ -14,7 +14,7 @@
 
 [project]
 name = "goe-framework"
-version = "1.0.7"
+version = "1.0.8.dev0"
 readme = "README.md"
 requires-python = ">=3.7"
 maintainers = [

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/config/orchestration_config.py

@@ -199,7 +199,6 @@ EXPECTED_CONFIG_ARGS = [
     "teradata_app_pass",
     "teradata_server",
     "teradata_repo_user",
-    "udf_db",
     "use_ssl",
     "use_oracle_wallet",
     "webhdfs_host",
@@ -825,7 +824,6 @@ class OrchestrationConfig:
                 "teradata_repo_user",
                 orchestration_defaults.teradata_repo_user_default(),
             ),
-            udf_db=config_dict.get("udf_db", orchestration_defaults.udf_db_default()),
             use_ssl=config_dict.get(
                 "use_ssl", orchestration_defaults.use_ssl_default()
             ),

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/config/orchestration_defaults.py

@@ -476,10 +476,6 @@ def ldap_password_file_default():
     return os.environ.get("HIVE_SERVER_LDAP_PASSWORD_FILE")
 
 
-def udf_db_default():
-    return os.environ.get("OFFLOAD_UDF_DB")
-
-
 def use_ssl_default():
     return bool(os.environ.get("SSL_ACTIVE", "").lower() == "true" or ca_cert_default())
 

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/connect/connect.py

@@ -486,20 +486,6 @@ def get_connect_opts():
         help="Adds missing configuration variables from the environment file template to the environment file",
     )
     # Hidden options to keep TeamCity testing functioning
-    opt.add_option(
-        "--validate-udfs",
-        dest="validate_udfs",
-        default=False,
-        action="store_true",
-        help=SUPPRESS_HELP,
-    )
-    opt.add_option(
-        "--install-udfs",
-        dest="install_udfs",
-        default=False,
-        action="store_true",
-        help=SUPPRESS_HELP,
-    )
     opt.add_option(
         "--create-backend-db",
         dest="create_backend_db",

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/goe.py

@@ -220,7 +220,6 @@ EXPECTED_OFFLOAD_ARGS = [
     "offload_predicate_modify_hybrid_view",
     "offload_stats_method",
     "offload_transport_method",
-    "offload_type",
     "offload_transport_consistent_read",
     "offload_transport_dsn",
     "offload_transport_fetch_size",
@@ -228,8 +227,10 @@ EXPECTED_OFFLOAD_ARGS = [
     "offload_transport_queue_name",
     "offload_transport_parallelism",
     "offload_transport_small_table_threshold",
+    "offload_transport_snapshot",
     "offload_transport_spark_properties",
     "offload_transport_validation_polling_interval",
+    "offload_type",
     "older_than_date",
     "older_than_days",
     "operation_name",
@@ -940,6 +941,11 @@ def normalise_offload_transport_user_options(options):
         "OFFLOAD_TRANSPORT_FETCH_SIZE/--offload-transport-fetch-size",
         options.offload_transport_fetch_size,
     )
+    if options.offload_transport_snapshot:
+        options.offload_transport_snapshot = check_opt_is_posint(
+            "--offload-transport-snapshot",
+            options.offload_transport_snapshot,
+        )
 
     verify_json_option(
         "OFFLOAD_TRANSPORT_SPARK_PROPERTIES/--offload-transport-spark-properties",
@@ -2242,6 +2248,7 @@ class OffloadOperation(BaseOperation):
             offload_transport_queue_name=options.offload_transport_queue_name,
             offload_transport_parallelism=options.offload_transport_parallelism,
             offload_transport_small_table_threshold=options.offload_transport_small_table_threshold,
+            offload_transport_snapshot=options.offload_transport_snapshot,
             offload_transport_spark_properties=options.offload_transport_spark_properties,
             offload_transport_validation_polling_interval=options.offload_transport_validation_polling_interval,
             offload_type=options.offload_type,
@@ -2416,6 +2423,7 @@ class OffloadOperation(BaseOperation):
                 "offload_transport_small_table_threshold",
                 orchestration_defaults.offload_transport_small_table_threshold_default(),
             ),
+            offload_transport_snapshot=operation_dict.get("offload_transport_snapshot"),
             offload_transport_spark_properties=operation_dict.get(
                 "offload_transport_spark_properties",
                 orchestration_defaults.offload_transport_spark_properties_default(),
@@ -2841,13 +2849,6 @@ def offload_table(
 
     rows_offloaded = None
 
-    pre_offload_snapshot = None
-    if offload_options.db_type == offload_constants.DBTYPE_ORACLE:
-        # Pre-offload SCN will be stored in metadata.
-        pre_offload_snapshot = offload_source_table.get_current_scn(
-            return_none_on_failure=True
-        )
-
     create_final_backend_table_step(offload_target_table, offload_operation)
 
     data_transport_client = offload_transport_factory(
@@ -2860,6 +2861,11 @@ def offload_table(
         dfs_client,
     )
 
+    pre_offload_snapshot = None
+    if offload_options.db_type == offload_constants.DBTYPE_ORACLE:
+        # Pre-offload SCN will be stored in metadata.
+        pre_offload_snapshot = data_transport_client.get_transport_snapshot()
+
     offload_target_table.set_final_table_casts(
         offload_source_table.columns,
         data_transport_client.get_staging_file().get_staging_columns(),

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/offload/backend_api.py

@@ -64,7 +64,6 @@ from goe.offload.offload_messages import VERBOSE, VVERBOSE
 from goe.offload.offload_constants import (
     CAPABILITY_BUCKET_HASH_COLUMN,
     CAPABILITY_CANONICAL_DATE,
-    CAPABILITY_CANONICAL_FLOAT,
     CAPABILITY_CANONICAL_TIME,
     CAPABILITY_CASE_SENSITIVE,
     CAPABILITY_COLUMN_STATS_SET,
@@ -78,11 +77,7 @@ from goe.offload.offload_constants import (
     CAPABILITY_FS_SCHEME_INHERIT,
     CAPABILITY_FS_SCHEME_WASB,
     CAPABILITY_GOE_COLUMN_TRANSFORMATIONS,
-    CAPABILITY_GOE_JOIN_PUSHDOWN,
-    CAPABILITY_GOE_MATERIALIZED_JOIN,
     CAPABILITY_GOE_PARTITION_FUNCTIONS,
-    CAPABILITY_GOE_SEQ_TABLE,
-    CAPABILITY_GOE_UDFS,
     CAPABILITY_LOAD_DB_TRANSPORT,
     CAPABILITY_NAN,
     CAPABILITY_NANOSECONDS,
@@ -121,10 +116,6 @@ class BackendStatsException(Exception):
     pass
 
 
-class MissingSequenceTableException(Exception):
-    pass
-
-
 class BackendApiConnectionException(Exception):
     pass
 
@@ -1595,25 +1586,6 @@ FROM   %(db)s.%(table)s%(where_clause)s%(group_by)s%(order_by)s""" % {
     def get_user_name(self):
         pass
 
-    @abstractmethod
-    def insert_literal_values(
-        self,
-        db_name,
-        table_name,
-        literal_list,
-        column_list=None,
-        max_rows_per_insert=250,
-        split_by_cr=True,
-    ):
-        """Used to insert specific data into a table. The table should already exist.
-        literal_list: A list of rows to insert (a list of lists).
-                      The row level lists should contain the exact right number of columns.
-        column_list: The column specs for the columns to be inserted, if left blank this will default to all
-                     columns in the table
-        Disclaimer: This code is used in testing and generating the sequence table. It is not robust enough to
-                    be a part of any Offload Transport
-        """
-
     @abstractmethod
     def is_nan_sql_expression(self, column_expr):
         """Return a SQL expression testing if an expression is NaN (not-a-number)."""
@@ -1738,18 +1710,6 @@ FROM   %(db)s.%(table)s%(where_clause)s%(group_by)s%(order_by)s""" % {
         """Returns True if the backend column requires a synthetic column if it is used for partitioning."""
         pass
 
-    @abstractmethod
-    def populate_sequence_table(
-        self, db_name, table_name, starting_seq, target_seq, split_by_cr=False
-    ):
-        """Populate options.sequence_table_name up to target_seq.
-        split_by_cr: This option is a bit of a fudge but allows a calling program to parse the output.
-                     The idea being that, on some backends, this method may produce lengthy output
-                     which we may not want on screen. With split_by_cr the calling program has the
-                     opportunity to split the executed text by cr and only log some lines.
-        """
-        pass
-
     @abstractmethod
     def refresh_table_files(self, db_name, table_name, sync=None):
         pass
@@ -1770,10 +1730,6 @@ FROM   %(db)s.%(table)s%(where_clause)s%(group_by)s%(order_by)s""" % {
         """Check the role exists, returns True/False"""
         pass
 
-    @abstractmethod
-    def sequence_table_max(self, db_name, table_name):
-        pass
-
     @abstractmethod
     def set_column_stats(
         self, db_name, table_name, new_column_stats, ndv_cap, num_null_factor
@@ -1886,28 +1842,6 @@ FROM   %(db)s.%(table)s%(where_clause)s%(group_by)s%(order_by)s""" % {
         """
         pass
 
-    @abstractmethod
-    def udf_installation_os(self, user_udf_version):
-        """Copies any libraries required to support UDFs from our software package to wherever they should be.
-        Returns a list of commands executed.
-        """
-        pass
-
-    @abstractmethod
-    def udf_installation_sql(self, create_udf_db, udf_db=None):
-        """Executes any SQL commands required for UDF installation.
-        Returns a list of commands executed, in dry_run mode that's all it does.
-        udf_db can be empty which means we won't specify one and pickup a default.
-        """
-        pass
-
-    @abstractmethod
-    def udf_installation_test(self, udf_db=None):
-        """Executes any SQL commands to test each UDF.
-        udf_db can be empty which means we won't specify one and pickup a default.
-        """
-        pass
-
     @abstractmethod
     def valid_canonical_override(self, column, canonical_override):
         """Present has a number of options for overriding the default canonical mapping in to_canonical_column().
@@ -1952,10 +1886,6 @@ FROM   %(db)s.%(table)s%(where_clause)s%(group_by)s%(order_by)s""" % {
         """
         return self.is_capability_supported(CAPABILITY_CANONICAL_DATE)
 
-    def canonical_float_supported(self):
-        """Does the backend have a pure 4 byte binary floating point data type"""
-        return self.is_capability_supported(CAPABILITY_CANONICAL_FLOAT)
-
     def canonical_time_supported(self):
         """Does the backend have a pure time data type (no date part).
         Note that there is an Impala override for this.
@@ -2008,21 +1938,9 @@ FROM   %(db)s.%(table)s%(where_clause)s%(group_by)s%(order_by)s""" % {
     def goe_column_transformations_supported(self):
         return self.is_capability_supported(CAPABILITY_GOE_COLUMN_TRANSFORMATIONS)
 
-    def goe_join_pushdown_supported(self):
-        return self.is_capability_supported(CAPABILITY_GOE_JOIN_PUSHDOWN)
-
-    def goe_materialized_join_supported(self):
-        return self.is_capability_supported(CAPABILITY_GOE_MATERIALIZED_JOIN)
-
     def goe_partition_functions_supported(self):
         return self.is_capability_supported(CAPABILITY_GOE_PARTITION_FUNCTIONS)
 
-    def goe_sequence_table_supported(self):
-        return self.is_capability_supported(CAPABILITY_GOE_SEQ_TABLE)
-
-    def goe_udfs_supported(self):
-        return self.is_capability_supported(CAPABILITY_GOE_UDFS)
-
     def load_db_transport_supported(self):
         return self.is_capability_supported(CAPABILITY_LOAD_DB_TRANSPORT)
 

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/offload/backend_table.py

@@ -209,7 +209,6 @@ class BackendTableInterface(metaclass=ABCMeta):
         self._offload_staging_format = getattr(
             self._orchestration_config, "offload_staging_format", None
         )
-        self._udf_db = getattr(self._orchestration_config, "udf_db", None)
         # If orchestration_operation is not set then we are not doing anything significant by way of offload/present
         self._ipa_predicate_type = None
         self._offload_distribute_enabled = None
@@ -1044,12 +1043,12 @@ class BackendTableInterface(metaclass=ABCMeta):
 
     def _partition_function_sql_expression(self, partition_info, sql_input_expression):
         """Return a string containing a call to a partition function UDF.
-        e.g. UDF_DB.UDF_NAME(sql_input_expression)
+        e.g. SCHEMA.UDF_NAME(sql_input_expression)
         """
         assert partition_info
-        udf_db, udf_name = partition_info.function.split(".")
+        udf_schema, udf_name = partition_info.function.split(".")
         return "{}({})".format(
-            self._db_api.enclose_object_reference(udf_db, udf_name),
+            self._db_api.enclose_object_reference(udf_schema, udf_name),
             sql_input_expression,
         )
 
@@ -1807,10 +1806,6 @@ class BackendTableInterface(metaclass=ABCMeta):
     def default_date_based_partition_granularity(self):
         return self._db_api.default_date_based_partition_granularity()
 
-    def default_udf_db_name(self):
-        """By default we support UDF_DB but individual backends may have their own override"""
-        return self._udf_db
-
     def delta_table_exists(self):
         return False
 
@@ -2375,18 +2370,6 @@ class BackendTableInterface(metaclass=ABCMeta):
     def staging_area_exists(self):
         pass
 
-    @abstractmethod
-    def synthetic_bucket_data_type(self):
-        pass
-
-    @abstractmethod
-    def synthetic_bucket_filter_capable_column(self, backend_column):
-        """Returns the metadata method and SQL expression pattern used to generate the bucket id
-        Currently only enabling bucket filtering for integral data
-        When we consider opening this up to other types we should consider excluding TZ aware data because
-        a backend upgrade could change the result of CASTs
-        """
-
     ###########################################################################
     # PUBLIC METHODS - HIGH LEVEL STEP METHODS AND SUPPORTING ABSTRACT METHODS
     ###########################################################################
@@ -2530,9 +2513,6 @@ class BackendTableInterface(metaclass=ABCMeta):
     def canonical_date_supported(self):
         return self._db_api.canonical_date_supported()
 
-    def canonical_float_supported(self):
-        return self._db_api.canonical_float_supported()
-
     def canonical_time_supported(self):
         return self._db_api.canonical_time_supported()
 
@@ -2545,18 +2525,9 @@ class BackendTableInterface(metaclass=ABCMeta):
     def goe_column_transformations_supported(self):
         return self._db_api.goe_column_transformations_supported()
 
-    def goe_join_pushdown_supported(self):
-        return self._db_api.goe_join_pushdown_supported()
-
-    def goe_materialized_join_supported(self):
-        return self._db_api.goe_materialized_join_supported()
-
     def goe_partition_functions_supported(self):
         return self._db_api.goe_partition_functions_supported()
 
-    def goe_udfs_supported(self):
-        return self._db_api.goe_udfs_supported()
-
     def nan_supported(self):
         return self._db_api.nan_supported()
 

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/offload/bigquery/bigquery_backend_api.py

@@ -23,7 +23,6 @@
 from datetime import datetime
 import logging
 from math import ceil
-import pprint
 import re
 from textwrap import dedent
 import traceback
@@ -255,9 +254,9 @@ class BackendBigQueryApi(BackendApiInterface):
     def _check_kms_key_name(self, kms_key_name: str, key_type="job"):
         """Use startswith() to verify custom key for key name.
         Example of custom key name:
-            projects/goe-teamcity/locations/us-west3/keyRings/krname/cryptoKeys/etl5
+            projects/goe-test/locations/us-west3/keyRings/krname/cryptoKeys/etl5
         Example of what we get from a query_job object:
-            projects/goe-teamcity/locations/us-west3/keyRings/krname/cryptoKeys/etl5/cryptoKeyVersions/1
+            projects/goe-test/locations/us-west3/keyRings/krname/cryptoKeys/etl5/cryptoKeyVersions/1
         It's worth noting that this has not always been the case, BigQuery behaviour has changed in the past.
         """
         if self._kms_key_name and not (kms_key_name or "").startswith(
@@ -1814,68 +1813,6 @@ FROM   %(from_db_table)s%(where)s""" % {
         row = self.execute_query_fetch_one("SELECT SESSION_USER()", log_level=None)
         return row[0] if row else None
 
-    def insert_literal_values(
-        self,
-        db_name,
-        table_name,
-        literal_list,
-        column_list=None,
-        max_rows_per_insert=250,
-        split_by_cr=True,
-    ):
-        """Insert an array of literals into a table using INSERT...VALUES(...),(...),(...).
-        We've done this for simplicity due to two issues:
-        1) BigQuery issue: https://github.com/googleapis/google-cloud-python/issues/5539
-            "The issue is eventual consistency with the backend. Replacing the table, while it has the
-             same table_id, represents a new table in terms of it's internal UUID and thus backends may
-             deliver to the "old" table for a short period (typically a few minutes)."
-           This means we can have problems with this method of inserting rows, it is likely only suitable for testing.
-        2) Python does not support encoding Decimal to JSON until Python 3.10:
-            https://bugs.python.org/issue16535
-           Which gives us issue with this API:
-            site-packages/google/cloud/bigquery/client.py", line 3421, in insert_rows
-                return self.insert_rows_json(table, json_rows, **kwargs)
-                ...
-            json/encoder.py", line 179, in default
-                raise TypeError(f'Object of type {o.__class__.__name__} '
-            TypeError: Object of type Decimal is not JSON serializable
-        """
-
-        def gen_literal(py_val, data_type):
-            return str(self.to_backend_literal(py_val, data_type))
-
-        assert db_name
-        assert table_name
-        assert literal_list and isinstance(literal_list, list)
-        assert isinstance(literal_list[0], list)
-
-        column_list = column_list or self.get_columns(db_name, table_name)
-        column_names = [_.name for _ in column_list]
-        data_type_strs = [_.format_data_type() for _ in column_list]
-
-        cmds = []
-        remaining_rows = literal_list[:]
-        while remaining_rows:
-            this_chunk = remaining_rows[:max_rows_per_insert]
-            remaining_rows = remaining_rows[max_rows_per_insert:]
-            formatted_rows = []
-            for row in this_chunk:
-                formatted_rows.append(
-                    ",".join(
-                        gen_literal(py_val, data_type)
-                        for py_val, data_type in zip(row, data_type_strs)
-                    )
-                )
-            sql = self._insert_literals_using_insert_values_sql_text(
-                db_name,
-                table_name,
-                column_names,
-                formatted_rows,
-                split_by_cr=split_by_cr,
-            )
-            cmds.extend(self.execute_dml(sql, log_level=VVERBOSE))
-        return cmds
-
     def is_nan_sql_expression(self, column_expr):
         """is_nan for BigQuery"""
         return "is_nan(%s)" % column_expr
@@ -2086,11 +2023,6 @@ FROM   %(from_db_table)s%(where)s""" % {
         """
         return -(2**63)
 
-    def populate_sequence_table(
-        self, db_name, table_name, starting_seq, target_seq, split_by_cr=False
-    ):
-        raise NotImplementedError("Sequence table does not apply for BigQuery")
-
     def refresh_table_files(self, db_name, table_name, sync=None):
         """No requirement to re-scan files for a table on BigQuery but drop from cache because that will be stale"""
         self.drop_state()
@@ -2128,9 +2060,6 @@ FROM   %(from_db_table)s%(where)s""" % {
         """No roles in BigQuery"""
         pass
 
-    def sequence_table_max(self, db_name, table_name):
-        raise NotImplementedError("Sequence table does not apply for BigQuery")
-
     def set_column_stats(
         self, db_name, table_name, new_column_stats, ndv_cap, num_null_factor
     ):
@@ -2287,15 +2216,6 @@ FROM   %(from_db_table)s%(where)s""" % {
         except NotFound:
             return False
 
-    def udf_installation_os(self, user_udf_version):
-        raise NotImplementedError("GOE UDFs are not supported on BigQuery")
-
-    def udf_installation_sql(self, create_udf_db, udf_db=None):
-        raise NotImplementedError("GOE UDFs are not supported on BigQuery")
-
-    def udf_installation_test(self, udf_db=None):
-        raise NotImplementedError("GOE UDFs are not supported on BigQuery")
-
     def valid_canonical_override(self, column, canonical_override):
         assert isinstance(column, BigQueryColumn)
         if isinstance(canonical_override, CanonicalColumn):

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/offload/bigquery/bigquery_backend_table.py

@@ -499,10 +499,6 @@ class BackendBigQueryTable(BackendTableInterface):
         else:
             return []
 
-    def default_udf_db_name(self):
-        """By default we support UDF_DB but on BigQuery we use the data db as a fall back"""
-        return self._udf_db or self.db_name
-
     def empty_staging_area(self, staging_file):
         self._rm_load_table_location()
 
@@ -623,12 +619,6 @@ class BackendBigQueryTable(BackendTableInterface):
             self._warning("Staging database %s does not exist" % self._load_db_name)
         return self._load_db_exists()
 
-    def synthetic_bucket_data_type(self):
-        raise NotImplementedError("Synthetic bucketing is not supported on BigQuery")
-
-    def synthetic_bucket_filter_capable_column(self, backend_column):
-        raise NotImplementedError("Synthetic bucketing is not supported on BigQuery")
-
     def validate_type_conversions(self, staging_columns: list):
         """Validate the staged data before we insert it into the final backend table.
         BigQuery catches overflowing casts but this way we can spot them beforehand and

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/offload/factory/offload_transport_factory.py

@@ -35,7 +35,6 @@ def offload_transport_factory(
     offload_options,
     messages,
     dfs_client,
-    rdbms_columns_override=None,
 ):
     """Constructs and returns an appropriate data transport object based on user inputs and RDBMS table"""
     if offload_transport_method == OFFLOAD_TRANSPORT_METHOD_QUERY_IMPORT:
@@ -51,7 +50,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SQOOP:
         from goe.offload.hadoop.sqoop_offload_transport import (
@@ -68,7 +66,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SQOOP_BY_QUERY:
         from goe.offload.hadoop.sqoop_offload_transport import (
@@ -85,7 +82,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SPARK_THRIFT:
         from goe.offload.offload_transport import OffloadTransportSparkThrift
@@ -100,7 +96,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SPARK_SUBMIT:
         from goe.offload.offload_transport import OffloadTransportSparkSubmit
@@ -115,7 +110,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SPARK_DATAPROC_GCLOUD:
         from goe.offload.spark.dataproc_offload_transport import (
@@ -133,7 +127,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SPARK_BATCHES_GCLOUD:
         from goe.offload.spark.dataproc_offload_transport import (
@@ -150,7 +143,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     elif offload_transport_method == OFFLOAD_TRANSPORT_METHOD_SPARK_LIVY:
         from goe.offload.spark.livy_offload_transport import OffloadTransportSparkLivy
@@ -165,7 +157,6 @@ def offload_transport_factory(
             offload_options,
             messages,
             dfs_client,
-            rdbms_columns_override=rdbms_columns_override,
         )
     else:
         raise NotImplementedError(

{goe_framework-1.0.7 → goe_framework-1.0.8.dev0}/src/goe/offload/frontend_api.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" FrontendApiInterface: Library for logic/interaction with an RDBMS frontend.
-    This module enforces an interface with common, high level, methods and an implementation
-    for each supported RDBMS, e.g. Oracle, MSSQL
+"""FrontendApiInterface: Library for logic/interaction with an RDBMS frontend.
+This module enforces an interface with common, high level, methods and an implementation
+for each supported RDBMS, e.g. Oracle, MSSQL
 """
 
 import logging
@@ -29,16 +29,13 @@ from goe.offload.column_metadata import (
 )
 from goe.offload.offload_constants import (
     CAPABILITY_CANONICAL_DATE,
-    CAPABILITY_CANONICAL_FLOAT,
     CAPABILITY_CANONICAL_TIME,
     CAPABILITY_CASE_SENSITIVE,
     CAPABILITY_GOE_HAS_DB_CODE_COMPONENT,
-    CAPABILITY_GOE_JOIN_PUSHDOWN,
     CAPABILITY_GOE_LIST_PARTITION_APPEND,
     CAPABILITY_GOE_MULTI_COLUMN_INCREMENTAL_KEY,
     CAPABILITY_GOE_OFFLOAD_STATUS_REPORT,
     CAPABILITY_GOE_SCHEMA_SYNC,
-    CAPABILITY_HYBRID_SCHEMA,
     CAPABILITY_LOW_HIGH_VALUE_FROM_STATS,
     CAPABILITY_NAN,
     CAPABILITY_PARAMETERIZED_QUERIES,
@@ -406,6 +403,10 @@ class FrontendApiInterface(metaclass=ABCMeta):
             [OracleColumn(...), OracleColumn(...)]
         """
 
+    @abstractmethod
+    def get_current_scn(self):
+        """Return the current system change number for the frontend."""
+
     @abstractmethod
     def get_db_unique_name(self) -> str:
         """Return the unique name for the frontend database"""
@@ -833,10 +834,6 @@ class FrontendApiInterface(metaclass=ABCMeta):
         """
         return self.is_capability_supported(CAPABILITY_CANONICAL_DATE)
 
-    def canonical_float_supported(self) -> bool:
-        """Does the frontend have a pure 4 byte binary floating point data type"""
-        return self.is_capability_supported(CAPABILITY_CANONICAL_FLOAT)
-
     def canonical_time_supported(self) -> bool:
         """Does the frontend have a pure time data type (no date part).
         Note that there is an Impala override for this.
@@ -851,9 +848,6 @@ class FrontendApiInterface(metaclass=ABCMeta):
         """Does the frontend have any in-database GOE code"""
         return self.is_capability_supported(CAPABILITY_GOE_HAS_DB_CODE_COMPONENT)
 
-    def goe_join_pushdown_supported(self) -> bool:
-        return self.is_capability_supported(CAPABILITY_GOE_JOIN_PUSHDOWN)
-
     def goe_lpa_supported(self) -> bool:
         return self.is_capability_supported(CAPABILITY_GOE_LIST_PARTITION_APPEND)