goe-framework 1.0.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. goe/__init__.py +17 -0
  2. goe/conductor/__init__.py +0 -0
  3. goe/conductor/hybrid_view_service.py +334 -0
  4. goe/config/__init__.py +0 -0
  5. goe/config/config_file.py +53 -0
  6. goe/config/config_validation_functions.py +663 -0
  7. goe/config/option_descriptions.py +38 -0
  8. goe/config/orchestration_config.py +996 -0
  9. goe/config/orchestration_defaults.py +999 -0
  10. goe/connect/__init__.py +0 -0
  11. goe/connect/connect.py +555 -0
  12. goe/connect/connect_backend.py +481 -0
  13. goe/connect/connect_constants.py +23 -0
  14. goe/connect/connect_frontend.py +265 -0
  15. goe/connect/connect_functions.py +112 -0
  16. goe/connect/connect_transport.py +336 -0
  17. goe/data_governance/__init__.py +0 -0
  18. goe/data_governance/cloudera_navigator.py +845 -0
  19. goe/data_governance/hadoop_data_governance.py +616 -0
  20. goe/data_governance/hadoop_data_governance_constants.py +90 -0
  21. goe/data_governance/hadoop_data_governance_interface.py +209 -0
  22. goe/exceptions.py +25 -0
  23. goe/filesystem/__init__.py +0 -0
  24. goe/filesystem/cli_hdfs.py +539 -0
  25. goe/filesystem/goe_azure.py +483 -0
  26. goe/filesystem/goe_dfs.py +424 -0
  27. goe/filesystem/goe_dfs_factory.py +110 -0
  28. goe/filesystem/goe_gcs.py +352 -0
  29. goe/filesystem/goe_s3.py +363 -0
  30. goe/filesystem/web_hdfs.py +507 -0
  31. goe/goe.py +3526 -0
  32. goe/listener/__init__.py +0 -0
  33. goe/listener/__main__.py +36 -0
  34. goe/listener/api/__init__.py +0 -0
  35. goe/listener/api/routes/__init__.py +0 -0
  36. goe/listener/api/routes/docs.py +68 -0
  37. goe/listener/api/routes/orchestration.py +185 -0
  38. goe/listener/api/routes/system.py +186 -0
  39. goe/listener/asgi.py +120 -0
  40. goe/listener/config/__init__.py +10 -0
  41. goe/listener/config/application.py +113 -0
  42. goe/listener/config/gunicorn.conf.py +284 -0
  43. goe/listener/config/logging.py +81 -0
  44. goe/listener/config/router.py +44 -0
  45. goe/listener/core/__init__.py +0 -0
  46. goe/listener/core/enum.py +24 -0
  47. goe/listener/core/events.py +44 -0
  48. goe/listener/core/middleware/__init__.py +6 -0
  49. goe/listener/core/middleware/compression.py +297 -0
  50. goe/listener/core/middleware/cors.py +38 -0
  51. goe/listener/core/middleware/secure_headers.py +37 -0
  52. goe/listener/core/security.py +46 -0
  53. goe/listener/core/worker.py +149 -0
  54. goe/listener/exceptions/__init__.py +38 -0
  55. goe/listener/exceptions/base.py +279 -0
  56. goe/listener/exceptions/errors.py +99 -0
  57. goe/listener/heartbeat.py +47 -0
  58. goe/listener/prestart.py +85 -0
  59. goe/listener/schemas/__init__.py +58 -0
  60. goe/listener/schemas/base.py +102 -0
  61. goe/listener/schemas/error.py +133 -0
  62. goe/listener/schemas/generics.py +40 -0
  63. goe/listener/schemas/orchestration.py +896 -0
  64. goe/listener/schemas/system.py +191 -0
  65. goe/listener/services/__init__.py +8 -0
  66. goe/listener/services/heartbeat.py +115 -0
  67. goe/listener/services/hybrid_view.py +356 -0
  68. goe/listener/services/orchestrate.py +24 -0
  69. goe/listener/services/periodic_tasks.py +196 -0
  70. goe/listener/services/system.py +126 -0
  71. goe/listener/utils/__init__.py +10 -0
  72. goe/listener/utils/cache.py +444 -0
  73. goe/listener/utils/group_by.py +26 -0
  74. goe/listener/utils/orchestrate.py +52 -0
  75. goe/listener/utils/ping.py +47 -0
  76. goe/listener/utils/system.py +83 -0
  77. goe/listener/worker.py +67 -0
  78. goe/listener/wsgi.py +130 -0
  79. goe/offload/__init__.py +0 -0
  80. goe/offload/backend_api.py +2081 -0
  81. goe/offload/backend_table.py +2645 -0
  82. goe/offload/bigquery/__init__.py +0 -0
  83. goe/offload/bigquery/bigquery_backend_api.py +2579 -0
  84. goe/offload/bigquery/bigquery_backend_table.py +664 -0
  85. goe/offload/bigquery/bigquery_column.py +167 -0
  86. goe/offload/bigquery/bigquery_literal.py +137 -0
  87. goe/offload/bigquery/bigquery_predicate.py +68 -0
  88. goe/offload/column_metadata.py +672 -0
  89. goe/offload/factory/__init__.py +0 -0
  90. goe/offload/factory/backend_api_factory.py +130 -0
  91. goe/offload/factory/backend_table_factory.py +162 -0
  92. goe/offload/factory/frontend_api_factory.py +96 -0
  93. goe/offload/factory/offload_source_table_factory.py +92 -0
  94. goe/offload/factory/offload_transport_factory.py +237 -0
  95. goe/offload/factory/offload_transport_rdbms_api_factory.py +64 -0
  96. goe/offload/factory/query_import_factory.py +45 -0
  97. goe/offload/factory/staging_file_factory.py +92 -0
  98. goe/offload/format_literal.py +70 -0
  99. goe/offload/frontend_api.py +882 -0
  100. goe/offload/hadoop/__init__.py +0 -0
  101. goe/offload/hadoop/hadoop_backend_api.py +1961 -0
  102. goe/offload/hadoop/hadoop_backend_table.py +898 -0
  103. goe/offload/hadoop/hadoop_column.py +151 -0
  104. goe/offload/hadoop/hadoop_predicate.py +82 -0
  105. goe/offload/hadoop/hive_backend_api.py +755 -0
  106. goe/offload/hadoop/hive_backend_table.py +257 -0
  107. goe/offload/hadoop/hive_literal.py +91 -0
  108. goe/offload/hadoop/impala_backend_api.py +1215 -0
  109. goe/offload/hadoop/impala_backend_table.py +273 -0
  110. goe/offload/hadoop/impala_literal.py +68 -0
  111. goe/offload/hadoop/sqoop_offload_transport.py +612 -0
  112. goe/offload/microsoft/__init__.py +0 -0
  113. goe/offload/microsoft/mssql_column.py +228 -0
  114. goe/offload/microsoft/mssql_frontend_api.py +650 -0
  115. goe/offload/microsoft/mssql_offload_source_table.py +735 -0
  116. goe/offload/microsoft/mssql_offload_transport_rdbms_api.py +204 -0
  117. goe/offload/microsoft/synapse_backend_api.py +2890 -0
  118. goe/offload/microsoft/synapse_backend_table.py +663 -0
  119. goe/offload/microsoft/synapse_column.py +220 -0
  120. goe/offload/microsoft/synapse_constants.py +29 -0
  121. goe/offload/microsoft/synapse_literal.py +156 -0
  122. goe/offload/microsoft/synapse_predicate.py +71 -0
  123. goe/offload/offload.py +706 -0
  124. goe/offload/offload_constants.py +605 -0
  125. goe/offload/offload_functions.py +588 -0
  126. goe/offload/offload_messages.py +708 -0
  127. goe/offload/offload_metadata_functions.py +462 -0
  128. goe/offload/offload_source_data.py +3257 -0
  129. goe/offload/offload_source_table.py +1295 -0
  130. goe/offload/offload_status_report.py +4038 -0
  131. goe/offload/offload_transport.py +2873 -0
  132. goe/offload/offload_transport_functions.py +466 -0
  133. goe/offload/offload_transport_rdbms_api.py +289 -0
  134. goe/offload/offload_validation.py +1052 -0
  135. goe/offload/offload_xform_functions.py +100 -0
  136. goe/offload/operation/__init__.py +0 -0
  137. goe/offload/operation/data_type_controls.py +365 -0
  138. goe/offload/operation/ddl_file.py +125 -0
  139. goe/offload/operation/not_null_columns.py +107 -0
  140. goe/offload/operation/partition_controls.py +521 -0
  141. goe/offload/operation/sort_columns.py +152 -0
  142. goe/offload/operation/stats_controls.py +425 -0
  143. goe/offload/operation/table_structure_checks.py +181 -0
  144. goe/offload/operation/transport.py +167 -0
  145. goe/offload/option_validation.py +217 -0
  146. goe/offload/oracle/__init__.py +0 -0
  147. goe/offload/oracle/oracle_column.py +326 -0
  148. goe/offload/oracle/oracle_frontend_api.py +1200 -0
  149. goe/offload/oracle/oracle_literal.py +129 -0
  150. goe/offload/oracle/oracle_offload_source_table.py +1765 -0
  151. goe/offload/oracle/oracle_offload_transport_rdbms_api.py +1106 -0
  152. goe/offload/oracle/oracle_predicate.py +129 -0
  153. goe/offload/predicate_offload.py +742 -0
  154. goe/offload/query_import_interface.py +92 -0
  155. goe/offload/snowflake/__init__.py +0 -0
  156. goe/offload/snowflake/snowflake_backend_api.py +2326 -0
  157. goe/offload/snowflake/snowflake_backend_table.py +747 -0
  158. goe/offload/snowflake/snowflake_column.py +161 -0
  159. goe/offload/snowflake/snowflake_literal.py +124 -0
  160. goe/offload/snowflake/snowflake_predicate.py +67 -0
  161. goe/offload/spark/__init__.py +0 -0
  162. goe/offload/spark/dataproc_offload_transport.py +615 -0
  163. goe/offload/spark/livy_offload_transport.py +551 -0
  164. goe/offload/spark/offload_transport_livy_requests.py +113 -0
  165. goe/offload/spark/pyspark_literal.py +69 -0
  166. goe/offload/spark/spark_thrift_backend_api.py +257 -0
  167. goe/offload/staging/__init__.py +0 -0
  168. goe/offload/staging/avro/__init__.py +0 -0
  169. goe/offload/staging/avro/avro_column.py +108 -0
  170. goe/offload/staging/avro/avro_staging_file.py +416 -0
  171. goe/offload/staging/parquet/__init__.py +0 -0
  172. goe/offload/staging/parquet/parquet_column.py +113 -0
  173. goe/offload/staging/parquet/parquet_staging_file.py +311 -0
  174. goe/offload/staging/staging_file.py +155 -0
  175. goe/offload/synthetic_partition_literal.py +249 -0
  176. goe/offload/teradata/__init__.py +0 -0
  177. goe/offload/teradata/teradata_column.py +309 -0
  178. goe/offload/teradata/teradata_frontend_api.py +908 -0
  179. goe/offload/teradata/teradata_literal.py +162 -0
  180. goe/offload/teradata/teradata_offload_source_table.py +1192 -0
  181. goe/offload/teradata/teradata_offload_transport_rdbms_api.py +620 -0
  182. goe/offload/teradata/teradata_partition_expression.py +205 -0
  183. goe/offload/teradata/teradata_predicate.py +86 -0
  184. goe/orchestration/__init__.py +0 -0
  185. goe/orchestration/cli_entry_points.py +79 -0
  186. goe/orchestration/command_steps.py +126 -0
  187. goe/orchestration/execution_id.py +101 -0
  188. goe/orchestration/orchestration_constants.py +56 -0
  189. goe/orchestration/orchestration_lock.py +164 -0
  190. goe/orchestration/orchestration_runner.py +539 -0
  191. goe/persistence/__init__.py +0 -0
  192. goe/persistence/factory/__init__.py +0 -0
  193. goe/persistence/factory/orchestration_repo_client_factory.py +48 -0
  194. goe/persistence/oracle/__init__.py +0 -0
  195. goe/persistence/oracle/oracle_orchestration_repo_client.py +1093 -0
  196. goe/persistence/orchestration_metadata.py +449 -0
  197. goe/persistence/orchestration_repo_client.py +436 -0
  198. goe/persistence/teradata/__init__.py +0 -0
  199. goe/persistence/teradata/teradata_orchestration_repo_client.py +358 -0
  200. goe/schema_sync/__init__.py +0 -0
  201. goe/schema_sync/schema_sync.py +170 -0
  202. goe/schema_sync/schema_sync_analyzer.py +663 -0
  203. goe/schema_sync/schema_sync_command_file.py +94 -0
  204. goe/schema_sync/schema_sync_constants.py +38 -0
  205. goe/schema_sync/schema_sync_processor.py +109 -0
  206. goe/schema_sync/schema_sync_step.py +91 -0
  207. goe/schema_sync/steps/__init__.py +0 -0
  208. goe/schema_sync/steps/add_backend_column.py +107 -0
  209. goe/schema_sync/steps/add_oracle_column.py +101 -0
  210. goe/schema_sync/steps/step_builder.py +30 -0
  211. goe/scripts/__init__.py +0 -0
  212. goe/scripts/agg_validate.py +288 -0
  213. goe/util/__init__.py +0 -0
  214. goe/util/avro_encoder.py +258 -0
  215. goe/util/better_impyla.py +1748 -0
  216. goe/util/exception_trigger.py +52 -0
  217. goe/util/goe_log.py +152 -0
  218. goe/util/goe_log_fh.py +67 -0
  219. goe/util/goe_version.py +18 -0
  220. goe/util/hive_ddl_transform.py +396 -0
  221. goe/util/hive_table_stats.py +1590 -0
  222. goe/util/hs2_connection.py +310 -0
  223. goe/util/json_tools.py +60 -0
  224. goe/util/local_cache.py +312 -0
  225. goe/util/misc_functions.py +775 -0
  226. goe/util/ora_query.py +417 -0
  227. goe/util/parallel_exec.py +222 -0
  228. goe/util/parquet_encoder.py +206 -0
  229. goe/util/password_tools.py +261 -0
  230. goe/util/polling_thread.py +97 -0
  231. goe/util/redis_tools.py +443 -0
  232. goe/util/retry_logic.py +165 -0
  233. goe/util/simple_timer.py +49 -0
  234. goe_framework-1.0.6.dist-info/AUTHORS +10 -0
  235. goe_framework-1.0.6.dist-info/LICENSE +201 -0
  236. goe_framework-1.0.6.dist-info/METADATA +210 -0
  237. goe_framework-1.0.6.dist-info/RECORD +239 -0
  238. goe_framework-1.0.6.dist-info/WHEEL +5 -0
  239. goe_framework-1.0.6.dist-info/top_level.txt +1 -0
goe/offload/hadoop/__init__.py ADDED
File without changes