rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
rucio/db/sqla/session.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,19 +18,19 @@ import os
|
|
|
19
18
|
import sys
|
|
20
19
|
from datetime import datetime, timedelta
|
|
21
20
|
from functools import update_wrapper
|
|
22
|
-
from inspect import
|
|
21
|
+
from inspect import getfullargspec, isgeneratorfunction
|
|
23
22
|
from os.path import basename
|
|
24
|
-
from time import sleep
|
|
25
23
|
from threading import Lock
|
|
26
|
-
from
|
|
24
|
+
from time import sleep
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Union
|
|
27
26
|
|
|
28
|
-
from sqlalchemy import create_engine, event,
|
|
29
|
-
from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError,
|
|
30
|
-
from sqlalchemy.orm import DeclarativeBase,
|
|
31
|
-
from sqlalchemy.pool import QueuePool, SingletonThreadPool
|
|
27
|
+
from sqlalchemy import MetaData, create_engine, event, text
|
|
28
|
+
from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, SQLAlchemyError, TimeoutError
|
|
29
|
+
from sqlalchemy.orm import DeclarativeBase, Session, scoped_session, sessionmaker
|
|
30
|
+
from sqlalchemy.pool import NullPool, Pool, QueuePool, SingletonThreadPool
|
|
32
31
|
|
|
33
32
|
from rucio.common.config import config_get
|
|
34
|
-
from rucio.common.exception import
|
|
33
|
+
from rucio.common.exception import DatabaseException, InputValidationError, RucioException
|
|
35
34
|
from rucio.common.extra import import_extras
|
|
36
35
|
from rucio.common.utils import retrying
|
|
37
36
|
|
|
@@ -41,8 +40,14 @@ if TYPE_CHECKING:
|
|
|
41
40
|
from collections.abc import Callable
|
|
42
41
|
from typing import Optional, ParamSpec, TypeVar
|
|
43
42
|
|
|
43
|
+
from pymysql import Connection as MySQLConnection
|
|
44
|
+
from sqlalchemy.engine.base import Engine
|
|
45
|
+
|
|
46
|
+
from rucio.common.types import LoggerFunction
|
|
47
|
+
|
|
44
48
|
P = ParamSpec('P')
|
|
45
49
|
R = TypeVar('R')
|
|
50
|
+
CallableTypeVar = TypeVar('CallableTypeVar', bound='Callable[..., Any]')
|
|
46
51
|
|
|
47
52
|
|
|
48
53
|
try:
|
|
@@ -70,7 +75,7 @@ class BASE(DeclarativeBase):
|
|
|
70
75
|
metadata = _METADATA
|
|
71
76
|
|
|
72
77
|
|
|
73
|
-
def _fk_pragma_on_connect(dbapi_con, con_record):
|
|
78
|
+
def _fk_pragma_on_connect(dbapi_con, con_record) -> None:
|
|
74
79
|
# Hack for previous versions of sqlite3
|
|
75
80
|
try:
|
|
76
81
|
dbapi_con.execute('pragma foreign_keys=ON')
|
|
@@ -78,7 +83,11 @@ def _fk_pragma_on_connect(dbapi_con, con_record):
|
|
|
78
83
|
pass
|
|
79
84
|
|
|
80
85
|
|
|
81
|
-
def mysql_ping_listener(
|
|
86
|
+
def mysql_ping_listener(
|
|
87
|
+
dbapi_conn: "MySQLConnection",
|
|
88
|
+
connection_rec,
|
|
89
|
+
connection_proxy
|
|
90
|
+
) -> None:
|
|
82
91
|
"""
|
|
83
92
|
Ensures that MySQL connections checked out of the
|
|
84
93
|
pool are alive.
|
|
@@ -101,7 +110,9 @@ def mysql_ping_listener(dbapi_conn, connection_rec, connection_proxy):
|
|
|
101
110
|
raise
|
|
102
111
|
|
|
103
112
|
|
|
104
|
-
def mysql_convert_decimal_to_float(
|
|
113
|
+
def mysql_convert_decimal_to_float(
|
|
114
|
+
pymysql: bool = False
|
|
115
|
+
) -> dict[Union[type[object], int], 'Callable[..., Any]']:
|
|
105
116
|
"""
|
|
106
117
|
The default datatype returned by mysql-python for numerics is decimal.Decimal.
|
|
107
118
|
This type cannot be serialised to JSON, therefore we need to autoconvert to floats.
|
|
@@ -111,7 +122,7 @@ def mysql_convert_decimal_to_float(pymysql=False):
|
|
|
111
122
|
:return converter: Converter object
|
|
112
123
|
"""
|
|
113
124
|
|
|
114
|
-
def pymysql_converter():
|
|
125
|
+
def pymysql_converter() -> dict[Union[type[object], int], 'Callable[..., Any]']:
|
|
115
126
|
from pymysql.constants import FIELD_TYPE
|
|
116
127
|
from pymysql.converters import conversions as conv
|
|
117
128
|
converter = conv.copy()
|
|
@@ -138,7 +149,7 @@ def mysql_convert_decimal_to_float(pymysql=False):
|
|
|
138
149
|
return converter
|
|
139
150
|
|
|
140
151
|
|
|
141
|
-
def psql_convert_decimal_to_float(dbapi_conn, connection_rec):
|
|
152
|
+
def psql_convert_decimal_to_float(dbapi_conn, connection_rec) -> None:
|
|
142
153
|
"""
|
|
143
154
|
The default datatype returned by psycopg2 for numerics is decimal.Decimal.
|
|
144
155
|
This type cannot be serialised to JSON, therefore we need to autoconvert to floats.
|
|
@@ -158,8 +169,8 @@ def psql_convert_decimal_to_float(dbapi_conn, connection_rec):
|
|
|
158
169
|
psycopg2.extensions.register_type(DEC2FLOAT)
|
|
159
170
|
|
|
160
171
|
|
|
161
|
-
def my_on_connect(dbapi_con, connection_record):
|
|
162
|
-
""" Adds information to track performance and
|
|
172
|
+
def my_on_connect(dbapi_con, connection_record) -> None:
|
|
173
|
+
""" Adds information to track performance and resource by module.
|
|
163
174
|
Info are recorded in the V$SESSION and V$SQLAREA views.
|
|
164
175
|
"""
|
|
165
176
|
caller = basename(sys.argv[0])
|
|
@@ -169,7 +180,7 @@ def my_on_connect(dbapi_con, connection_record):
|
|
|
169
180
|
dbapi_con.action = caller
|
|
170
181
|
|
|
171
182
|
|
|
172
|
-
def _get_engine_poolclass(poolclass):
|
|
183
|
+
def _get_engine_poolclass(poolclass: str) -> Pool:
|
|
173
184
|
"""Resolve the correct SQLAlchemy Pool type to use from the
|
|
174
185
|
poolclass config option.
|
|
175
186
|
|
|
@@ -192,7 +203,7 @@ def _get_engine_poolclass(poolclass):
|
|
|
192
203
|
return SQLA_CONFIG_POOLCLASS_MAPPING[poolclass]
|
|
193
204
|
|
|
194
205
|
|
|
195
|
-
def get_engine():
|
|
206
|
+
def get_engine() -> 'Engine':
|
|
196
207
|
""" Creates a engine to a specific database.
|
|
197
208
|
:returns: engine
|
|
198
209
|
"""
|
|
@@ -223,11 +234,14 @@ def get_engine():
|
|
|
223
234
|
event.listen(_ENGINE, 'connect', _fk_pragma_on_connect)
|
|
224
235
|
elif 'oracle' in sql_connection:
|
|
225
236
|
event.listen(_ENGINE, 'connect', my_on_connect)
|
|
226
|
-
|
|
237
|
+
if not _ENGINE:
|
|
238
|
+
raise RuntimeError("Could not form database engine.")
|
|
227
239
|
return _ENGINE
|
|
228
240
|
|
|
229
241
|
|
|
230
|
-
def get_dump_engine(
|
|
242
|
+
def get_dump_engine(
|
|
243
|
+
echo: bool = False
|
|
244
|
+
) -> 'Engine':
|
|
231
245
|
""" Creates a dump engine to a specific database.
|
|
232
246
|
:returns: engine """
|
|
233
247
|
|
|
@@ -253,20 +267,21 @@ def get_dump_engine(echo=False):
|
|
|
253
267
|
return engine
|
|
254
268
|
|
|
255
269
|
|
|
256
|
-
def get_maker():
|
|
270
|
+
def get_maker() -> sessionmaker:
|
|
257
271
|
"""
|
|
258
272
|
Return a SQLAlchemy sessionmaker.
|
|
259
273
|
May assign __MAKER if not already assigned.
|
|
260
274
|
"""
|
|
261
275
|
global _MAKER, _ENGINE
|
|
262
|
-
|
|
276
|
+
if not _ENGINE:
|
|
277
|
+
raise RuntimeError("Could not form database engine.")
|
|
263
278
|
if not _MAKER:
|
|
264
279
|
# turn on sqlAlchemy 2.0 with future=True.
|
|
265
280
|
_MAKER = sessionmaker(bind=_ENGINE, autocommit=False, autoflush=True, expire_on_commit=True, future=True)
|
|
266
281
|
return _MAKER
|
|
267
282
|
|
|
268
283
|
|
|
269
|
-
def get_session():
|
|
284
|
+
def get_session() -> scoped_session:
|
|
270
285
|
""" Creates a session to a specific database, assumes that schema already in place.
|
|
271
286
|
:returns: session
|
|
272
287
|
"""
|
|
@@ -278,12 +293,18 @@ def get_session():
|
|
|
278
293
|
get_maker()
|
|
279
294
|
finally:
|
|
280
295
|
_LOCK.release()
|
|
281
|
-
|
|
296
|
+
if not _MAKER:
|
|
297
|
+
raise RuntimeError("Session factory is not defined.")
|
|
282
298
|
session = scoped_session(_MAKER)
|
|
283
299
|
return session
|
|
284
300
|
|
|
285
301
|
|
|
286
|
-
def wait_for_database(
|
|
302
|
+
def wait_for_database(
|
|
303
|
+
timeout: int = 60,
|
|
304
|
+
interval: int = 2,
|
|
305
|
+
*,
|
|
306
|
+
logger: "LoggerFunction" = logging.log
|
|
307
|
+
) -> None:
|
|
287
308
|
""" Wait for the database for a specific amount of time """
|
|
288
309
|
|
|
289
310
|
end_time = datetime.utcnow() + timedelta(seconds=timeout)
|
|
@@ -304,7 +325,7 @@ def wait_for_database(timeout: int = 60, interval: int = 2, *, logger=logging.lo
|
|
|
304
325
|
sleep(interval)
|
|
305
326
|
|
|
306
327
|
|
|
307
|
-
def retry_if_db_connection_error(exception):
|
|
328
|
+
def retry_if_db_connection_error(exception: Exception) -> bool:
|
|
308
329
|
"""Return True if error in connecting to db."""
|
|
309
330
|
if isinstance(exception, (OperationalError, DatabaseException)):
|
|
310
331
|
conn_err_codes = ('2002', '2003', '2006', # MySQL
|
|
@@ -320,11 +341,14 @@ def retry_if_db_connection_error(exception):
|
|
|
320
341
|
return False
|
|
321
342
|
|
|
322
343
|
|
|
323
|
-
def _update_session_wrapper(
|
|
344
|
+
def _update_session_wrapper(
|
|
345
|
+
wrapper: "CallableTypeVar",
|
|
346
|
+
wrapped: 'Callable'
|
|
347
|
+
) -> "CallableTypeVar":
|
|
324
348
|
"""
|
|
325
349
|
In addition to the work done by functools.update_wrapper, this function also preservers
|
|
326
350
|
the signature of the initial function. With the exception that the 'session' parameter
|
|
327
|
-
is
|
|
351
|
+
is overridden to have a default value of 'None'.
|
|
328
352
|
|
|
329
353
|
wrapper is the function to be updated
|
|
330
354
|
wrapped is the original function
|
|
@@ -373,17 +397,17 @@ def read_session(function: "Callable[P, R]"):
|
|
|
373
397
|
if not session:
|
|
374
398
|
session_scoped = get_session()
|
|
375
399
|
session = session_scoped()
|
|
376
|
-
session.begin()
|
|
400
|
+
session.begin() # type: ignore
|
|
377
401
|
try:
|
|
378
402
|
return function(*args, session=session, **kwargs)
|
|
379
403
|
except TimeoutError as error:
|
|
380
|
-
session.rollback() #
|
|
404
|
+
session.rollback() # type: ignore
|
|
381
405
|
raise DatabaseException(str(error))
|
|
382
406
|
except DatabaseError as error:
|
|
383
|
-
session.rollback() #
|
|
407
|
+
session.rollback() # type: ignore
|
|
384
408
|
raise DatabaseException(str(error))
|
|
385
409
|
except:
|
|
386
|
-
session.rollback() #
|
|
410
|
+
session.rollback() # type: ignore
|
|
387
411
|
raise
|
|
388
412
|
finally:
|
|
389
413
|
session_scoped.remove()
|
|
@@ -414,18 +438,18 @@ def stream_session(function: "Callable[P, R]"):
|
|
|
414
438
|
if not session:
|
|
415
439
|
session_scoped = get_session()
|
|
416
440
|
session = session_scoped()
|
|
417
|
-
session.begin()
|
|
441
|
+
session.begin() # type: ignore
|
|
418
442
|
try:
|
|
419
443
|
for row in function(*args, session=session, **kwargs):
|
|
420
444
|
yield row
|
|
421
445
|
except TimeoutError as error:
|
|
422
|
-
session.rollback() #
|
|
446
|
+
session.rollback() # type: ignore
|
|
423
447
|
raise DatabaseException(str(error))
|
|
424
448
|
except DatabaseError as error:
|
|
425
|
-
session.rollback() #
|
|
449
|
+
session.rollback() # type: ignore
|
|
426
450
|
raise DatabaseException(str(error))
|
|
427
451
|
except:
|
|
428
|
-
session.rollback() #
|
|
452
|
+
session.rollback() # type: ignore
|
|
429
453
|
raise
|
|
430
454
|
finally:
|
|
431
455
|
session_scoped.remove()
|
|
@@ -438,29 +462,33 @@ def stream_session(function: "Callable[P, R]"):
|
|
|
438
462
|
return _update_session_wrapper(new_funct, function)
|
|
439
463
|
|
|
440
464
|
|
|
441
|
-
def transactional_session(function: "Callable[P, R]"):
|
|
465
|
+
def transactional_session(function: "Callable[P, R]") -> 'Callable':
|
|
442
466
|
'''
|
|
443
467
|
decorator that set the session variable to use inside a function.
|
|
444
468
|
With that decorator it's possible to use the session variable like if a global variable session is declared.
|
|
445
469
|
|
|
446
470
|
session is a sqlalchemy session, and you can get one calling get_session().
|
|
447
471
|
'''
|
|
448
|
-
def new_funct(
|
|
472
|
+
def new_funct(
|
|
473
|
+
*args: "P.args",
|
|
474
|
+
session: "Optional[Session]" = None,
|
|
475
|
+
**kwargs
|
|
476
|
+
) -> "R": # pylint:disable=missing-kwoa
|
|
449
477
|
if not session:
|
|
450
478
|
session_scoped = get_session()
|
|
451
479
|
session = session_scoped()
|
|
452
|
-
session.begin()
|
|
480
|
+
session.begin() # type: ignore
|
|
453
481
|
try:
|
|
454
482
|
result = function(*args, session=session, **kwargs)
|
|
455
|
-
session.commit() #
|
|
483
|
+
session.commit() # type: ignore
|
|
456
484
|
except TimeoutError as error:
|
|
457
|
-
session.rollback() #
|
|
485
|
+
session.rollback() # type: ignore
|
|
458
486
|
raise DatabaseException(str(error))
|
|
459
487
|
except DatabaseError as error:
|
|
460
|
-
session.rollback() #
|
|
488
|
+
session.rollback() # type: ignore
|
|
461
489
|
raise DatabaseException(str(error))
|
|
462
490
|
except:
|
|
463
|
-
session.rollback() #
|
|
491
|
+
session.rollback() # type: ignore
|
|
464
492
|
raise
|
|
465
493
|
finally:
|
|
466
494
|
session_scoped.remove() # pylint: disable=maybe-no-member
|
rucio/db/sqla/types.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -17,10 +16,10 @@ import uuid
|
|
|
17
16
|
|
|
18
17
|
import sqlalchemy.types as types
|
|
19
18
|
from sqlalchemy.dialects.mysql import BINARY
|
|
20
|
-
from sqlalchemy.dialects.oracle import
|
|
21
|
-
from sqlalchemy.dialects.postgresql import
|
|
19
|
+
from sqlalchemy.dialects.oracle import CLOB, RAW
|
|
20
|
+
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
|
22
21
|
from sqlalchemy.sql import operators
|
|
23
|
-
from sqlalchemy.types import
|
|
22
|
+
from sqlalchemy.types import CHAR, String, TypeDecorator
|
|
24
23
|
|
|
25
24
|
from rucio.common.exception import InvalidType
|
|
26
25
|
from rucio.common.types import InternalAccount, InternalScope
|
rucio/db/sqla/util.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -17,17 +16,17 @@ import logging
|
|
|
17
16
|
from datetime import datetime
|
|
18
17
|
from hashlib import sha256
|
|
19
18
|
from os import urandom
|
|
20
|
-
from typing import TYPE_CHECKING
|
|
19
|
+
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
|
|
21
20
|
|
|
22
21
|
import sqlalchemy
|
|
23
22
|
from alembic import command, op
|
|
24
23
|
from alembic.config import Config
|
|
25
24
|
from dogpile.cache.api import NoValue
|
|
26
|
-
from sqlalchemy import
|
|
25
|
+
from sqlalchemy import Column, PrimaryKeyConstraint, func, inspect
|
|
27
26
|
from sqlalchemy.dialects.postgresql.base import PGInspector
|
|
28
|
-
from sqlalchemy.exc import
|
|
27
|
+
from sqlalchemy.exc import DatabaseError, IntegrityError
|
|
29
28
|
from sqlalchemy.orm import declarative_base
|
|
30
|
-
from sqlalchemy.schema import CreateSchema,
|
|
29
|
+
from sqlalchemy.schema import CreateSchema, CreateTable, DropConstraint, DropTable, ForeignKeyConstraint, MetaData, Table
|
|
31
30
|
from sqlalchemy.sql.ddl import DropSchema
|
|
32
31
|
from sqlalchemy.sql.expression import select, text
|
|
33
32
|
|
|
@@ -35,22 +34,26 @@ from rucio import alembicrevision
|
|
|
35
34
|
from rucio.common.cache import make_region_memcached
|
|
36
35
|
from rucio.common.config import config_get, config_get_list
|
|
37
36
|
from rucio.common.schema import get_schema_value
|
|
38
|
-
from rucio.common.types import InternalAccount
|
|
37
|
+
from rucio.common.types import InternalAccount, LoggerFunction
|
|
39
38
|
from rucio.common.utils import generate_uuid
|
|
40
39
|
from rucio.db.sqla import models
|
|
41
40
|
from rucio.db.sqla.constants import AccountStatus, AccountType, IdentityType
|
|
42
|
-
from rucio.db.sqla.session import get_engine, get_session
|
|
41
|
+
from rucio.db.sqla.session import get_dump_engine, get_engine, get_session
|
|
43
42
|
from rucio.db.sqla.types import InternalScopeString, String
|
|
44
43
|
|
|
45
44
|
if TYPE_CHECKING:
|
|
46
|
-
from
|
|
47
|
-
|
|
48
|
-
from sqlalchemy.engine import Inspector
|
|
45
|
+
from collections.abc import Sequence
|
|
46
|
+
|
|
47
|
+
from sqlalchemy.engine import Inspector
|
|
48
|
+
from sqlalchemy.orm import Query, Session
|
|
49
|
+
|
|
50
|
+
# TypeVar representing the DeclarativeObj class defined inside _create_temp_table
|
|
51
|
+
DeclarativeObj = TypeVar('DeclarativeObj')
|
|
49
52
|
|
|
50
53
|
REGION = make_region_memcached(expiration_time=600, memcached_expire_time=3660)
|
|
51
54
|
|
|
52
55
|
|
|
53
|
-
def build_database():
|
|
56
|
+
def build_database() -> None:
|
|
54
57
|
""" Applies the schema to the database. Run this command once to build the database. """
|
|
55
58
|
engine = get_engine()
|
|
56
59
|
|
|
@@ -71,13 +74,13 @@ def build_database():
|
|
|
71
74
|
command.stamp(alembic_cfg, "head")
|
|
72
75
|
|
|
73
76
|
|
|
74
|
-
def dump_schema():
|
|
77
|
+
def dump_schema() -> None:
|
|
75
78
|
""" Creates a schema dump to a specific database. """
|
|
76
79
|
engine = get_dump_engine()
|
|
77
80
|
models.register_models(engine)
|
|
78
81
|
|
|
79
82
|
|
|
80
|
-
def destroy_database():
|
|
83
|
+
def destroy_database() -> None:
|
|
81
84
|
""" Removes the schema from the database. Only useful for test cases or malicious intents. """
|
|
82
85
|
engine = get_engine()
|
|
83
86
|
|
|
@@ -87,7 +90,7 @@ def destroy_database():
|
|
|
87
90
|
print('Cannot destroy schema -- assuming already gone, continuing:', e)
|
|
88
91
|
|
|
89
92
|
|
|
90
|
-
def drop_everything():
|
|
93
|
+
def drop_everything() -> None:
|
|
91
94
|
"""
|
|
92
95
|
Pre-gather all named constraints and table names, and drop everything.
|
|
93
96
|
This is better than using metadata.reflect(); metadata.drop_all()
|
|
@@ -101,7 +104,7 @@ def drop_everything():
|
|
|
101
104
|
|
|
102
105
|
with engine.connect() as conn:
|
|
103
106
|
|
|
104
|
-
inspector
|
|
107
|
+
inspector: Union["Inspector", PGInspector] = inspect(conn)
|
|
105
108
|
|
|
106
109
|
for tname, fkcs in reversed(
|
|
107
110
|
inspector.get_sorted_table_and_fkc_names(schema='*')):
|
|
@@ -122,12 +125,13 @@ def drop_everything():
|
|
|
122
125
|
conn.execute(DropSchema(schema, cascade=True))
|
|
123
126
|
|
|
124
127
|
if engine.dialect.name == 'postgresql':
|
|
125
|
-
|
|
128
|
+
if not isinstance(inspector, PGInspector):
|
|
129
|
+
raise ValueError('expected a PGInspector')
|
|
126
130
|
for enum in inspector.get_enums(schema='*'):
|
|
127
131
|
sqlalchemy.Enum(**enum).drop(bind=conn)
|
|
128
132
|
|
|
129
133
|
|
|
130
|
-
def create_base_vo():
|
|
134
|
+
def create_base_vo() -> None:
|
|
131
135
|
""" Creates the base VO """
|
|
132
136
|
|
|
133
137
|
session_scoped = get_session()
|
|
@@ -138,40 +142,28 @@ def create_base_vo():
|
|
|
138
142
|
s.add_all([vo])
|
|
139
143
|
|
|
140
144
|
|
|
141
|
-
def create_root_account():
|
|
145
|
+
def create_root_account() -> None:
|
|
142
146
|
"""
|
|
143
147
|
Inserts the default root account to an existing database. Make sure to change the default password later.
|
|
144
148
|
"""
|
|
145
149
|
|
|
146
150
|
multi_vo = bool(config_get('common', 'multi_vo', False, False))
|
|
147
151
|
|
|
148
|
-
up_id = 'ddmlab'
|
|
149
|
-
up_pwd = 'secret'
|
|
150
|
-
up_email = 'ph-adp-ddm-lab@cern.ch'
|
|
151
|
-
x509_id = 'emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH'
|
|
152
|
-
x509_email = 'ph-adp-ddm-lab@cern.ch'
|
|
153
|
-
gss_id = 'ddmlab@CERN.CH'
|
|
154
|
-
gss_email = 'ph-adp-ddm-lab@cern.ch'
|
|
155
|
-
ssh_id = '
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
up_id = config_get('bootstrap', 'userpass_identity')
|
|
164
|
-
up_pwd = config_get('bootstrap', 'userpass_pwd')
|
|
165
|
-
up_email = config_get('bootstrap', 'userpass_email')
|
|
166
|
-
x509_id = config_get('bootstrap', 'x509_identity')
|
|
167
|
-
x509_email = config_get('bootstrap', 'x509_email')
|
|
168
|
-
gss_id = config_get('bootstrap', 'gss_identity')
|
|
169
|
-
gss_email = config_get('bootstrap', 'gss_email')
|
|
170
|
-
ssh_id = config_get('bootstrap', 'ssh_identity')
|
|
171
|
-
ssh_email = config_get('bootstrap', 'ssh_email')
|
|
172
|
-
except:
|
|
173
|
-
pass
|
|
174
|
-
# print 'Config values are missing (check rucio.cfg{.template}). Using hardcoded defaults.'
|
|
152
|
+
up_id = config_get('bootstrap', 'userpass_identity', default='ddmlab')
|
|
153
|
+
up_pwd = config_get('bootstrap', 'userpass_pwd', default='secret')
|
|
154
|
+
up_email = config_get('bootstrap', 'userpass_email', default='ph-adp-ddm-lab@cern.ch')
|
|
155
|
+
x509_id = config_get('bootstrap', 'x509_identity', default='emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH')
|
|
156
|
+
x509_email = config_get('bootstrap', 'x509_email', default='ph-adp-ddm-lab@cern.ch')
|
|
157
|
+
gss_id = config_get('bootstrap', 'gss_identity', default='ddmlab@CERN.CH')
|
|
158
|
+
gss_email = config_get('bootstrap', 'gss_email', default='ph-adp-ddm-lab@cern.ch')
|
|
159
|
+
ssh_id = config_get('bootstrap', 'ssh_identity',
|
|
160
|
+
default='ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/'
|
|
161
|
+
'0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydn'
|
|
162
|
+
'Vy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2'
|
|
163
|
+
'iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5'
|
|
164
|
+
'Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab'
|
|
165
|
+
)
|
|
166
|
+
ssh_email = config_get('bootstrap', 'ssh_email', default='ph-adp-ddm-lab@cern.ch')
|
|
175
167
|
|
|
176
168
|
session_scoped = get_session()
|
|
177
169
|
|
|
@@ -185,25 +177,41 @@ def create_root_account():
|
|
|
185
177
|
salt = urandom(255)
|
|
186
178
|
salted_password = salt + up_pwd.encode()
|
|
187
179
|
hashed_password = sha256(salted_password).hexdigest()
|
|
188
|
-
|
|
189
|
-
|
|
180
|
+
|
|
181
|
+
identities = []
|
|
182
|
+
associations = []
|
|
183
|
+
|
|
184
|
+
if up_id and up_pwd:
|
|
185
|
+
identity1 = models.Identity(identity=up_id, identity_type=IdentityType.USERPASS, password=hashed_password, salt=salt, email=up_email)
|
|
186
|
+
iaa1 = models.IdentityAccountAssociation(identity=identity1.identity, identity_type=identity1.identity_type, account=account.account, is_default=True)
|
|
187
|
+
identities.append(identity1)
|
|
188
|
+
associations.append(iaa1)
|
|
190
189
|
|
|
191
190
|
# X509 authentication
|
|
192
|
-
|
|
193
|
-
|
|
191
|
+
if x509_id and x509_email:
|
|
192
|
+
identity2 = models.Identity(identity=x509_id, identity_type=IdentityType.X509, email=x509_email)
|
|
193
|
+
iaa2 = models.IdentityAccountAssociation(identity=identity2.identity, identity_type=identity2.identity_type, account=account.account, is_default=True)
|
|
194
|
+
identities.append(identity2)
|
|
195
|
+
associations.append(iaa2)
|
|
194
196
|
|
|
195
197
|
# GSS authentication
|
|
196
|
-
|
|
197
|
-
|
|
198
|
+
if gss_id and gss_email:
|
|
199
|
+
identity3 = models.Identity(identity=gss_id, identity_type=IdentityType.GSS, email=gss_email)
|
|
200
|
+
iaa3 = models.IdentityAccountAssociation(identity=identity3.identity, identity_type=identity3.identity_type, account=account.account, is_default=True)
|
|
201
|
+
identities.append(identity3)
|
|
202
|
+
associations.append(iaa3)
|
|
198
203
|
|
|
199
204
|
# SSH authentication
|
|
200
|
-
|
|
201
|
-
|
|
205
|
+
if ssh_id and ssh_email:
|
|
206
|
+
identity4 = models.Identity(identity=ssh_id, identity_type=IdentityType.SSH, email=ssh_email)
|
|
207
|
+
iaa4 = models.IdentityAccountAssociation(identity=identity4.identity, identity_type=identity4.identity_type, account=account.account, is_default=True)
|
|
208
|
+
identities.append(identity4)
|
|
209
|
+
associations.append(iaa4)
|
|
202
210
|
|
|
203
211
|
with session_scoped() as s:
|
|
204
212
|
s.begin()
|
|
205
213
|
# Apply
|
|
206
|
-
for identity in
|
|
214
|
+
for identity in identities:
|
|
207
215
|
try:
|
|
208
216
|
s.add(identity)
|
|
209
217
|
s.commit()
|
|
@@ -212,11 +220,11 @@ def create_root_account():
|
|
|
212
220
|
s.rollback()
|
|
213
221
|
s.add(account)
|
|
214
222
|
s.flush()
|
|
215
|
-
s.add_all(
|
|
223
|
+
s.add_all(associations)
|
|
216
224
|
s.commit()
|
|
217
225
|
|
|
218
226
|
|
|
219
|
-
def get_db_time():
|
|
227
|
+
def get_db_time() -> Optional[datetime]:
|
|
220
228
|
""" Gives the utc time on the db. """
|
|
221
229
|
session_scoped = get_session()
|
|
222
230
|
try:
|
|
@@ -241,7 +249,7 @@ def get_db_time():
|
|
|
241
249
|
session_scoped.remove()
|
|
242
250
|
|
|
243
251
|
|
|
244
|
-
def get_count(q):
|
|
252
|
+
def get_count(q: "Query") -> int:
|
|
245
253
|
"""
|
|
246
254
|
Fast way to get count in SQLAlchemy
|
|
247
255
|
Source: https://gist.github.com/hest/8798884
|
|
@@ -253,7 +261,7 @@ def get_count(q):
|
|
|
253
261
|
return count
|
|
254
262
|
|
|
255
263
|
|
|
256
|
-
def is_old_db():
|
|
264
|
+
def is_old_db() -> bool:
|
|
257
265
|
"""
|
|
258
266
|
Returns true, if alembic is used and the database is not on the
|
|
259
267
|
same revision as the code base.
|
|
@@ -274,7 +282,7 @@ def is_old_db():
|
|
|
274
282
|
return (len(query) != 0 and str(query[0].version_num) != alembicrevision.ALEMBIC_REVISION)
|
|
275
283
|
|
|
276
284
|
|
|
277
|
-
def json_implemented(*, session=None):
|
|
285
|
+
def json_implemented(*, session: Optional["Session"] = None) -> bool:
|
|
278
286
|
"""
|
|
279
287
|
Checks if the database on the current server installation can support json fields.
|
|
280
288
|
|
|
@@ -295,7 +303,7 @@ def json_implemented(*, session=None):
|
|
|
295
303
|
return True
|
|
296
304
|
|
|
297
305
|
|
|
298
|
-
def try_drop_constraint(constraint_name, table_name):
|
|
306
|
+
def try_drop_constraint(constraint_name: str, table_name: str) -> None:
|
|
299
307
|
"""
|
|
300
308
|
Tries to drop the given constrained and returns successfully if the
|
|
301
309
|
constraint already existed on Oracle databases.
|
|
@@ -306,10 +314,11 @@ def try_drop_constraint(constraint_name, table_name):
|
|
|
306
314
|
try:
|
|
307
315
|
op.drop_constraint(constraint_name, table_name)
|
|
308
316
|
except DatabaseError as e:
|
|
309
|
-
|
|
317
|
+
if 'nonexistent constraint' not in str(e):
|
|
318
|
+
raise RuntimeError(e)
|
|
310
319
|
|
|
311
320
|
|
|
312
|
-
def list_oracle_global_temp_tables(session):
|
|
321
|
+
def list_oracle_global_temp_tables(session: "Session") -> list[str]:
|
|
313
322
|
"""
|
|
314
323
|
Retrieve the list of global temporary tables in oracle
|
|
315
324
|
"""
|
|
@@ -338,7 +347,14 @@ def list_oracle_global_temp_tables(session):
|
|
|
338
347
|
return global_temp_tables
|
|
339
348
|
|
|
340
349
|
|
|
341
|
-
def _create_temp_table(
|
|
350
|
+
def _create_temp_table(
|
|
351
|
+
name: str,
|
|
352
|
+
*columns: "Sequence[Column]",
|
|
353
|
+
primary_key: Optional["Sequence[Any]"] = None,
|
|
354
|
+
oracle_global_name: Optional[str] = None,
|
|
355
|
+
session: Optional["Session"] = None,
|
|
356
|
+
logger: LoggerFunction = logging.log
|
|
357
|
+
) -> type["DeclarativeObj"]:
|
|
342
358
|
"""
|
|
343
359
|
Create a temporary table with the given columns, register it into a declarative base, and return it.
|
|
344
360
|
|
|
@@ -366,7 +382,7 @@ def _create_temp_table(name, *columns, primary_key=None, oracle_global_name=None
|
|
|
366
382
|
oracle_table_is_global = False
|
|
367
383
|
if session.bind.dialect.name == 'oracle':
|
|
368
384
|
# Retrieve the list of global temporary tables on oracle.
|
|
369
|
-
# If the requested table is found to be global,
|
|
385
|
+
# If the requested table is found to be global, reuse it,
|
|
370
386
|
# otherwise create a private temporary table with random name
|
|
371
387
|
global_temp_tables = list_oracle_global_temp_tables(session=session)
|
|
372
388
|
if oracle_global_name is None:
|
|
@@ -452,18 +468,24 @@ class TempTableManager:
|
|
|
452
468
|
sessions in multiple threads at a time, so no need to protect indexes with a mutex.
|
|
453
469
|
"""
|
|
454
470
|
|
|
455
|
-
def __init__(self, session):
|
|
471
|
+
def __init__(self, session: "Session"):
|
|
456
472
|
self.session = session
|
|
457
473
|
|
|
458
474
|
self.next_idx_to_use = {}
|
|
459
475
|
|
|
460
|
-
def create_temp_table(
|
|
476
|
+
def create_temp_table(
|
|
477
|
+
self,
|
|
478
|
+
name: str,
|
|
479
|
+
*columns: "Sequence[Column]",
|
|
480
|
+
primary_key: Optional["Sequence[Any]"] = None,
|
|
481
|
+
logger: LoggerFunction = logging.log
|
|
482
|
+
) -> type["DeclarativeObj"]:
|
|
461
483
|
idx = self.next_idx_to_use.setdefault(name, 0)
|
|
462
484
|
table = _create_temp_table(f'{name}_{idx}', *columns, primary_key=primary_key, session=self.session, logger=logger)
|
|
463
485
|
self.next_idx_to_use[name] = idx + 1
|
|
464
486
|
return table
|
|
465
487
|
|
|
466
|
-
def create_scope_name_table(self, logger=logging.log):
|
|
488
|
+
def create_scope_name_table(self, logger: LoggerFunction = logging.log) -> type["DeclarativeObj"]:
|
|
467
489
|
"""
|
|
468
490
|
Create a temporary table with columns 'scope' and 'name'
|
|
469
491
|
"""
|
|
@@ -479,7 +501,7 @@ class TempTableManager:
|
|
|
479
501
|
logger=logger,
|
|
480
502
|
)
|
|
481
503
|
|
|
482
|
-
def create_association_table(self, logger=logging.log):
|
|
504
|
+
def create_association_table(self, logger: LoggerFunction = logging.log) -> type["DeclarativeObj"]:
|
|
483
505
|
"""
|
|
484
506
|
Create a temporary table with columns 'scope', 'name', 'child_scope'and 'child_name'
|
|
485
507
|
"""
|
|
@@ -497,7 +519,7 @@ class TempTableManager:
|
|
|
497
519
|
logger=logger,
|
|
498
520
|
)
|
|
499
521
|
|
|
500
|
-
def create_id_table(self, logger=logging.log):
|
|
522
|
+
def create_id_table(self, logger: LoggerFunction = logging.log) -> type["DeclarativeObj"]:
|
|
501
523
|
"""
|
|
502
524
|
Create a temp table with a single id column of uuid type
|
|
503
525
|
"""
|