edu-rdm-integration 3.3.5__tar.gz → 3.3.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/CHANGELOG.md +15 -0
  2. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/PKG-INFO +16 -1
  3. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/setup.py +1 -1
  4. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/export.py +116 -54
  5. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/export_manger.py +36 -9
  6. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/helpers.py +16 -13
  7. edu-rdm-integration-3.3.7/src/edu_rdm_integration/migrations/0014_uploaddatacommand.py +43 -0
  8. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/models.py +50 -0
  9. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/tasks.py +27 -15
  10. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/PKG-INFO +16 -1
  11. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/SOURCES.txt +1 -0
  12. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/CONDUCT.md +0 -0
  13. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/LICENSE +0 -0
  14. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/MANIFEST.in +0 -0
  15. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/README.md +0 -0
  16. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/pyproject.toml +0 -0
  17. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/development_packages.txt +0 -0
  18. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/external_packages.txt +0 -0
  19. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/internal_packages.txt +0 -0
  20. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/internal_sources.txt +0 -0
  21. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/production_packages.txt +0 -0
  22. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/system_packages.txt +0 -0
  23. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/dependencies/testing_packages.txt +0 -0
  24. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/development.txt +0 -0
  25. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/production.txt +0 -0
  26. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/requirements/stage.txt +0 -0
  27. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/setup.cfg +0 -0
  28. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/__init__.py +0 -0
  29. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/__init__.py +0 -0
  30. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/apps.py +0 -0
  31. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/caches.py +0 -0
  32. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/consts.py +0 -0
  33. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/enums.py +0 -0
  34. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/errors.py +0 -0
  35. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/functions.py +0 -0
  36. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/helpers.py +0 -0
  37. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/managers.py +0 -0
  38. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/presenters.py +0 -0
  39. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/receivers.py +0 -0
  40. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/results.py +0 -0
  41. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/runners.py +0 -0
  42. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/strategies.py +0 -0
  43. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/strings.py +0 -0
  44. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/tests.py +0 -0
  45. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/adapters/validators.py +0 -0
  46. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/app_meta.py +0 -0
  47. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/app_settings.py +0 -0
  48. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/apps.py +0 -0
  49. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/base.py +0 -0
  50. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_and_export_data/__init__.py +0 -0
  51. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_and_export_data/apps.py +0 -0
  52. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_and_export_data/migrations/0001_initial.py +0 -0
  53. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_and_export_data/migrations/__init__.py +0 -0
  54. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_and_export_data/models.py +0 -0
  55. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_and_export_data/utils.py +0 -0
  56. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/__init__.py +0 -0
  57. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/__init__.py +0 -0
  58. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/caches.py +0 -0
  59. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/functions.py +0 -0
  60. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/helpers.py +0 -0
  61. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/managers.py +0 -0
  62. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/mixins.py +0 -0
  63. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/base/runners.py +0 -0
  64. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/__init__.py +0 -0
  65. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/__init__.py +0 -0
  66. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/caches.py +0 -0
  67. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/consts.py +0 -0
  68. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/enums.py +0 -0
  69. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/errors.py +0 -0
  70. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/functions.py +0 -0
  71. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/helpers.py +0 -0
  72. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/managers.py +0 -0
  73. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/presenters.py +0 -0
  74. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/results.py +0 -0
  75. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/runners.py +0 -0
  76. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/strings.py +0 -0
  77. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/tests.py +0 -0
  78. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/base/validators.py +0 -0
  79. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/calculated/strategies.py +0 -0
  80. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/collect.py +0 -0
  81. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/generators.py +0 -0
  82. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/helpers.py +0 -0
  83. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/__init__.py +0 -0
  84. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/__init__.py +0 -0
  85. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/caches.py +0 -0
  86. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/consts.py +0 -0
  87. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/enums.py +0 -0
  88. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/errors.py +0 -0
  89. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/functions.py +0 -0
  90. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/helpers.py +0 -0
  91. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/managers.py +0 -0
  92. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/presenters.py +0 -0
  93. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/results.py +0 -0
  94. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/runners.py +0 -0
  95. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/strings.py +0 -0
  96. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/tests.py +0 -0
  97. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/base/validators.py +0 -0
  98. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/non_calculated/strategies.py +0 -0
  99. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/collect_data/tests.py +0 -0
  100. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/consts.py +0 -0
  101. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/entities.py +0 -0
  102. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/enum_register/__init__.py +0 -0
  103. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/enum_register/mixins.py +0 -0
  104. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/enum_register/register.py +0 -0
  105. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/enums.py +0 -0
  106. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/__init__.py +0 -0
  107. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/__init__.py +0 -0
  108. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/caches.py +0 -0
  109. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/consts.py +0 -0
  110. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/enums.py +0 -0
  111. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/errors.py +0 -0
  112. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/functions.py +0 -0
  113. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/helpers.py +0 -0
  114. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/managers.py +0 -0
  115. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/presenters.py +0 -0
  116. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/requests.py +0 -0
  117. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/results.py +0 -0
  118. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/runners.py +0 -0
  119. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/strings.py +0 -0
  120. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/tests.py +0 -0
  121. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/base/validators.py +0 -0
  122. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/consts.py +0 -0
  123. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/dataclasses.py +0 -0
  124. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/generators.py +0 -0
  125. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/helpers.py +0 -0
  126. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/queue.py +0 -0
  127. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/export_data/strategies.py +0 -0
  128. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/__init__.py +0 -0
  129. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/__init__.py-tpl +0 -0
  130. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/apps.py-tpl +0 -0
  131. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/caches.py-tpl +0 -0
  132. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/consts.py-tpl +0 -0
  133. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/enums.py-tpl +0 -0
  134. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/errors.py-tpl +0 -0
  135. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/functions.py-tpl +0 -0
  136. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/helpers.py-tpl +0 -0
  137. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/managers.py-tpl +0 -0
  138. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/presenters.py-tpl +0 -0
  139. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/results.py-tpl +0 -0
  140. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/runners.py-tpl +0 -0
  141. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/strings.py-tpl +0 -0
  142. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/tests.py-tpl +0 -0
  143. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_collect_data_template/validators.py-tpl +0 -0
  144. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/__init__.py-tpl +0 -0
  145. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/apps.py-tpl +0 -0
  146. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/caches.py-tpl +0 -0
  147. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/consts.py-tpl +0 -0
  148. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/enums.py-tpl +0 -0
  149. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/errors.py-tpl +0 -0
  150. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/functions.py-tpl +0 -0
  151. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/helpers.py-tpl +0 -0
  152. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/managers.py-tpl +0 -0
  153. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/presenters.py-tpl +0 -0
  154. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/results.py-tpl +0 -0
  155. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/runners.py-tpl +0 -0
  156. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/strings.py-tpl +0 -0
  157. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/tests.py-tpl +0 -0
  158. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/function_templates/function_export_data_template/validators.py-tpl +0 -0
  159. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/__init__.py +0 -0
  160. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/__init__.py +0 -0
  161. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/check_upload_status.py +0 -0
  162. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/collect_latest_models_data.py +0 -0
  163. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/collect_models_data.py +0 -0
  164. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/datamart_status.py +0 -0
  165. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/datamart_upload.py +0 -0
  166. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/export_entities_data.py +0 -0
  167. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/commands/export_latest_entities_data.py +0 -0
  168. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/management/general.py +0 -0
  169. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/mapping.py +0 -0
  170. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0001_initial.py +0 -0
  171. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0002_init_data_uploadstatus.py +0 -0
  172. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0003_create_index_file_upload_status.py +0 -0
  173. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0004_uploaderclientlog.py +0 -0
  174. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0005_auto_20231204_1224.py +0 -0
  175. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0006_request_status_data.py +0 -0
  176. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0007_delete_upload_status.py +0 -0
  177. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0008_transferredentity.py +0 -0
  178. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0009_auto_20240522_1619.py +0 -0
  179. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0010_transferredentity_export_enabled.py +0 -0
  180. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py +0 -0
  181. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py +0 -0
  182. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/0013_set_attachment_size.py +0 -0
  183. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/migrations/__init__.py +0 -0
  184. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/redis_cache.py +0 -0
  185. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/registry/__init__.py +0 -0
  186. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/registry/actions.py +0 -0
  187. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/registry/ui.py +0 -0
  188. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/signals.py +0 -0
  189. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/storages.py +0 -0
  190. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/templates/ui-js/transferred-entity-list.js +0 -0
  191. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/__init__.py +0 -0
  192. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/actions.py +0 -0
  193. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/apps.py +0 -0
  194. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/enums.py +0 -0
  195. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/managers.py +0 -0
  196. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/templates/ui-js/object-grid-buttons.js +0 -0
  197. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/uploader_log/ui.py +0 -0
  198. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration/utils.py +0 -0
  199. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/dependency_links.txt +0 -0
  200. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/namespace_packages.txt +0 -0
  201. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/not-zip-safe +0 -0
  202. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/requires.txt +0 -0
  203. {edu-rdm-integration-3.3.5 → edu-rdm-integration-3.3.7}/src/edu_rdm_integration.egg-info/top_level.txt +0 -0
@@ -29,6 +29,21 @@
29
29
  ### Удалено
30
30
 
31
31
 
32
+ ## 3.3.7 - 2024-09-06
33
+ Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных в витрину
34
+ - [EDUSCHL-22042](https://jira.bars.group/browse/EDUSCHL-22042)
35
+ PATCH Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных
36
+ в витрину
37
+
38
+
39
+ ## 3.3.6 - 2024-09-02
40
+ Добавлена фильтрация по дате выгрузки при обновлении данных на экспорт в BaseExportLatestEntitiesData
41
+
42
+ ### Изменено
43
+ - [EDUSCHL-22335](https://jira.bars.group/browse/EDUSCHL-22335)
44
+ PATCH Добавлена фильтрация по дате выгрузки при обновлении данных на экспорт в BaseExportLatestEntitiesData
45
+
46
+
32
47
  ## 3.3.5 - 2024-08-26
33
48
  Классы UniquePeriodicAsyncTask, PeriodicTaskLocker перенесены в educommon.
34
49
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: edu-rdm-integration
3
- Version: 3.3.5
3
+ Version: 3.3.7
4
4
  Summary: Интеграция с Региональной витриной данных
5
5
  Home-page:
6
6
  Download-URL:
@@ -302,6 +302,21 @@ Requires-Dist: uploader-client<1,>=0.2.1
302
302
  ### Удалено
303
303
 
304
304
 
305
+ ## 3.3.7 - 2024-09-06
306
+ Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных в витрину
307
+ - [EDUSCHL-22042](https://jira.bars.group/browse/EDUSCHL-22042)
308
+ PATCH Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных
309
+ в витрину
310
+
311
+
312
+ ## 3.3.6 - 2024-09-02
313
+ Добавлена фильтрация по дате выгрузки при обновлении данных на экспорт в BaseExportLatestEntitiesData
314
+
315
+ ### Изменено
316
+ - [EDUSCHL-22335](https://jira.bars.group/browse/EDUSCHL-22335)
317
+ PATCH Добавлена фильтрация по дате выгрузки при обновлении данных на экспорт в BaseExportLatestEntitiesData
318
+
319
+
305
320
  ## 3.3.5 - 2024-08-26
306
321
  Классы UniquePeriodicAsyncTask, PeriodicTaskLocker перенесены в educommon.
307
322
 
@@ -17,7 +17,7 @@ from setuptools import (
17
17
 
18
18
  PROJECT = 'edu_rdm_integration'
19
19
 
20
- VERSION = '3.3.5'
20
+ VERSION = '3.3.7'
21
21
 
22
22
  current_dir_path = Path().resolve()
23
23
 
@@ -15,6 +15,7 @@ from typing import (
15
15
  List,
16
16
  Set,
17
17
  Type,
18
+ Union,
18
19
  )
19
20
 
20
21
  from django.conf import (
@@ -69,6 +70,13 @@ from edu_rdm_integration.consts import (
69
70
  from edu_rdm_integration.enums import (
70
71
  FileUploadStatusEnum,
71
72
  )
73
+ from edu_rdm_integration.export_data.export_manger import (
74
+ ExportEntityQueueSender,
75
+ WorkerSender,
76
+ )
77
+ from edu_rdm_integration.export_data.queue import (
78
+ Queue,
79
+ )
72
80
  from edu_rdm_integration.helpers import (
73
81
  get_exporting_managers_max_period_ended_dates,
74
82
  )
@@ -77,6 +85,9 @@ from edu_rdm_integration.models import (
77
85
  ExportingDataSubStageStatus,
78
86
  RegionalDataMartEntityEnum,
79
87
  )
88
+ from edu_rdm_integration.redis_cache import (
89
+ AbstractCache,
90
+ )
80
91
  from edu_rdm_integration.signals import (
81
92
  manager_created,
82
93
  )
@@ -121,8 +132,6 @@ class BaseExportEntitiesData(BaseOperationData):
121
132
  # Карта соответствия manager_id сущности и его основной модели
122
133
  self.manager_main_model_map: Dict[str, ModelBase] = {}
123
134
 
124
- self._configure_agent_client()
125
-
126
135
  @property
127
136
  def _log_file_path(self) -> str:
128
137
  """
@@ -130,57 +139,6 @@ class BaseExportEntitiesData(BaseOperationData):
130
139
  """
131
140
  return os.path.join(settings.MEDIA_ROOT, settings.RDM_EXPORT_LOG_DIR, f'{self.command_id}.log')
132
141
 
133
- def _configure_agent_client(self):
134
- """
135
- Конфигурирование клиента загрузчика данных в Витрину.
136
-
137
- #TODO Вынужденная мера, т.к. при запуске команды не производится проверка готовности конфигов приложений.
138
- # Нужно переработать механизм конфигурирования клиента загрузчика.
139
- """
140
- import uploader_client
141
- from django.core.cache import (
142
- DEFAULT_CACHE_ALIAS,
143
- caches,
144
- )
145
- from uploader_client.contrib.rdm.interfaces.configurations import (
146
- RegionalDataMartUploaderConfig,
147
- )
148
- if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
149
- uploader_client.set_config(
150
- RegionalDataMartUploaderConfig(
151
- interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
152
- url=settings.RDM_UPLOADER_CLIENT_URL,
153
- datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
154
- timeout=1,
155
- request_retries=1,
156
- )
157
- )
158
- elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
159
- uploader_client.set_config(
160
- RegionalDataMartUploaderConfig(
161
- interface='uploader_client.contrib.rdm.interfaces.rest.ProxyAPIInterface',
162
- cache=caches[DEFAULT_CACHE_ALIAS],
163
- url=settings.RDM_UPLOADER_CLIENT_URL,
164
- datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
165
- timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
166
- request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
167
- organization_ogrn=settings.RDM_UPLOADER_CLIENT_ORGANIZATION_OGRN,
168
- installation_name=settings.RDM_UPLOADER_CLIENT_INSTALLATION_NAME,
169
- installation_id=settings.RDM_UPLOADER_CLIENT_INSTALLATION_ID,
170
- username=settings.RDM_UPLOADER_CLIENT_USERNAME,
171
- password=settings.RDM_UPLOADER_CLIENT_PASSWORD,
172
- )
173
- )
174
- else:
175
- uploader_client.set_config(
176
- RegionalDataMartUploaderConfig(
177
- url=settings.RDM_UPLOADER_CLIENT_URL,
178
- datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
179
- timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
180
- request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
181
- )
182
- )
183
-
184
142
  def _has_stage_created_or_in_progress(self, manager_id: str, entity: str) -> bool:
185
143
  """Проверяет есть ли готовый к работе stage или в работе для данной сущности."""
186
144
  stage_created_or_in_progress = ExportingDataStage.objects.filter(
@@ -342,13 +300,15 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
342
300
  # Связь лога подэтапа выгрузки с логом загрузки данных (ExportingDataSubStageUploaderClientLog)
343
301
  # имеет статус FileUploadStatusEnum.ERROR:
344
302
  model.objects.filter(
303
+ exporting_sub_stage__ended_at__gt=now - timedelta(days=365),
304
+ exporting_sub_stage__exportingdatasubstageuploaderclientlog__created__lt=last_finished_export_data,
345
305
  exporting_sub_stage__exportingdatasubstageuploaderclientlog__file_upload_status=(
346
306
  FileUploadStatusEnum.ERROR),
347
307
  ),
348
308
  )
349
309
 
350
310
  for queryset in querysets_to_update:
351
- not_exported_model_ids = queryset.values_list('id', flat=True)
311
+ not_exported_model_ids = queryset.values_list('id', flat=True).iterator()
352
312
 
353
313
  with atomic():
354
314
 
@@ -412,3 +372,105 @@ class ExportLatestEntitiesData(BaseExportLatestEntitiesData):
412
372
  def _get_async_task(self) -> Model:
413
373
  """Возвращает модель асинхронной задачи."""
414
374
  return RunningTask
375
+
376
+
377
+ class UploadEntitiesData(BaseOperationData):
378
+ """Класс отправки файлов с сущностями в витрину."""
379
+
380
+ def __init__(
381
+ self,
382
+ entities: Iterable[str],
383
+ data_cache: AbstractCache,
384
+ queue: Queue,
385
+ **kwargs,
386
+ ):
387
+ super().__init__(**kwargs)
388
+
389
+ # Если сущности не указаны, берется значение по умолчанию - все сущности:
390
+ self.entities = entities if entities else RegionalDataMartEntityEnum.get_enum_data().keys()
391
+
392
+ self.data_cache = data_cache
393
+ self.queue = queue
394
+
395
+ self.task_id = kwargs.get('task_id')
396
+
397
+ self._configure_agent_client()
398
+
399
+ @property
400
+ def _log_file_path(self) -> Union[str, bytes]:
401
+ """
402
+ Путь до лог файла.
403
+ """
404
+ if self.command_id:
405
+ log_file_path = os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, f'{self.command_id}.log')
406
+ else:
407
+ log_file_path = os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, 'upload_entity.log')
408
+
409
+ return log_file_path
410
+
411
+ # TODO https://jira.bars.group/browse/EDUSCHL-22492. Вынужденная мера, т.к. при запуске команды не производится
412
+ # проверка готовности конфигов приложений. Нужно переработать механизм конфигурирования клиента загрузчика.
413
+ def _configure_agent_client(self):
414
+ """
415
+ Конфигурирование клиента загрузчика данных в Витрину.
416
+ """
417
+ import uploader_client
418
+ from django.core.cache import (
419
+ DEFAULT_CACHE_ALIAS,
420
+ caches,
421
+ )
422
+ from uploader_client.contrib.rdm.interfaces.configurations import (
423
+ RegionalDataMartUploaderConfig,
424
+ )
425
+ if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
426
+ uploader_client.set_config(
427
+ RegionalDataMartUploaderConfig(
428
+ interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
429
+ url=settings.RDM_UPLOADER_CLIENT_URL,
430
+ datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
431
+ timeout=1,
432
+ request_retries=1,
433
+ )
434
+ )
435
+ elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
436
+ uploader_client.set_config(
437
+ RegionalDataMartUploaderConfig(
438
+ interface='uploader_client.contrib.rdm.interfaces.rest.ProxyAPIInterface',
439
+ cache=caches[DEFAULT_CACHE_ALIAS],
440
+ url=settings.RDM_UPLOADER_CLIENT_URL,
441
+ datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
442
+ timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
443
+ request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
444
+ organization_ogrn=settings.RDM_UPLOADER_CLIENT_ORGANIZATION_OGRN,
445
+ installation_name=settings.RDM_UPLOADER_CLIENT_INSTALLATION_NAME,
446
+ installation_id=settings.RDM_UPLOADER_CLIENT_INSTALLATION_ID,
447
+ username=settings.RDM_UPLOADER_CLIENT_USERNAME,
448
+ password=settings.RDM_UPLOADER_CLIENT_PASSWORD,
449
+ )
450
+ )
451
+ else:
452
+ uploader_client.set_config(
453
+ RegionalDataMartUploaderConfig(
454
+ url=settings.RDM_UPLOADER_CLIENT_URL,
455
+ datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
456
+ timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
457
+ request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
458
+ )
459
+ )
460
+
461
+ def upload_data(self, *args, **kwargs):
462
+ """
463
+ Запускает отправку данных в витрину.
464
+ """
465
+ try:
466
+ exporter = ExportEntityQueueSender(self.data_cache, self.queue, self.entities)
467
+ exporter.run()
468
+
469
+ sender = WorkerSender(self.queue)
470
+ sender.run()
471
+
472
+ except Exception as err:
473
+ logger.exception(err)
474
+ raise err
475
+ finally:
476
+ self._remove_file_handler()
@@ -1,10 +1,8 @@
1
- from datetime import (
2
- datetime,
3
- )
4
1
  from pathlib import (
5
2
  Path,
6
3
  )
7
4
  from typing import (
5
+ Iterable,
8
6
  List,
9
7
  )
10
8
 
@@ -13,6 +11,7 @@ from django.conf import (
13
11
  )
14
12
  from django.db.models import (
15
13
  F,
14
+ Q,
16
15
  Sum,
17
16
  )
18
17
  from django.db.transaction import (
@@ -83,10 +82,14 @@ class ExportQueueSender:
83
82
  return file_size or 0
84
83
 
85
84
  @staticmethod
86
- def get_sub_stages_attachments_to_export():
85
+ def _make_stage_filter(self) -> Q:
86
+ """Формирование фильтра для выборки подэтапов."""
87
+ return Q(status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key)
88
+
89
+ def get_sub_stages_attachments_to_export(self):
87
90
  """Выборка готовых к экспорту подэтапов."""
88
91
  return ExportingDataSubStage.objects.filter(
89
- status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key
92
+ self._make_stage_filter()
90
93
  ).annotate(
91
94
  attachment_id=F('exportingdatasubstageattachment__id'),
92
95
  attachment_name=F('exportingdatasubstageattachment__attachment'),
@@ -117,10 +120,14 @@ class ExportQueueSender:
117
120
  timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
118
121
  )
119
122
 
123
+ logger.info(
124
+ f'ExportedDataSubStage {sub_stage_id} {entity_name} added to the queue'
125
+ )
126
+
120
127
  return True
121
128
 
122
129
  def run(self):
123
- """Запуск работы менджера."""
130
+ """Запуск работы очереди."""
124
131
  if not self.queue_total_file_size:
125
132
  self.queue_total_file_size = self.get_exported_file_size()
126
133
 
@@ -133,9 +140,7 @@ class ExportQueueSender:
133
140
  if prev_sub_stage != stage_attachment['id']:
134
141
  if stage_files:
135
142
  to_cache = self.set_sub_stage_to_cache(prev_sub_stage, entity, stage_files)
136
- logger.info(
137
- f'ExportedDataSubStage {prev_sub_stage} {entity} added to the queue'
138
- )
143
+
139
144
  stage_files = []
140
145
 
141
146
  if not to_cache:
@@ -162,6 +167,25 @@ class ExportQueueSender:
162
167
  self.queue_total_file_size,
163
168
  timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
164
169
  )
170
+ logger.warning(
171
+ f'Total exported file size: {self.queue_total_file_size} - queue is full!!!'
172
+ )
173
+
174
+
175
+ class ExportEntityQueueSender(ExportQueueSender):
176
+ """Класс отправки данных по конкретной сущности в очередь РВД."""
177
+
178
+ def __init__(self, data_cache: AbstractCache, queue: Queue, entity: Iterable[str]):
179
+ super().__init__(data_cache, queue)
180
+
181
+ self.entity = entity
182
+
183
+ def _make_stage_filter(self) -> Q:
184
+ """Формирование фильтра для выборки подэтапов."""
185
+ return Q(
186
+ status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key,
187
+ exportingdatasubstageentity__entity_id__in=self.entity
188
+ )
165
189
 
166
190
 
167
191
  class WorkerSender:
@@ -240,6 +264,9 @@ class WorkerSender:
240
264
  sub_stage.save()
241
265
 
242
266
  self.queue.delete_from_queue(sub_stage_id=sub_stage_id, entity_name=entity_key)
267
+ logger.info(
268
+ f'ExportedDataSubStage {sub_stage_id} {entity_key} sended from the queue'
269
+ )
243
270
 
244
271
  def run(self):
245
272
  """Запуск воркера отправки."""
@@ -10,8 +10,8 @@ from typing import (
10
10
  TYPE_CHECKING,
11
11
  Any,
12
12
  Dict,
13
- List,
14
13
  Iterable,
14
+ List,
15
15
  Optional,
16
16
  Tuple,
17
17
  Union,
@@ -33,16 +33,16 @@ from django.db.models.functions import (
33
33
  Cast,
34
34
  Least,
35
35
  )
36
+ from uploader_client.adapters import (
37
+ adapter,
38
+ )
36
39
 
37
40
  from educommon import (
38
41
  logger,
39
42
  )
40
- from uploader_client.adapters import (
41
- adapter,
42
- )
43
+
43
44
  from edu_rdm_integration.collect_and_export_data.models import (
44
- EduRdmCollectDataCommandProgress,
45
- EduRdmExportDataCommandProgress,
45
+ AbstractExportDataCommandProgress,
46
46
  )
47
47
  from edu_rdm_integration.enums import (
48
48
  FileUploadStatusEnum,
@@ -54,32 +54,35 @@ from edu_rdm_integration.export_data.consts import (
54
54
  TOTAL_ATTACHMENTS_SIZE_KEY,
55
55
  )
56
56
  from edu_rdm_integration.models import (
57
- DataMartRequestStatus,
58
- ExportingDataStage,
59
57
  CollectingDataStageStatus,
60
58
  CollectingExportedDataStage,
59
+ DataMartRequestStatus,
60
+ ExportingDataStage,
61
+ ExportingDataStageStatus,
61
62
  ExportingDataSubStageUploaderClientLog,
63
+ UploadDataCommand,
62
64
  UploadStatusRequestLog,
63
- ExportingDataStageStatus,
64
65
  )
65
66
  from edu_rdm_integration.redis_cache import (
66
67
  AbstractCache,
67
68
  )
68
69
 
70
+
69
71
  if TYPE_CHECKING:
70
72
  from datetime import (
71
73
  datetime,
72
74
  )
73
75
 
76
+ from uploader_client.models import (
77
+ Entry,
78
+ )
79
+
74
80
  from edu_rdm_integration.collect_data.non_calculated.base.managers import (
75
81
  BaseCollectingExportedDataRunnerManager,
76
82
  )
77
83
  from edu_rdm_integration.export_data.base.managers import (
78
84
  BaseExportDataRunnerManager,
79
85
  )
80
- from uploader_client.models import (
81
- Entry,
82
- )
83
86
 
84
87
 
85
88
  class UploadStatusHelper:
@@ -311,7 +314,7 @@ class Graph:
311
314
 
312
315
 
313
316
  def save_command_log_link(
314
- command: Union[EduRdmCollectDataCommandProgress, EduRdmExportDataCommandProgress],
317
+ command: Union[AbstractExportDataCommandProgress, UploadDataCommand],
315
318
  log_dir: str
316
319
  ) -> None:
317
320
  """Сохраняет ссылку на лог команды."""
@@ -0,0 +1,43 @@
1
+ # Generated by Django 3.1.14 on 2024-09-05 20:35
2
+
3
+ import uuid
4
+
5
+ import django.db.models.deletion
6
+ import django.utils.timezone
7
+ from django.db import (
8
+ migrations,
9
+ models,
10
+ )
11
+
12
+ import educommon.django.db.mixins
13
+
14
+ import edu_rdm_integration.utils
15
+
16
+
17
+ class Migration(migrations.Migration):
18
+
19
+ dependencies = [
20
+ ('async_task', '0002_task_type_and_status_data'),
21
+ ('edu_rdm_integration', '0013_set_attachment_size'),
22
+ ]
23
+
24
+ operations = [
25
+ migrations.CreateModel(
26
+ name='UploadDataCommand',
27
+ fields=[
28
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
29
+ ('logs_link', models.FileField(max_length=255, upload_to=edu_rdm_integration.utils.get_data_command_progress_attachment_path, verbose_name='Ссылка на файл логов')),
30
+ ('type', models.PositiveSmallIntegerField(choices=[(1, 'Автоматический'), (2, 'Ручной')], verbose_name='Тип команды')),
31
+ ('created', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Дата создания')),
32
+ ('generation_id', models.UUIDField(default=uuid.uuid4, verbose_name='Идентификатор генерации')),
33
+ ('entity', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.regionaldatamartentityenum', verbose_name='Сущность РВД')),
34
+ ('task', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='async_task.runningtask', verbose_name='Асинхронная задача')),
35
+ ],
36
+ options={
37
+ 'verbose_name': 'Команда отправки данных в витрину',
38
+ 'verbose_name_plural': 'Команды отправки данных в витрину',
39
+ 'db_table': 'rdm_upload_data_command',
40
+ },
41
+ bases=(educommon.django.db.mixins.ReprStrPreModelMixin, models.Model),
42
+ ),
43
+ ]
@@ -26,6 +26,7 @@ from django.db.models import (
26
26
  Manager,
27
27
  OneToOneField,
28
28
  PositiveIntegerField,
29
+ PositiveSmallIntegerField,
29
30
  SmallIntegerField,
30
31
  UUIDField,
31
32
  )
@@ -65,12 +66,14 @@ from m3_db_utils.models import (
65
66
  )
66
67
 
67
68
  from edu_rdm_integration.enums import (
69
+ CommandType,
68
70
  FileUploadStatusEnum,
69
71
  )
70
72
  from edu_rdm_integration.uploader_log.managers import (
71
73
  UploaderClientLogManager,
72
74
  )
73
75
  from edu_rdm_integration.utils import (
76
+ get_data_command_progress_attachment_path,
74
77
  get_exporting_data_stage_attachment_path,
75
78
  )
76
79
 
@@ -150,6 +153,7 @@ class CollectingExportedDataStage(ReprStrPreModelMixin, BaseObjectModel):
150
153
 
151
154
  @property
152
155
  def attrs_for_repr_str(self):
156
+ """Список атрибутов для отображения экземпляра модели."""
153
157
  return ['manager_id', 'logs_period_started_at', 'logs_period_ended_at', 'started_at', 'ended_at', 'status_id']
154
158
 
155
159
  def save(self, *args, **kwargs):
@@ -245,6 +249,7 @@ class CollectingExportedDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
245
249
 
246
250
  @property
247
251
  def attrs_for_repr_str(self):
252
+ """Список атрибутов для отображения экземпляра модели."""
248
253
  return ['stage_id', 'function_id', 'started_at', 'ended_at', 'previous_id', 'status_id']
249
254
 
250
255
  def save(self, *args, **kwargs):
@@ -335,6 +340,7 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
335
340
 
336
341
  @property
337
342
  def attrs_for_repr_str(self):
343
+ """Список атрибутов для отображения экземпляра модели."""
338
344
  return ['manager_id', 'started_at', 'ended_at', 'status_id']
339
345
 
340
346
  def save(self, *args, **kwargs):
@@ -421,9 +427,11 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
421
427
 
422
428
  @property
423
429
  def attrs_for_repr_str(self):
430
+ """Список атрибутов для отображения экземпляра модели."""
424
431
  return ['function_id', 'collecting_data_sub_stage_id', 'stage_id', 'started_at', 'ended_at', 'status_id']
425
432
 
426
433
  def save(self, *args, **kwargs):
434
+ """Сохранение экземпляра модели."""
427
435
  if (
428
436
  self.status_id in {
429
437
  ExportingDataSubStageStatus.FAILED.key,
@@ -487,6 +495,7 @@ class ExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
487
495
 
488
496
  @property
489
497
  def attrs_for_repr_str(self):
498
+ """Список атрибутов для отображения экземпляра модели."""
490
499
  return ['exporting_data_sub_stage_id', 'attachment', 'operation', 'created', 'modified']
491
500
 
492
501
 
@@ -675,6 +684,7 @@ class BaseEntityModel(ReprStrPreModelMixin, BaseObjectModel):
675
684
 
676
685
  @property
677
686
  def attrs_for_repr_str(self):
687
+ """Список атрибутов для отображения экземпляра модели."""
678
688
  return ['collecting_sub_stage', 'exporting_sub_stage', 'operation', 'created', 'modified']
679
689
 
680
690
  class Meta:
@@ -931,6 +941,7 @@ class TransferredEntity(BaseObjectModel):
931
941
 
932
942
  @json_encode
933
943
  def no_export(self):
944
+ """Формирует отображение признака отключения экспорта."""
934
945
  return 'Нет' if self.export_enabled else 'Да'
935
946
 
936
947
 
@@ -953,3 +964,42 @@ class ExportingDataSubStageEntity(BaseObjectModel):
953
964
  db_table = 'rdm_exporting_data_sub_stage_entity'
954
965
  verbose_name = 'Связь сущности и подэтапа выгрузки'
955
966
  verbose_name_plural = 'Связи сущности и подэтапа выгрузки'
967
+
968
+
969
+ class UploadDataCommand(ReprStrPreModelMixin, BaseObjectModel):
970
+ """Модель, хранящая данные для формирования и отслеживания асинхронной задачи по отправке данных в витрину."""
971
+
972
+ task = ForeignKey(
973
+ to='async_task.RunningTask',
974
+ verbose_name='Асинхронная задача',
975
+ blank=True,
976
+ null=True,
977
+ on_delete=SET_NULL,
978
+ )
979
+ logs_link = FileField(
980
+ upload_to=get_data_command_progress_attachment_path,
981
+ max_length=255,
982
+ verbose_name='Ссылка на файл логов',
983
+ )
984
+ type = PositiveSmallIntegerField( # noqa: A003
985
+ verbose_name='Тип команды',
986
+ choices=CommandType.get_choices(),
987
+ )
988
+ entity = ForeignKey(
989
+ to=RegionalDataMartEntityEnum,
990
+ verbose_name='Сущность РВД',
991
+ on_delete=PROTECT,
992
+ )
993
+ created = DateTimeField(
994
+ verbose_name='Дата создания',
995
+ default=timezone.now,
996
+ )
997
+ generation_id = UUIDField(
998
+ 'Идентификатор генерации',
999
+ default=uuid.uuid4,
1000
+ )
1001
+
1002
+ class Meta:
1003
+ db_table = 'rdm_upload_data_command'
1004
+ verbose_name = 'Команда отправки данных в витрину'
1005
+ verbose_name_plural = 'Команды отправки данных в витрину'