dkist-processing-common 10.8.1rc1__py3-none-any.whl → 10.8.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_common/codecs/fits.py +6 -12
- dkist_processing_common/manual.py +5 -3
- dkist_processing_common/models/fried_parameter.py +41 -0
- dkist_processing_common/models/graphql.py +3 -13
- dkist_processing_common/models/parameters.py +28 -65
- dkist_processing_common/parsers/quality.py +1 -0
- dkist_processing_common/tasks/mixin/input_dataset.py +166 -0
- dkist_processing_common/tasks/mixin/metadata_store.py +4 -7
- dkist_processing_common/tasks/mixin/quality/_metrics.py +19 -14
- dkist_processing_common/tasks/quality_metrics.py +1 -1
- dkist_processing_common/tasks/transfer_input_data.py +70 -61
- dkist_processing_common/tasks/write_l1.py +29 -3
- dkist_processing_common/tests/conftest.py +7 -24
- dkist_processing_common/tests/test_codecs.py +0 -38
- dkist_processing_common/tests/test_fried_parameter.py +27 -0
- dkist_processing_common/tests/test_input_dataset.py +308 -79
- dkist_processing_common/tests/test_parameters.py +22 -71
- dkist_processing_common/tests/test_quality_mixin.py +32 -22
- dkist_processing_common/tests/test_transfer_input_data.py +45 -131
- dkist_processing_common/tests/test_write_l1.py +143 -10
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.3.dist-info}/METADATA +2 -2
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.3.dist-info}/RECORD +24 -27
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.3.dist-info}/WHEEL +1 -1
- changelog/235.feature.rst +0 -3
- changelog/235.misc.1.rst +0 -2
- changelog/235.misc.rst +0 -1
- dkist_processing_common/codecs/array.py +0 -19
- dkist_processing_common/codecs/basemodel.py +0 -21
- dkist_processing_common/models/input_dataset.py +0 -113
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.3.dist-info}/top_level.txt +0 -0
{dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.3.dist-info}/RECORD
RENAMED
|
@@ -1,21 +1,16 @@
|
|
|
1
1
|
changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
changelog/235.feature.rst,sha256=F27MUnNkqjcmlmRQ6NQUi2wd5M9dOjN2jZ70TV95Y20,365
|
|
3
|
-
changelog/235.misc.1.rst,sha256=kQujVYRa_axHQSRFxW3SNPNMgEL-YgtnyLud-aJljL8,234
|
|
4
|
-
changelog/235.misc.rst,sha256=ooK2tbA5EEFyYNZLLvgYiMZ-Eza5UzPHN9MvpuX-cu4,92
|
|
5
2
|
dkist_processing_common/__init__.py,sha256=490Fwm_GgqpwriQlsYfKcLUZNhZ6GkINtJqcYSIEKoU,319
|
|
6
3
|
dkist_processing_common/config.py,sha256=IcpaD_NvHZU-aLlUNOTdRC4V7ADIvVQwrZ2dHhIr4NY,4247
|
|
7
|
-
dkist_processing_common/manual.py,sha256=
|
|
4
|
+
dkist_processing_common/manual.py,sha256=IH72QxdGlr-BuWq2EDrfC9yStegO-elUWrobQqT4710,7042
|
|
8
5
|
dkist_processing_common/_util/__init__.py,sha256=xf6JNpMKQgbhE2Jivymt-WO0WF6PpGt9rl604YpuTWk,92
|
|
9
6
|
dkist_processing_common/_util/constants.py,sha256=b0zlRaT09aGj2RU72OQ5J-8u6Z_RevPXtcyx5tlnf-Y,3244
|
|
10
7
|
dkist_processing_common/_util/graphql.py,sha256=qjsvLWDHqb1X7hDLA8uqbpOIDjZZ2mjsSIL0Wkt1TJc,3420
|
|
11
8
|
dkist_processing_common/_util/scratch.py,sha256=4X5K260ffBHWvyVfP0SCk_-QTWEcTougB7qu9TdF7LM,10364
|
|
12
9
|
dkist_processing_common/_util/tags.py,sha256=8r62_-x3xpbCoCu5dd09Q2u-OYzYiML6SlPUf9LXCEw,6220
|
|
13
10
|
dkist_processing_common/codecs/__init__.py,sha256=du1iitvsudSSOMENSywXmXSLOlvIocJsPbvfEcyqFNc,159
|
|
14
|
-
dkist_processing_common/codecs/array.py,sha256=GeIB6mTMZuQK4Jxn2tGmMmYgA-bLik2SAlWopRZhEO8,575
|
|
15
11
|
dkist_processing_common/codecs/asdf.py,sha256=2GHCFOZk1j-ml4EolXac_sUzk7aPYJUGqKYxZk4mG_c,1046
|
|
16
|
-
dkist_processing_common/codecs/basemodel.py,sha256=6w7OjsPLdsmY_tiwveBmDthYW3WLtAWqM3hjrQa5UhA,814
|
|
17
12
|
dkist_processing_common/codecs/bytes.py,sha256=tiVEUu_Gzc5NfW1_qsJtHDlYAZzgIqA7f4cfAwN734k,495
|
|
18
|
-
dkist_processing_common/codecs/fits.py,sha256=
|
|
13
|
+
dkist_processing_common/codecs/fits.py,sha256=0D6tDvt13OhVXok4tS8VGHdd8OSV8DrRet8VCD6Zt3g,2346
|
|
19
14
|
dkist_processing_common/codecs/iobase.py,sha256=r0ImN0CxfjAnfMflNv7w2pGDp2i6EQg0p2OaEkE82pk,977
|
|
20
15
|
dkist_processing_common/codecs/json.py,sha256=OWXzoFWccJiojkiKSeDrMdL9f7EpdNIOMvO9YBBg-Yg,939
|
|
21
16
|
dkist_processing_common/codecs/path.py,sha256=LU5Kh1ew2PQI9hcpzbnZkE47k-zAMZDDV4cgqHRcDkY,197
|
|
@@ -27,12 +22,12 @@ dkist_processing_common/models/__init__.py,sha256=6LMqemdzVZ87fRrpAsbEnTtWZ02_Gu
|
|
|
27
22
|
dkist_processing_common/models/constants.py,sha256=1Eb8RDeuCr6brl237iGKxYLWCH49I6bOUEj_Tv-zFbQ,5441
|
|
28
23
|
dkist_processing_common/models/fits_access.py,sha256=Au9JROwhVla9zb_u0dN8mIWiSJd_Pca0oOr4N1hN0HY,4113
|
|
29
24
|
dkist_processing_common/models/flower_pot.py,sha256=59C5uGYKyMyncqQYxhzDZWl8k1DRZFB6s9RF-HFp9mY,5128
|
|
30
|
-
dkist_processing_common/models/
|
|
31
|
-
dkist_processing_common/models/
|
|
25
|
+
dkist_processing_common/models/fried_parameter.py,sha256=ro_H2Eo3I88lRf1wJjZfTc_XOjhgLt4whIQR_sjAFbM,1609
|
|
26
|
+
dkist_processing_common/models/graphql.py,sha256=t2PYEjEwSq08zq0HcVH2RG5FX4PVXgs4BdM2gunJ8fs,4747
|
|
32
27
|
dkist_processing_common/models/message.py,sha256=DRW7Qhl01dF5KagcqLta5U-uzdOMewrsHvMatDT6jnk,1684
|
|
33
28
|
dkist_processing_common/models/message_queue_binding.py,sha256=ROQ2ZQE3TCr4gVbz4WggvUSExAiWP8SD_GjjQl482M8,1012
|
|
34
29
|
dkist_processing_common/models/metric_code.py,sha256=McXAEF1Sa0_YlR1niXYLJWLFHhdLQhmYw9Xtpr5FGws,815
|
|
35
|
-
dkist_processing_common/models/parameters.py,sha256=
|
|
30
|
+
dkist_processing_common/models/parameters.py,sha256=Ymx-wvPVMkXg5emhOivv7NG0QsAt98If4KotopLIRrw,7736
|
|
36
31
|
dkist_processing_common/models/quality.py,sha256=ONz1A6_qyEoZhQkVp9LChAgm93aGt1O5WSRneE3XCCA,2319
|
|
37
32
|
dkist_processing_common/models/tags.py,sha256=ykOYqWMU7_ffvRCv84-avjXyty9pHBo7EXwsjIjStjs,12058
|
|
38
33
|
dkist_processing_common/models/task_name.py,sha256=NL0n92A9vVYBV-yvh8d-qFOCxVy0X2GECDmLgIzrmOY,565
|
|
@@ -46,7 +41,7 @@ dkist_processing_common/parsers/l0_fits_access.py,sha256=mhgezAA0X5zvgEmZFPIgKYp
|
|
|
46
41
|
dkist_processing_common/parsers/l1_fits_access.py,sha256=o0_7XfcCcuKpuN8K0E_dfEnPvB1a9VxqKBaxpRGDOL0,2595
|
|
47
42
|
dkist_processing_common/parsers/near_bud.py,sha256=gLTr7EILVCREAhmE0VaU7-KKc7aosoMmZmQtJEXxs2I,3733
|
|
48
43
|
dkist_processing_common/parsers/proposal_id_bud.py,sha256=61uRxT538UDiEg3_By_asSzQ96xkAuCEEZZTrk0UF0E,677
|
|
49
|
-
dkist_processing_common/parsers/quality.py,sha256=
|
|
44
|
+
dkist_processing_common/parsers/quality.py,sha256=Lg5z53kAvJuw-lPrX9HSkkFi0Bk0D5oDv8TAIgbLOMo,1140
|
|
50
45
|
dkist_processing_common/parsers/retarder.py,sha256=OzhdC8VoV2zdcvq8Nn5Ghc-Q3BFfMRITLx5jMHrWNqI,1440
|
|
51
46
|
dkist_processing_common/parsers/single_value_single_key_flower.py,sha256=cQX_C_Ms6EIOCW5TTDo6PnLoJ6In6MQTShKdMvt57OU,1230
|
|
52
47
|
dkist_processing_common/parsers/task.py,sha256=RTNGSg3BUpEYLOcolN79Wc16s86FqRcDP-H7Hj9ovGg,3938
|
|
@@ -59,40 +54,42 @@ dkist_processing_common/tasks/base.py,sha256=k_IJR5sVV6ennX0sbeb0C6dciqshdY7CKjt
|
|
|
59
54
|
dkist_processing_common/tasks/l1_output_data.py,sha256=IM-nvGaTM5r-z-9vHr2wovPVUpuNCah-cWIFMO2fcII,10576
|
|
60
55
|
dkist_processing_common/tasks/output_data_base.py,sha256=CC1TnCrChi8_iuMymr425CJqpY4jCggnVUMfqzFkpnw,3682
|
|
61
56
|
dkist_processing_common/tasks/parse_l0_input_data.py,sha256=iRMGdvhxBobNsTDQ0IEl0myDfB4P_xpxA00guuBWDj8,7986
|
|
62
|
-
dkist_processing_common/tasks/quality_metrics.py,sha256=
|
|
57
|
+
dkist_processing_common/tasks/quality_metrics.py,sha256=dWuPKBD5elRCZEs4ZC91tyXxkuPUwMuvy9OoGU9Zs04,12489
|
|
63
58
|
dkist_processing_common/tasks/teardown.py,sha256=e4LKnphJDYDVDAez2tH7MxpZgCmxYsKrq9Zk0qAkzzM,2355
|
|
64
|
-
dkist_processing_common/tasks/transfer_input_data.py,sha256=
|
|
59
|
+
dkist_processing_common/tasks/transfer_input_data.py,sha256=afEW0glpCFMZRj90nFtQo_4XOQ4CuoOh86jahP6a-a0,5548
|
|
65
60
|
dkist_processing_common/tasks/trial_catalog.py,sha256=Y3DKstRfMS8nWWtJFMB0MUVPlZ1jWS_2jhJGMWwxy50,8748
|
|
66
61
|
dkist_processing_common/tasks/trial_output_data.py,sha256=aI_aRuu0qVO8zFGrr_9baxx9i3jUEHZSmsmbO6ytlkE,6960
|
|
67
|
-
dkist_processing_common/tasks/write_l1.py,sha256=
|
|
62
|
+
dkist_processing_common/tasks/write_l1.py,sha256=EBQEViaLG2MzWsXOvNi8tsVsgzVJXBrLSGmYtsVgSlo,23781
|
|
68
63
|
dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
|
|
69
64
|
dkist_processing_common/tasks/mixin/globus.py,sha256=QAV8VElxMAqxJ2KSB_bJaraceovYfjHXjOdocrTCkIA,6592
|
|
65
|
+
dkist_processing_common/tasks/mixin/input_dataset.py,sha256=dkW5vf_QPgWedHO_Lf9GjBxr1QrUCKs6gIXufUTi7GE,6813
|
|
70
66
|
dkist_processing_common/tasks/mixin/interservice_bus.py,sha256=I7BUh0o8AEX-FZv7gxCts6is0uq9lycWjtTB2KqwBrU,1080
|
|
71
|
-
dkist_processing_common/tasks/mixin/metadata_store.py,sha256=
|
|
67
|
+
dkist_processing_common/tasks/mixin/metadata_store.py,sha256=dMwjlhBI89xX0Pagub2Bk09z0fToipFN51DwyQEIHbk,11510
|
|
72
68
|
dkist_processing_common/tasks/mixin/object_store.py,sha256=Vn4l2XuCimii9Fc3gM-pQGIkTKMv_ldqljlxkLesZLU,3236
|
|
73
69
|
dkist_processing_common/tasks/mixin/quality/__init__.py,sha256=Bgu-DHW7yXLiehglldOCWluEkAP5qh0Hp1F30rh5NFw,383
|
|
74
70
|
dkist_processing_common/tasks/mixin/quality/_base.py,sha256=nZ9IC-O-hsLXa5-tk29B13CZyQIdhJCv0eO9cdkAhWc,8303
|
|
75
|
-
dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=
|
|
71
|
+
dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=lkHU6SeWeMfzf0QPM28_K9znllKx6rK91v1TiQ7vYfw,54556
|
|
76
72
|
dkist_processing_common/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
|
-
dkist_processing_common/tests/conftest.py,sha256=
|
|
73
|
+
dkist_processing_common/tests/conftest.py,sha256=h_ObhpXb1S0-db0Je8XoHNLkrXxXW_0B0TMhbnbDjMo,30160
|
|
78
74
|
dkist_processing_common/tests/test_assemble_movie.py,sha256=XY_ruXSYP5k6s2gUAwlFdnhJ81eyWLSd2O9IkX4RXeo,4165
|
|
79
75
|
dkist_processing_common/tests/test_assemble_quality.py,sha256=fWSHK4UdVqgNjvxQuD40NBUnXrtmthUP7PUbISPV4MQ,16897
|
|
80
76
|
dkist_processing_common/tests/test_base.py,sha256=4ST3__jEHitEQaQs9-0OcqtyEJfIjZsk_6PRYZFV2-U,7124
|
|
81
|
-
dkist_processing_common/tests/test_codecs.py,sha256=
|
|
77
|
+
dkist_processing_common/tests/test_codecs.py,sha256=9Ln8FJs319rbHpCukO9lKLk3aDrdyDREjA4nCHsxDCA,20796
|
|
82
78
|
dkist_processing_common/tests/test_constants.py,sha256=Kc9k5TdYy5QkRRlGav6kfI2dy5HHKqtpf9qOuaAfDZU,5903
|
|
83
79
|
dkist_processing_common/tests/test_cs_step.py,sha256=RA0QD3D8eaL3YSOL_gIJ9wkngy14RQ2jbD-05KAziW4,2408
|
|
84
80
|
dkist_processing_common/tests/test_fits_access.py,sha256=tyUPNbUqoTPhQgzua_doWP9l9ee4ir_LLV-I9rHktcs,11393
|
|
85
81
|
dkist_processing_common/tests/test_flower_pot.py,sha256=X9_UI3maa3ZQncV3jYHgovWnawDsdEkEB5vw6EAB96o,3151
|
|
86
|
-
dkist_processing_common/tests/
|
|
82
|
+
dkist_processing_common/tests/test_fried_parameter.py,sha256=iXtlQIifZ6cDOkEi-YDgP3oAlss2loq08Uohgvy1byQ,1295
|
|
83
|
+
dkist_processing_common/tests/test_input_dataset.py,sha256=AI5uqaDea4kOwpwAU5qQdzUbxMpBwD20YCAvB7nzD5o,18766
|
|
87
84
|
dkist_processing_common/tests/test_interservice_bus.py,sha256=M_iv2CLmx5TnCB1VUN4YjkQ2LEUjfCKk7-ZlkV62XEQ,3000
|
|
88
85
|
dkist_processing_common/tests/test_interservice_bus_mixin.py,sha256=8TTl0aypkq5gBPeyqSaQHbz_jmt5RmSD2oI8kT4Q1ZA,4195
|
|
89
86
|
dkist_processing_common/tests/test_manual_processing.py,sha256=wAZJztsF33jzJE3m3vJ6cJT0ujgIkMg01jGq-Ys_a4c,1045
|
|
90
87
|
dkist_processing_common/tests/test_output_data_base.py,sha256=Y9MFz5xw11tAnKjpHH7qrzsRYP1rZM_Trt4AylY0S6k,3120
|
|
91
|
-
dkist_processing_common/tests/test_parameters.py,sha256=
|
|
88
|
+
dkist_processing_common/tests/test_parameters.py,sha256=Tr0Wu8-_rj9Lp7CkUVmJjSouW7CAL_ZwhShK_U1GzVs,12384
|
|
92
89
|
dkist_processing_common/tests/test_parse_l0_input_data.py,sha256=SMNV1qyQTvnMx94MCNsiA-RyS9uxaxIABEDDxsuVzqY,10629
|
|
93
90
|
dkist_processing_common/tests/test_publish_catalog_messages.py,sha256=wB2lcAnT77yVnqO0cFWOPxGf-tZ8U62kvvpiB5roBwQ,3268
|
|
94
91
|
dkist_processing_common/tests/test_quality.py,sha256=vomy2YSPadKqJj2tG8sCs-UkQVvfKus7Cum7_Hpee4I,10257
|
|
95
|
-
dkist_processing_common/tests/test_quality_mixin.py,sha256=
|
|
92
|
+
dkist_processing_common/tests/test_quality_mixin.py,sha256=hliXew6Sq7R166B2WZf022LyMKDio5G8jvkebHTm4ok,47067
|
|
96
93
|
dkist_processing_common/tests/test_scratch.py,sha256=7f28FMiSskSNX-bkRSrpJf2u1HQIbSvYajbjkeGMF9s,16481
|
|
97
94
|
dkist_processing_common/tests/test_stems.py,sha256=DRTfSVC1bVJ8mD9E9eR1W5lUHXBR2uBNkYBcWnVlA5E,28471
|
|
98
95
|
dkist_processing_common/tests/test_submit_dataset_metadata.py,sha256=F1IKBFWhjjMhONxEgs5p-cpjbQwxh7BLhDFw9b6j874,3806
|
|
@@ -100,12 +97,12 @@ dkist_processing_common/tests/test_tags.py,sha256=UwlOJ45rkvbfbd5L5m5YltvOxQc8kG
|
|
|
100
97
|
dkist_processing_common/tests/test_task_name.py,sha256=kqFr59XX2K87xzfTlClzDV4-Je1dx72LvdaJ22UE8UU,1233
|
|
101
98
|
dkist_processing_common/tests/test_task_parsing.py,sha256=QXt1X6DTO3_liBD2c-t84DToLeEn7B3J-eteIyN4HEM,4027
|
|
102
99
|
dkist_processing_common/tests/test_teardown.py,sha256=w2sATQHkg2lMLvm6VFZF1mNGFYHwWj_SxvF9RQu-tuY,5362
|
|
103
|
-
dkist_processing_common/tests/test_transfer_input_data.py,sha256=
|
|
100
|
+
dkist_processing_common/tests/test_transfer_input_data.py,sha256=kE-FQTcN9nft5bh2Rhtp-8ldCTvGXTvWFcsNm6DY7lk,6619
|
|
104
101
|
dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=27PifkyH3RZg0nsM-AjmrFJ-hbYuCk5Tt_0Zx8PJBfM,2109
|
|
105
102
|
dkist_processing_common/tests/test_trial_catalog.py,sha256=SZ-nyn0MXU9Lkg_94FbKER_cwiGoi06GYlzF_3AmvKg,6802
|
|
106
103
|
dkist_processing_common/tests/test_trial_output_data.py,sha256=cBCj0kXyF5NEMzKh6zPVksdoXyE8ju1opJgWgjdcJWA,12790
|
|
107
104
|
dkist_processing_common/tests/test_workflow_task_base.py,sha256=Z5aPW5LQtS0UWJiYho4X0r-2gPLfzpkmMwfmaoFLjMg,10517
|
|
108
|
-
dkist_processing_common/tests/test_write_l1.py,sha256=
|
|
105
|
+
dkist_processing_common/tests/test_write_l1.py,sha256=FHQMH1_JHXCEOpzLx97iRpj9pEIW5xQ8RlqvRspkSFY,27055
|
|
109
106
|
docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
|
|
110
107
|
docs/changelog.rst,sha256=S2jPASsWlQxSlAPqdvNrYvhk9k3FcFWNXFNDYXBSjl4,120
|
|
111
108
|
docs/conf.py,sha256=FkX575cqTqZGCcLAjg2MlvE8Buj1Vt3CpHNgZxG256E,1890
|
|
@@ -114,7 +111,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
|
|
|
114
111
|
docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
|
|
115
112
|
docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
|
|
116
113
|
licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
|
|
117
|
-
dkist_processing_common-10.8.
|
|
118
|
-
dkist_processing_common-10.8.
|
|
119
|
-
dkist_processing_common-10.8.
|
|
120
|
-
dkist_processing_common-10.8.
|
|
114
|
+
dkist_processing_common-10.8.3.dist-info/METADATA,sha256=pE3CBPNa-RjcRqsAt7RjadTRF7su-d8BGuzS4wklOnk,7147
|
|
115
|
+
dkist_processing_common-10.8.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
116
|
+
dkist_processing_common-10.8.3.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
|
|
117
|
+
dkist_processing_common-10.8.3.dist-info/RECORD,,
|
changelog/235.feature.rst
DELETED
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
Add two new codecs: Basemodel codecs are used for encoding and decoding Pydantic BaseModel objects. For decoding, the intended model
|
|
2
|
-
is passed to the decoder through a keyword argument in the task read method. Array codecs are used for encoding and decoding numpy
|
|
3
|
-
arrays similar to the standard np.load() and np.save(), but with the task tag-based write method.
|
changelog/235.misc.1.rst
DELETED
changelog/235.misc.rst
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
Remove the input_dataset mixin and replace it with input_dataset Pydantic BaseModel models.
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"""Encoder/decoder for writing/reading numpy arrays."""
|
|
2
|
-
import io
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
import numpy as np
|
|
6
|
-
|
|
7
|
-
from dkist_processing_common.codecs.iobase import iobase_encoder
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def array_encoder(data: np.ndarray, **np_kwargs) -> bytes:
|
|
11
|
-
"""Convert a numpy array to bytes compatible with np.load()."""
|
|
12
|
-
buffer = io.BytesIO()
|
|
13
|
-
np.save(buffer, data, **np_kwargs)
|
|
14
|
-
return iobase_encoder(buffer)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def array_decoder(path: Path, **np_kwargs) -> np.ndarray:
|
|
18
|
-
"""Return the data in the file as a numpy array using np.load()."""
|
|
19
|
-
return np.load(path, **np_kwargs)
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
"""Encoder/decoder for writing and reading Pydantic BaseModel objects."""
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from typing import Type
|
|
4
|
-
|
|
5
|
-
from pydantic import BaseModel
|
|
6
|
-
|
|
7
|
-
from dkist_processing_common.codecs.bytes import bytes_decoder
|
|
8
|
-
from dkist_processing_common.codecs.str import str_encoder
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def basemodel_encoder(data: BaseModel, **basemodel_kwargs) -> bytes:
|
|
12
|
-
"""Convert a Pydantic BaseModel object into bytes for writing to file."""
|
|
13
|
-
data_dump = data.model_dump_json(**basemodel_kwargs)
|
|
14
|
-
return str_encoder(data_dump)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def basemodel_decoder(path: Path, model: Type[BaseModel], **basemodel_kwargs) -> BaseModel:
|
|
18
|
-
"""Return the data in the file as a Pydantic BaseModel object."""
|
|
19
|
-
data = bytes_decoder(path)
|
|
20
|
-
model_validated = model.model_validate_json(data, **basemodel_kwargs)
|
|
21
|
-
return model_validated
|
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
"""Input dataset models for the inputDatasetPartDocument from the metadata store api."""
|
|
2
|
-
import json
|
|
3
|
-
from datetime import datetime
|
|
4
|
-
from typing import Any
|
|
5
|
-
|
|
6
|
-
from pydantic import BaseModel
|
|
7
|
-
from pydantic import ConfigDict
|
|
8
|
-
from pydantic import Field
|
|
9
|
-
from pydantic import field_serializer
|
|
10
|
-
from pydantic import field_validator
|
|
11
|
-
from pydantic import Json
|
|
12
|
-
from pydantic import PlainSerializer
|
|
13
|
-
from pydantic.alias_generators import to_camel
|
|
14
|
-
from typing_extensions import Annotated
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class InputDatasetBaseModel(BaseModel):
|
|
18
|
-
"""Custom BaseModel for input datasets."""
|
|
19
|
-
|
|
20
|
-
model_config = ConfigDict(
|
|
21
|
-
alias_generator=to_camel, validate_by_name=True, validate_by_alias=True
|
|
22
|
-
)
|
|
23
|
-
|
|
24
|
-
def model_dump(self, **kwargs) -> dict:
|
|
25
|
-
"""Dump models as they were in the metadata store."""
|
|
26
|
-
kwargs.setdefault("exclude_defaults", True)
|
|
27
|
-
kwargs.setdefault("by_alias", True) # will not be needed in Pydantic v3
|
|
28
|
-
return super().model_dump(**kwargs)
|
|
29
|
-
|
|
30
|
-
def model_dump_json(self, **kwargs) -> str:
|
|
31
|
-
"""Dump models as they were in the metadata store."""
|
|
32
|
-
kwargs.setdefault("exclude_defaults", True)
|
|
33
|
-
kwargs.setdefault("by_alias", True) # will not be needed in Pydantic v3
|
|
34
|
-
return super().model_dump_json(**kwargs)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
class InputDatasetObject(InputDatasetBaseModel):
|
|
38
|
-
"""Input dataset object validator for a single file."""
|
|
39
|
-
|
|
40
|
-
bucket: str
|
|
41
|
-
object_key: str
|
|
42
|
-
tag: str | None = None
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
class InputDatasetFilePointer(InputDatasetBaseModel):
|
|
46
|
-
"""Wrapper for InputDatasetObject files."""
|
|
47
|
-
|
|
48
|
-
file_pointer: InputDatasetObject = Field(alias="__file__")
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
class InputDatasetParameterValue(InputDatasetBaseModel):
|
|
52
|
-
"""Input dataset parameter value validator."""
|
|
53
|
-
|
|
54
|
-
parameter_value_id: int
|
|
55
|
-
# parameter_value: Json[InputDatasetFilePointer] | Json[Any] # will work in gqlclient v2
|
|
56
|
-
parameter_value: Json[Any]
|
|
57
|
-
parameter_value_start_date: Annotated[
|
|
58
|
-
datetime, Field(default=datetime(1, 1, 1)), PlainSerializer(lambda x: x.isoformat())
|
|
59
|
-
]
|
|
60
|
-
|
|
61
|
-
@field_validator("parameter_value", mode="after")
|
|
62
|
-
@classmethod
|
|
63
|
-
def validate_parameter_value(cls, param_val):
|
|
64
|
-
"""Decode and provide additional validation for parameter_value types."""
|
|
65
|
-
match param_val:
|
|
66
|
-
case {"__file__": _}:
|
|
67
|
-
return InputDatasetFilePointer.model_validate(param_val)
|
|
68
|
-
case _:
|
|
69
|
-
return param_val
|
|
70
|
-
|
|
71
|
-
@field_serializer("parameter_value")
|
|
72
|
-
def serialize_parameter_value(self, param_val):
|
|
73
|
-
"""Serialize the parameter_value types."""
|
|
74
|
-
if isinstance(param_val, InputDatasetBaseModel):
|
|
75
|
-
return json.dumps(param_val.model_dump())
|
|
76
|
-
return json.dumps(param_val)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
class InputDatasetParameter(InputDatasetBaseModel):
|
|
80
|
-
"""Parsing of the inputDatasetPartDocument that is relevant for parameters."""
|
|
81
|
-
|
|
82
|
-
parameter_name: str
|
|
83
|
-
parameter_values: list[InputDatasetParameterValue]
|
|
84
|
-
|
|
85
|
-
@property
|
|
86
|
-
def input_dataset_objects(self) -> list[InputDatasetObject]:
|
|
87
|
-
"""Find and return list of InputDatasetObjects."""
|
|
88
|
-
object_list = []
|
|
89
|
-
for param in self.parameter_values:
|
|
90
|
-
if isinstance(param.parameter_value, InputDatasetFilePointer):
|
|
91
|
-
object_list.append(param.parameter_value.file_pointer)
|
|
92
|
-
return object_list
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
class InputDatasetFrames(InputDatasetBaseModel):
|
|
96
|
-
"""Parsing of the inputDatasetPartDocument that is relevant for frames."""
|
|
97
|
-
|
|
98
|
-
bucket: str
|
|
99
|
-
object_keys: list[str] = Field(alias="object_keys") # not camel case in metadata store
|
|
100
|
-
|
|
101
|
-
@property
|
|
102
|
-
def input_dataset_objects(self) -> list[InputDatasetObject]:
|
|
103
|
-
"""Convert a single bucket and a list of object_keys list into a list of InputDatasetObjects."""
|
|
104
|
-
object_list = []
|
|
105
|
-
for frame in self.object_keys:
|
|
106
|
-
object_list.append(InputDatasetObject(bucket=self.bucket, object_key=frame))
|
|
107
|
-
return object_list
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
class InputDatasetPartDocumentList(InputDatasetBaseModel):
|
|
111
|
-
"""List of either InputDatasetFrames or InputDatasetParameter objects."""
|
|
112
|
-
|
|
113
|
-
doc_list: list[InputDatasetFrames] | list[InputDatasetParameter] = Field(alias="doc_list")
|
|
File without changes
|