eodag 3.8.0__py3-none-any.whl → 3.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. eodag/api/core.py +3 -2
  2. eodag/api/product/drivers/generic.py +5 -1
  3. eodag/api/product/metadata_mapping.py +110 -9
  4. eodag/cli.py +36 -4
  5. eodag/config.py +5 -2
  6. eodag/plugins/apis/ecmwf.py +3 -1
  7. eodag/plugins/apis/usgs.py +2 -1
  8. eodag/plugins/authentication/aws_auth.py +228 -37
  9. eodag/plugins/authentication/base.py +12 -2
  10. eodag/plugins/authentication/oauth.py +5 -0
  11. eodag/plugins/authentication/sas_auth.py +15 -0
  12. eodag/plugins/base.py +3 -2
  13. eodag/plugins/download/aws.py +44 -285
  14. eodag/plugins/download/base.py +3 -2
  15. eodag/plugins/download/creodias_s3.py +1 -38
  16. eodag/plugins/download/http.py +111 -103
  17. eodag/plugins/download/s3rest.py +3 -1
  18. eodag/plugins/manager.py +2 -1
  19. eodag/plugins/search/__init__.py +2 -1
  20. eodag/plugins/search/base.py +2 -1
  21. eodag/plugins/search/build_search_result.py +2 -2
  22. eodag/plugins/search/creodias_s3.py +9 -1
  23. eodag/plugins/search/qssearch.py +3 -1
  24. eodag/resources/ext_product_types.json +1 -1
  25. eodag/resources/product_types.yml +220 -30
  26. eodag/resources/providers.yml +634 -89
  27. eodag/resources/stac_provider.yml +6 -3
  28. eodag/resources/user_conf_template.yml +0 -5
  29. eodag/rest/core.py +8 -0
  30. eodag/rest/errors.py +9 -0
  31. eodag/rest/server.py +8 -0
  32. eodag/rest/stac.py +8 -0
  33. eodag/rest/utils/__init__.py +2 -4
  34. eodag/rest/utils/rfc3339.py +1 -1
  35. eodag/utils/__init__.py +69 -54
  36. eodag/utils/dates.py +204 -0
  37. eodag/utils/s3.py +187 -168
  38. {eodag-3.8.0.dist-info → eodag-3.9.0.dist-info}/METADATA +4 -3
  39. {eodag-3.8.0.dist-info → eodag-3.9.0.dist-info}/RECORD +43 -43
  40. {eodag-3.8.0.dist-info → eodag-3.9.0.dist-info}/entry_points.txt +1 -1
  41. eodag/utils/rest.py +0 -100
  42. {eodag-3.8.0.dist-info → eodag-3.9.0.dist-info}/WHEEL +0 -0
  43. {eodag-3.8.0.dist-info → eodag-3.9.0.dist-info}/licenses/LICENSE +0 -0
  44. {eodag-3.8.0.dist-info → eodag-3.9.0.dist-info}/top_level.txt +0 -0
@@ -349,7 +349,7 @@
349
349
  - '$.absoluteOrbitNumber'
350
350
  orbitDirection:
351
351
  - '{{"search":{{"passDirection":"{orbitDirection}" }} }}'
352
- - '$.passDirection'
352
+ - '{$.passDirection#to_lower}'
353
353
  sensorMode:
354
354
  - '{{"search":{{"mode":"{sensorMode}" }} }}'
355
355
  - '$.mode'
@@ -360,9 +360,9 @@
360
360
  completionTimeFromAscendingNode:
361
361
  - '{{"search":{{"date":{{"to":"{completionTimeFromAscendingNode}"}} }} }}'
362
362
  - '$.date'
363
- polarizationMode:
364
- - '{{"search":{{"polarization":"{polarizationMode}" }} }}'
365
- - '$.polarization'
363
+ polarizationChannels:
364
+ - '{{"search":{{"polarization":"{polarizationChannels#replace_str_tuple((("HH","SH"),("VV","SV"),("HH+HV", "DH"),("VV+VH","DV")))}" }} }}'
365
+ - '{$.polarization#replace_str_tuple((("SH","HH"),("SV","VV"),("DH","HH+HV"),("DV","VV+VH")))}'
366
366
  # Custom parameters (not defined in the base document referenced above)
367
367
  awsPath:
368
368
  - '{{"search":{{"awsPath":"{awsPath}" }} }}'
@@ -492,7 +492,6 @@
492
492
 
493
493
  download: !plugin
494
494
  type: AwsDownload
495
- requester_pays: True
496
495
  ssl_verify: true
497
496
  products:
498
497
  CBERS4_MUX_L2:
@@ -531,6 +530,7 @@
531
530
  download_auth: !plugin
532
531
  type: AwsAuth
533
532
  matching_url: s3://
533
+ requester_pays: True
534
534
  search_auth: !plugin
535
535
  type: HttpQueryStringAuth
536
536
  matching_url: https://gate.eos.com
@@ -630,9 +630,9 @@
630
630
  completionTimeFromAscendingNode:
631
631
  - completionDate
632
632
  - '$.properties.completionDate'
633
- polarizationMode:
633
+ polarizationChannels:
634
634
  - 'polarisation'
635
- - '$.properties.polarisation'
635
+ - '{$.properties.polarisation#replace_str(" ","+")}'
636
636
 
637
637
  # Custom parameters (not defined in the base document referenced above)
638
638
  id:
@@ -712,6 +712,7 @@
712
712
  - '='
713
713
  - '&'
714
714
  - ':'
715
+ - '%'
715
716
  pagination:
716
717
  next_page_url_tpl: '{url}?{search}&$top={items_per_page}&$skip={skip}&$expand=Attributes&$expand=Assets'
717
718
  count_tpl: '&$count=True'
@@ -741,17 +742,17 @@
741
742
  - "OData.CSC.Intersects(area=geography'{geometry#to_ewkt}')"
742
743
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'productType' and att/OData.CSC.StringAttribute/Value eq '{productType}')"
743
744
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformShortName' and att/OData.CSC.StringAttribute/Value eq '{platform}')"
744
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformSerialIdentifier' and att/OData.CSC.StringAttribute/Value eq '{platformSerialIdentifier}')"
745
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformSerialIdentifier' and att/OData.CSC.StringAttribute/Value eq '{platformSerialIdentifier#replace_str(\"^S[1-3]\", \"\")}')"
745
746
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'spatialResolution' and att/OData.CSC.StringAttribute/Value eq '{resolution}')"
746
747
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'authority' and att/OData.CSC.StringAttribute/Value eq '{organisationName}')"
747
748
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'orbitNumber' and att/OData.CSC.StringAttribute/Value eq '{orbitNumber}')"
748
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'orbitDirection' and att/OData.CSC.StringAttribute/Value eq '{orbitDirection}')"
749
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'orbitDirection' and att/OData.CSC.StringAttribute/Value eq '{orbitDirection#to_upper}')"
749
750
  - "Attributes/OData.CSC.DoubleAttribute/any(att:att/Name eq 'cloudCover' and att/OData.CSC.DoubleAttribute/Value le {cloudCover})"
750
751
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'operationalMode' and att/OData.CSC.StringAttribute/Value eq '{sensorMode}')"
751
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'polarisationChannels' and att/OData.CSC.StringAttribute/Value eq '{polarizationChannels}')"
752
752
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'tileId' and att/OData.CSC.StringAttribute/Value eq '{tileIdentifier}')"
753
753
  - "ContentDate/Start lt {completionTimeFromAscendingNode#to_iso_utc_datetime}"
754
754
  - "ContentDate/End gt {startTimeFromAscendingNode#to_iso_utc_datetime}"
755
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'polarisationChannels' and att/OData.CSC.StringAttribute/Value eq '{polarizationChannels#replace_str('\\+','%26')}')"
755
756
  - contains(Name,'{id}')
756
757
  discover_metadata:
757
758
  auto_discovery: true
@@ -807,7 +808,7 @@
807
808
  - '$.Attributes.orbitNumber'
808
809
  orbitDirection:
809
810
  - null
810
- - '$.Attributes.orbitDirection'
811
+ - '{$.Attributes.orbitDirection#to_lower}'
811
812
  cloudCover:
812
813
  - null
813
814
  - '$.Attributes.cloudCover'
@@ -824,7 +825,7 @@
824
825
  - '$.ContentDate.End'
825
826
  polarizationChannels:
826
827
  - null
827
- - '$.Attributes.polarisationChannels'
828
+ - '{$.Attributes.polarisationChannels#replace_str("&","+")}'
828
829
  # Custom parameters (not defined in the base document referenced above)
829
830
  id:
830
831
  - null
@@ -867,110 +868,297 @@
867
868
  allowed_audiences: ["CLOUDFERRO_PUBLIC"]
868
869
  products:
869
870
  # S1
871
+ S1_AUX_GNSSRD:
872
+ productType: AUX_GNSSRD
873
+ collection: SENTINEL-1
874
+ metadata_mapping:
875
+ cloudCover: '$.null'
876
+ platformSerialIdentifier:
877
+ - null
878
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
879
+ S1_AUX_MOEORB:
880
+ productType: AUX_MOEORB
881
+ collection: SENTINEL-1
882
+ metadata_mapping:
883
+ cloudCover: '$.null'
884
+ platformSerialIdentifier:
885
+ - null
886
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
887
+ S1_AUX_POEORB:
888
+ productType: AUX_POEORB
889
+ collection: SENTINEL-1
890
+ metadata_mapping:
891
+ cloudCover: '$.null'
892
+ platformSerialIdentifier:
893
+ - null
894
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
895
+ S1_AUX_PREORB:
896
+ productType: AUX_PREORB
897
+ collection: SENTINEL-1
898
+ metadata_mapping:
899
+ cloudCover: '$.null'
900
+ platformSerialIdentifier:
901
+ - null
902
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
903
+ S1_AUX_PROQUA:
904
+ productType: AUX_PROQUA
905
+ collection: SENTINEL-1
906
+ metadata_mapping:
907
+ cloudCover: '$.null'
908
+ platformSerialIdentifier:
909
+ - null
910
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
911
+ S1_AUX_RESORB:
912
+ productType: AUX_RESORB
913
+ collection: SENTINEL-1
914
+ metadata_mapping:
915
+ cloudCover: '$.null'
916
+ platformSerialIdentifier:
917
+ - null
918
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
870
919
  S1_SAR_RAW:
871
920
  productType: RAW
872
921
  collection: SENTINEL-1
873
922
  metadata_mapping:
874
923
  cloudCover: '$.null'
924
+ platformSerialIdentifier:
925
+ - null
926
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
875
927
  S1_SAR_GRD:
876
928
  productType: GRD
877
929
  collection: SENTINEL-1
878
930
  metadata_mapping:
879
931
  cloudCover: '$.null'
932
+ platformSerialIdentifier:
933
+ - null
934
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
880
935
  S1_SAR_SLC:
881
936
  productType: SLC
882
937
  collection: SENTINEL-1
883
938
  metadata_mapping:
884
939
  cloudCover: '$.null'
940
+ platformSerialIdentifier:
941
+ - null
942
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
885
943
  S1_SAR_OCN:
886
944
  productType: OCN
887
945
  collection: SENTINEL-1
888
946
  metadata_mapping:
889
947
  cloudCover: '$.null'
948
+ platformSerialIdentifier:
949
+ - null
950
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
951
+ S1_SAR_L3_IW_MCM:
952
+ productType: S1SAR_L3_IW_MCM
953
+ collection: GLOBAL-MOSAICS
954
+ metadata_mapping:
955
+ cloudCover: '$.null'
956
+ platformSerialIdentifier:
957
+ - null
958
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
959
+ S1_SAR_L3_DH_MCM:
960
+ productType: S1SAR_L3_DH_MCM
961
+ collection: GLOBAL-MOSAICS
962
+ metadata_mapping:
963
+ cloudCover: '$.null'
964
+ platformSerialIdentifier:
965
+ - null
966
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
890
967
  # S2
891
968
  S2_MSI_L1C:
892
969
  collection: SENTINEL-2
893
970
  productType: S2MSI1C
971
+ metadata_mapping:
972
+ platformSerialIdentifier:
973
+ - null
974
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S2)`'
894
975
  S2_MSI_L2A:
895
976
  collection: SENTINEL-2
896
977
  productType: S2MSI2A
978
+ metadata_mapping:
979
+ platformSerialIdentifier:
980
+ - null
981
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S2)`'
897
982
  # S3 SRAL
898
983
  S3_SRA:
899
984
  productType: SR_1_SRA___
900
985
  collection: SENTINEL-3
901
986
  metadata_mapping:
902
987
  cloudCover: '$.null'
988
+ platformSerialIdentifier:
989
+ - null
990
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
903
991
  S3_SRA_A:
904
992
  productType: SR_1_SRA_A_
905
993
  collection: SENTINEL-3
906
994
  metadata_mapping:
907
995
  cloudCover: '$.null'
996
+ platformSerialIdentifier:
997
+ - null
998
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
908
999
  S3_SRA_BS:
909
1000
  productType: SR_1_SRA_BS
910
1001
  collection: SENTINEL-3
911
1002
  metadata_mapping:
912
1003
  cloudCover: '$.null'
1004
+ platformSerialIdentifier:
1005
+ - null
1006
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
913
1007
  S3_LAN:
914
1008
  productType: SR_2_LAN___
915
1009
  collection: SENTINEL-3
916
1010
  metadata_mapping:
917
1011
  cloudCover: '$.null'
1012
+ platformSerialIdentifier:
1013
+ - null
1014
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
1015
+ S3_LAN_HY:
1016
+ productType: SR_2_LAN_HY
1017
+ collection: SENTINEL-3
1018
+ metadata_mapping:
1019
+ cloudCover: '$.null'
1020
+ platformSerialIdentifier:
1021
+ - null
1022
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
1023
+ S3_LAN_SI:
1024
+ productType: SR_2_LAN_SI
1025
+ collection: SENTINEL-3
1026
+ metadata_mapping:
1027
+ cloudCover: '$.null'
1028
+ platformSerialIdentifier:
1029
+ - null
1030
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
1031
+ S3_LAN_LI:
1032
+ productType: SR_2_LAN_LI
1033
+ collection: SENTINEL-3
1034
+ metadata_mapping:
1035
+ cloudCover: '$.null'
1036
+ platformSerialIdentifier:
1037
+ - null
1038
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
918
1039
  S3_WAT:
919
1040
  productType: SR_2_WAT___
920
1041
  collection: SENTINEL-3
921
1042
  metadata_mapping:
922
1043
  cloudCover: '$.null'
1044
+ platformSerialIdentifier:
1045
+ - null
1046
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
923
1047
  # S3 OLCI
924
1048
  S3_EFR:
925
1049
  productType: OL_1_EFR___
926
1050
  collection: SENTINEL-3
1051
+ metadata_mapping:
1052
+ platformSerialIdentifier:
1053
+ - null
1054
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
927
1055
  S3_ERR:
928
1056
  productType: OL_1_ERR___
929
1057
  collection: SENTINEL-3
1058
+ metadata_mapping:
1059
+ platformSerialIdentifier:
1060
+ - null
1061
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
930
1062
  S3_OLCI_L2LRR:
931
1063
  productType: OL_2_LRR___
932
1064
  collection: SENTINEL-3
1065
+ metadata_mapping:
1066
+ platformSerialIdentifier:
1067
+ - null
1068
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
933
1069
  S3_OLCI_L2LFR:
934
1070
  productType: OL_2_LFR___
935
1071
  collection: SENTINEL-3
1072
+ metadata_mapping:
1073
+ platformSerialIdentifier:
1074
+ - null
1075
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
936
1076
  S3_OLCI_L2WRR:
937
1077
  productType: OL_2_WRR___
938
1078
  collection: SENTINEL-3
1079
+ metadata_mapping:
1080
+ platformSerialIdentifier:
1081
+ - null
1082
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
939
1083
  S3_OLCI_L2WFR:
940
1084
  productType: OL_2_WFR___
941
1085
  collection: SENTINEL-3
1086
+ metadata_mapping:
1087
+ platformSerialIdentifier:
1088
+ - null
1089
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
942
1090
  # S3 SLSTR
943
1091
  S3_SLSTR_L1RBT:
944
1092
  productType: SL_1_RBT___
945
1093
  collection: SENTINEL-3
1094
+ metadata_mapping:
1095
+ platformSerialIdentifier:
1096
+ - null
1097
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
946
1098
  S3_SLSTR_L2LST:
947
1099
  productType: SL_2_LST___
948
1100
  collection: SENTINEL-3
1101
+ metadata_mapping:
1102
+ platformSerialIdentifier:
1103
+ - null
1104
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
949
1105
  S3_SLSTR_L2WST:
950
1106
  productType: SL_2_WST___
951
1107
  collection: SENTINEL-3
1108
+ metadata_mapping:
1109
+ platformSerialIdentifier:
1110
+ - null
1111
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
952
1112
  S3_SLSTR_L2AOD:
953
1113
  productType: SL_2_AOD___
954
1114
  collection: SENTINEL-3
1115
+ metadata_mapping:
1116
+ platformSerialIdentifier:
1117
+ - null
1118
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
955
1119
  S3_SLSTR_L2FRP:
956
1120
  productType: SL_2_FRP___
957
1121
  collection: SENTINEL-3
1122
+ metadata_mapping:
1123
+ platformSerialIdentifier:
1124
+ - null
1125
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
958
1126
  # S3 SY
959
1127
  S3_SY_AOD:
960
1128
  productType: SY_2_AOD___
961
1129
  collection: SENTINEL-3
1130
+ metadata_mapping:
1131
+ platformSerialIdentifier:
1132
+ - null
1133
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
962
1134
  S3_SY_SYN:
963
1135
  productType: SY_2_SYN___
964
1136
  collection: SENTINEL-3
1137
+ metadata_mapping:
1138
+ platformSerialIdentifier:
1139
+ - null
1140
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
965
1141
  S3_SY_V10:
966
1142
  productType: SY_2_V10___
967
1143
  collection: SENTINEL-3
1144
+ metadata_mapping:
1145
+ platformSerialIdentifier:
1146
+ - null
1147
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
968
1148
  S3_SY_VG1:
969
1149
  productType: SY_2_VG1___
970
1150
  collection: SENTINEL-3
1151
+ metadata_mapping:
1152
+ platformSerialIdentifier:
1153
+ - null
1154
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
971
1155
  S3_SY_VGP:
972
1156
  productType: SY_2_VGP___
973
1157
  collection: SENTINEL-3
1158
+ metadata_mapping:
1159
+ platformSerialIdentifier:
1160
+ - null
1161
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
974
1162
  # S5P L1
975
1163
  S5P_L1B_IR_SIR:
976
1164
  productType: L1B_IR_SIR
@@ -1103,7 +1291,7 @@
1103
1291
  illuminationAzimuthAngle: properties.view:sun_azimuth
1104
1292
  cloudCover: properties.eo:cloud_cover
1105
1293
  metadata_mapping:
1106
- assets: '{$.assets#recursive_sub_str(r"https?(.*)landsatlook.usgs.gov/data/",r"s3\1usgs-landsat/")}'
1294
+ assets: '{$.assets#from_alternate(s3)}'
1107
1295
  awsProductId: '{$.assets.thumbnail.href#replace_str(r".+/([A-Z0-9_]+)/[\w.]+$",r"\1")}'
1108
1296
  products:
1109
1297
  LANDSAT_C2L1:
@@ -1124,11 +1312,11 @@
1124
1312
  productType: '{productType}'
1125
1313
  download: !plugin
1126
1314
  type: AwsDownload
1127
- requester_pays: True
1128
1315
  ssl_verify: true
1129
1316
  auth: !plugin
1130
1317
  type: AwsAuth
1131
1318
  matching_url: s3://
1319
+ requester_pays: True
1132
1320
 
1133
1321
  ---
1134
1322
  !provider # MARK: earth_search
@@ -1211,6 +1399,10 @@
1211
1399
  tileIdentifier:
1212
1400
  - '{{"query":{{"mgrs:utm_zone":{{"eq":"{tileIdentifier#slice_str(0,2,1)}"}},"mgrs:latitude_band":{{"eq":"{tileIdentifier#slice_str(2,3,1)}"}},"mgrs:grid_square":{{"eq":"{tileIdentifier#slice_str(3,5,1)}"}}}}}}'
1213
1401
  - '{utmZone}{latitudeBand}{gridSquare}'
1402
+ S2_MSI_L2A_COG:
1403
+ productType: sentinel-2-l2a
1404
+ metadata_mapping:
1405
+ assets: '{$.assets#dict_filter_and_sub("$[?(@.href =~ \"^https://sentinel-cogs\\.s3\\.us-west-2\\.amazonaws\\.com/.*\")]","https","s3")}'
1214
1406
  LANDSAT_C2L2:
1215
1407
  productType: landsat-c2-l2
1216
1408
  NAIP:
@@ -1223,7 +1415,6 @@
1223
1415
  productType: '{productType}'
1224
1416
  download: !plugin
1225
1417
  type: AwsDownload
1226
- requester_pays: True
1227
1418
  ssl_verify: true
1228
1419
  products:
1229
1420
  S2_MSI_L1C:
@@ -1235,61 +1426,9 @@
1235
1426
  auth: !plugin
1236
1427
  type: AwsAuth
1237
1428
  matching_url: s3://
1429
+ requester_pays: True
1238
1430
 
1239
1431
  ---
1240
- !provider # MARK: earth_search_cog
1241
- name: earth_search_cog
1242
- priority: 0
1243
- roles:
1244
- - host
1245
- description: Earth Search with Cloud Optimized GeoTIFF (COG) formatted assets
1246
- url: https://www.element84.com/earth-search/
1247
- search: !plugin
1248
- type: StacSearch
1249
- api_endpoint: https://earth-search.aws.element84.com/v1/search
1250
- need_auth: false
1251
- ssl_verify: true
1252
- discover_product_types:
1253
- fetch_url: null
1254
- discover_queryables:
1255
- fetch_url: null
1256
- product_type_fetch_url: null
1257
- pagination:
1258
- # Override the default next page url key path of StacSearch because the next link returned
1259
- # by Earth Search is invalid (as of 2021/04/29). Entry set to null (None) to avoid using the
1260
- # next page retrieval mechanism, `next_page_url_tpl` will be used instead (inherited from StacSearch)
1261
- # Remove that entry if Earth Search updates that and returns a valid link.
1262
- next_page_url_key_path: null
1263
- # 2021/04/28: Earth-Search relies on Sat-API whose docs (http://sat-utils.github.io/sat-api/#search-stac-items-by-simple-filtering-)
1264
- # say the max is 10_000. In practice a too high number (e.g. 5_000) returns a 502 error ({"message": "Internal server error"}).
1265
- # Let's set it to a more robust number: 500
1266
- max_items_per_page: 500
1267
- sort:
1268
- sort_by_default:
1269
- - !!python/tuple [startTimeFromAscendingNode, ASC]
1270
- sort_param_mapping:
1271
- id: id
1272
- startTimeFromAscendingNode: properties.datetime
1273
- creationDate: properties.created
1274
- modificationDate: properties.updated
1275
- platform: properties.constellation
1276
- platformSerialIdentifier: properties.platform
1277
- resolution: properties.gsd
1278
- cloudCover: properties.eo:cloud_cover
1279
- metadata_mapping:
1280
- platformSerialIdentifier: '$.id.`split(_, 0, -1)`'
1281
- polarizationMode: '$.id.`sub(/.{14}([A-Z]{2}).*/, \\1)`'
1282
- title: '{$.properties."s2:product_uri"#remove_extension}'
1283
- assets: '{$.assets#dict_filter($[?(href=~"^http.*")])}'
1284
- products:
1285
- S2_MSI_L2A_COG:
1286
- productType: sentinel-2-l2a
1287
- GENERIC_PRODUCT_TYPE:
1288
- productType: '{productType}'
1289
- download: !plugin
1290
- type: HTTPDownload
1291
- ssl_verify: true
1292
- ---
1293
1432
  !provider # MARK: earth_search_gcs
1294
1433
  name: earth_search_gcs
1295
1434
  priority: 0
@@ -1358,6 +1497,7 @@
1358
1497
  auth: !plugin
1359
1498
  type: AwsAuth
1360
1499
  matching_url: s3://
1500
+ s3_endpoint: https://storage.googleapis.com
1361
1501
  matching_conf:
1362
1502
  s3_endpoint: https://storage.googleapis.com
1363
1503
  ---
@@ -1936,6 +2076,8 @@
1936
2076
  gcm: ec_earth
1937
2077
  ensemble_member: r12i1p1
1938
2078
  period: 1_5_c
2079
+ CMIP6_CLIMATE_PROJECTIONS:
2080
+ dataset: projections-cmip6
1939
2081
  GENERIC_PRODUCT_TYPE:
1940
2082
  dataset: '{productType}'
1941
2083
  ---
@@ -1970,6 +2112,19 @@
1970
2112
  ascending: asc
1971
2113
  descending: desc
1972
2114
  max_sort_params: 1
2115
+ free_text_search_operations:
2116
+ orbitDirection:
2117
+ union: '&'
2118
+ wrapper: '{}'
2119
+ operations:
2120
+ and:
2121
+ - '{orbitDirection#to_title}'
2122
+ polarisation:
2123
+ union: '&'
2124
+ wrapper: '{}'
2125
+ operations:
2126
+ and:
2127
+ - '{polarizationChannels#replace_str("\\+",",")}'
1973
2128
  discover_metadata:
1974
2129
  auto_discovery: true
1975
2130
  metadata_pattern: '^(?!collection)[a-zA-Z0-9_]+$'
@@ -2015,8 +2170,8 @@
2015
2170
  - 'orbitNumber'
2016
2171
  - '$.properties.orbitNumber'
2017
2172
  orbitDirection:
2018
- - 'orbitDirection'
2019
- - '$.properties.orbitDirection'
2173
+ - null
2174
+ - '{$.properties.orbitDirection#to_lower}'
2020
2175
  swathIdentifier:
2021
2176
  - 'swath'
2022
2177
  - '$.properties.swath'
@@ -2042,9 +2197,9 @@
2042
2197
  completionTimeFromAscendingNode:
2043
2198
  - completionDate
2044
2199
  - '$.properties.completionDate'
2045
- polarizationMode:
2046
- - 'polarisation'
2047
- - '$.properties.polarisation'
2200
+ polarizationChannels:
2201
+ - null
2202
+ - '{$.properties.polarisation#replace_str(",","+")}'
2048
2203
  # Custom parameters (not defined in the base document referenced above)
2049
2204
  id:
2050
2205
  - 'productIdentifier={id#remove_extension}'
@@ -2339,6 +2494,7 @@
2339
2494
  - '='
2340
2495
  - '&'
2341
2496
  - ':'
2497
+ - '%'
2342
2498
  pagination:
2343
2499
  next_page_url_tpl: '{url}?{search}&$top={items_per_page}&$skip={skip}&$expand=Attributes&$expand=Assets'
2344
2500
  count_tpl: '&$count=True'
@@ -2368,15 +2524,15 @@
2368
2524
  - "OData.CSC.Intersects(area=geography'{geometry#to_ewkt}')"
2369
2525
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'productType' and att/OData.CSC.StringAttribute/Value eq '{productType}')"
2370
2526
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformShortName' and att/OData.CSC.StringAttribute/Value eq '{platform}')"
2371
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformSerialIdentifier' and att/OData.CSC.StringAttribute/Value eq '{platformSerialIdentifier}')"
2527
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformSerialIdentifier' and att/OData.CSC.StringAttribute/Value eq '{platformSerialIdentifier#replace_str(\"^S[1-3]\", \"\")}')"
2372
2528
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'spatialResolution' and att/OData.CSC.StringAttribute/Value eq '{resolution}')"
2373
2529
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'authority' and att/OData.CSC.StringAttribute/Value eq '{organisationName}')"
2374
2530
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'orbitNumber' and att/OData.CSC.IntegerAttribute/Value eq {orbitNumber})"
2375
2531
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'relativeOrbitNumber' and att/OData.CSC.IntegerAttribute/Value eq {relativeOrbitNumber})"
2376
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'orbitDirection' and att/OData.CSC.StringAttribute/Value eq '{orbitDirection}')"
2532
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'orbitDirection' and att/OData.CSC.StringAttribute/Value eq '{orbitDirection#to_upper}')"
2377
2533
  - "Attributes/OData.CSC.DoubleAttribute/any(att:att/Name eq 'cloudCover' and att/OData.CSC.DoubleAttribute/Value le {cloudCover})"
2378
2534
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'operationalMode' and att/OData.CSC.StringAttribute/Value eq '{sensorMode}')"
2379
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'polarisationChannels' and att/OData.CSC.StringAttribute/Value eq '{polarizationChannels}')"
2535
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'polarisationChannels' and att/OData.CSC.StringAttribute/Value eq '{polarizationChannels#replace_str('\\+','%26')}')"
2380
2536
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'tileId' and att/OData.CSC.StringAttribute/Value eq '{tileIdentifier}')"
2381
2537
  - "ModificationDate gt {modifiedAfter#to_iso_utc_datetime}"
2382
2538
  - "ModificationDate lt {modifiedBefore#to_iso_utc_datetime}"
@@ -2443,7 +2599,7 @@
2443
2599
  - '$.Attributes.orbitNumber'
2444
2600
  orbitDirection:
2445
2601
  - null
2446
- - '$.Attributes.orbitDirection'
2602
+ - '{$.Attributes.orbitDirection#to_lower}'
2447
2603
  cloudCover:
2448
2604
  - null
2449
2605
  - '$.Attributes.cloudCover'
@@ -2471,7 +2627,7 @@
2471
2627
  - '$.ContentDate.End'
2472
2628
  polarizationChannels:
2473
2629
  - null
2474
- - '$.Attributes.polarisationChannels'
2630
+ - '{$.Attributes.polarisationChannels#replace_str("&","+")}'
2475
2631
  # Custom parameters (not defined in the base document referenced above)
2476
2632
  id:
2477
2633
  - null
@@ -2517,112 +2673,302 @@
2517
2673
  S2_MSI_L1C:
2518
2674
  collection: SENTINEL-2
2519
2675
  productType: S2MSI1C
2676
+ metadata_mapping:
2677
+ platformSerialIdentifier:
2678
+ - null
2679
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S2)`'
2520
2680
  S2_MSI_L2A:
2521
2681
  collection: SENTINEL-2
2522
2682
  productType: S2MSI2A
2683
+ metadata_mapping:
2684
+ platformSerialIdentifier:
2685
+ - null
2686
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S2)`'
2523
2687
  # S1
2688
+ S1_AUX_GNSSRD:
2689
+ productType: AUX_GNSSRD
2690
+ collection: SENTINEL-1
2691
+ metadata_mapping:
2692
+ cloudCover: '$.null'
2693
+ platformSerialIdentifier:
2694
+ - null
2695
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2696
+ S1_AUX_MOEORB:
2697
+ productType: AUX_MOEORB
2698
+ collection: SENTINEL-1
2699
+ metadata_mapping:
2700
+ cloudCover: '$.null'
2701
+ platformSerialIdentifier:
2702
+ - null
2703
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2704
+ S1_AUX_POEORB:
2705
+ productType: AUX_POEORB
2706
+ collection: SENTINEL-1
2707
+ metadata_mapping:
2708
+ cloudCover: '$.null'
2709
+ platformSerialIdentifier:
2710
+ - null
2711
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2712
+ S1_AUX_PREORB:
2713
+ productType: AUX_PREORB
2714
+ collection: SENTINEL-1
2715
+ metadata_mapping:
2716
+ cloudCover: '$.null'
2717
+ platformSerialIdentifier:
2718
+ - null
2719
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2720
+ S1_AUX_PROQUA:
2721
+ productType: AUX_PROQUA
2722
+ collection: SENTINEL-1
2723
+ metadata_mapping:
2724
+ cloudCover: '$.null'
2725
+ platformSerialIdentifier:
2726
+ - null
2727
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2728
+ S1_AUX_RESORB:
2729
+ productType: AUX_RESORB
2730
+ collection: SENTINEL-1
2731
+ metadata_mapping:
2732
+ cloudCover: '$.null'
2733
+ platformSerialIdentifier:
2734
+ - null
2735
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2524
2736
  S1_SAR_RAW:
2525
2737
  productType: RAW
2526
2738
  collection: SENTINEL-1
2527
2739
  metadata_mapping:
2528
2740
  cloudCover: '$.null'
2741
+ platformSerialIdentifier:
2742
+ - null
2743
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2529
2744
  S1_SAR_GRD:
2530
2745
  productType: GRD
2531
2746
  collection: SENTINEL-1
2532
2747
  metadata_mapping:
2533
2748
  cloudCover: '$.null'
2749
+ platformSerialIdentifier:
2750
+ - null
2751
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2534
2752
  S1_SAR_GRD_COG:
2535
2753
  productType: GRD-COG
2536
2754
  collection: SENTINEL-1
2537
2755
  metadata_mapping:
2538
2756
  cloudCover: '$.null'
2757
+ platformSerialIdentifier:
2758
+ - null
2759
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2539
2760
  S1_SAR_SLC:
2540
2761
  productType: SLC
2541
2762
  collection: SENTINEL-1
2542
2763
  metadata_mapping:
2543
2764
  cloudCover: '$.null'
2765
+ platformSerialIdentifier:
2766
+ - null
2767
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2544
2768
  S1_SAR_OCN:
2545
2769
  productType: OCN
2546
2770
  collection: SENTINEL-1
2547
2771
  metadata_mapping:
2548
2772
  cloudCover: '$.null'
2773
+ platformSerialIdentifier:
2774
+ - null
2775
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2776
+ S1_SAR_L3_IW_MCM:
2777
+ productType: S1SAR_L3_IW_MCM
2778
+ collection: GLOBAL-MOSAICS
2779
+ metadata_mapping:
2780
+ cloudCover: '$.null'
2781
+ platformSerialIdentifier:
2782
+ - null
2783
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2784
+ S1_SAR_L3_DH_MCM:
2785
+ productType: S1SAR_L3_DH_MCM
2786
+ collection: GLOBAL-MOSAICS
2787
+ metadata_mapping:
2788
+ cloudCover: '$.null'
2789
+ platformSerialIdentifier:
2790
+ - null
2791
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
2549
2792
  # S3 SRAL
2550
2793
  S3_SRA:
2551
2794
  productType: SR_1_SRA___
2552
2795
  collection: SENTINEL-3
2553
2796
  metadata_mapping:
2554
2797
  cloudCover: '$.null'
2798
+ platformSerialIdentifier:
2799
+ - null
2800
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2555
2801
  S3_SRA_A:
2556
2802
  productType: SR_1_SRA_A_
2557
2803
  collection: SENTINEL-3
2558
2804
  metadata_mapping:
2559
2805
  cloudCover: '$.null'
2806
+ platformSerialIdentifier:
2807
+ - null
2808
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2560
2809
  S3_SRA_BS:
2561
2810
  productType: SR_1_SRA_BS
2562
2811
  collection: SENTINEL-3
2563
2812
  metadata_mapping:
2564
2813
  cloudCover: '$.null'
2814
+ platformSerialIdentifier:
2815
+ - null
2816
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2565
2817
  S3_LAN:
2566
2818
  productType: SR_2_LAN___
2567
2819
  collection: SENTINEL-3
2568
2820
  metadata_mapping:
2569
2821
  cloudCover: '$.null'
2822
+ platformSerialIdentifier:
2823
+ - null
2824
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2825
+ S3_LAN_HY:
2826
+ productType: SR_2_LAN_HY
2827
+ collection: SENTINEL-3
2828
+ metadata_mapping:
2829
+ cloudCover: '$.null'
2830
+ platformSerialIdentifier:
2831
+ - null
2832
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2833
+ S3_LAN_SI:
2834
+ productType: SR_2_LAN_SI
2835
+ collection: SENTINEL-3
2836
+ metadata_mapping:
2837
+ cloudCover: '$.null'
2838
+ platformSerialIdentifier:
2839
+ - null
2840
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2841
+ S3_LAN_LI:
2842
+ productType: SR_2_LAN_LI
2843
+ collection: SENTINEL-3
2844
+ metadata_mapping:
2845
+ cloudCover: '$.null'
2846
+ platformSerialIdentifier:
2847
+ - null
2848
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2570
2849
  S3_WAT:
2571
2850
  productType: SR_2_WAT___
2572
2851
  collection: SENTINEL-3
2573
2852
  metadata_mapping:
2574
2853
  cloudCover: '$.null'
2854
+ platformSerialIdentifier:
2855
+ - null
2856
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2575
2857
  # S3 OLCI
2576
2858
  S3_EFR:
2577
2859
  productType: OL_1_EFR___
2578
2860
  collection: SENTINEL-3
2861
+ metadata_mapping:
2862
+ platformSerialIdentifier:
2863
+ - null
2864
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2579
2865
  S3_ERR:
2580
2866
  productType: OL_1_ERR___
2581
2867
  collection: SENTINEL-3
2868
+ metadata_mapping:
2869
+ platformSerialIdentifier:
2870
+ - null
2871
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2582
2872
  S3_OLCI_L2LRR:
2583
2873
  productType: OL_2_LRR___
2584
2874
  collection: SENTINEL-3
2875
+ metadata_mapping:
2876
+ platformSerialIdentifier:
2877
+ - null
2878
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2585
2879
  S3_OLCI_L2LFR:
2586
2880
  productType: OL_2_LFR___
2587
2881
  collection: SENTINEL-3
2882
+ metadata_mapping:
2883
+ platformSerialIdentifier:
2884
+ - null
2885
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2588
2886
  S3_OLCI_L2WRR:
2589
2887
  productType: OL_2_WRR___
2590
2888
  collection: SENTINEL-3
2889
+ metadata_mapping:
2890
+ platformSerialIdentifier:
2891
+ - null
2892
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2591
2893
  S3_OLCI_L2WFR:
2592
2894
  productType: OL_2_WFR___
2593
2895
  collection: SENTINEL-3
2896
+ metadata_mapping:
2897
+ platformSerialIdentifier:
2898
+ - null
2899
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2594
2900
  # S3 SLSTR
2595
2901
  S3_SLSTR_L1RBT:
2596
2902
  productType: SL_1_RBT___
2597
2903
  collection: SENTINEL-3
2904
+ metadata_mapping:
2905
+ platformSerialIdentifier:
2906
+ - null
2907
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2598
2908
  S3_SLSTR_L2LST:
2599
2909
  productType: SL_2_LST___
2600
2910
  collection: SENTINEL-3
2911
+ metadata_mapping:
2912
+ platformSerialIdentifier:
2913
+ - null
2914
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2601
2915
  S3_SLSTR_L2WST:
2602
2916
  productType: SL_2_WST___
2603
2917
  collection: SENTINEL-3
2918
+ metadata_mapping:
2919
+ platformSerialIdentifier:
2920
+ - null
2921
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2604
2922
  S3_SLSTR_L2AOD:
2605
2923
  productType: SL_2_AOD___
2606
2924
  collection: SENTINEL-3
2925
+ metadata_mapping:
2926
+ platformSerialIdentifier:
2927
+ - null
2928
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2607
2929
  S3_SLSTR_L2FRP:
2608
2930
  productType: SL_2_FRP___
2609
2931
  collection: SENTINEL-3
2932
+ metadata_mapping:
2933
+ platformSerialIdentifier:
2934
+ - null
2935
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2610
2936
  # S3 SY
2611
2937
  S3_SY_AOD:
2612
2938
  productType: SY_2_AOD___
2613
2939
  collection: SENTINEL-3
2940
+ metadata_mapping:
2941
+ platformSerialIdentifier:
2942
+ - null
2943
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2614
2944
  S3_SY_SYN:
2615
2945
  productType: SY_2_SYN___
2616
2946
  collection: SENTINEL-3
2947
+ metadata_mapping:
2948
+ platformSerialIdentifier:
2949
+ - null
2950
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2617
2951
  S3_SY_V10:
2618
2952
  productType: SY_2_V10___
2619
2953
  collection: SENTINEL-3
2954
+ metadata_mapping:
2955
+ platformSerialIdentifier:
2956
+ - null
2957
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2620
2958
  S3_SY_VG1:
2621
2959
  productType: SY_2_VG1___
2622
2960
  collection: SENTINEL-3
2961
+ metadata_mapping:
2962
+ platformSerialIdentifier:
2963
+ - null
2964
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2623
2965
  S3_SY_VGP:
2624
2966
  productType: SY_2_VGP___
2625
2967
  collection: SENTINEL-3
2968
+ metadata_mapping:
2969
+ platformSerialIdentifier:
2970
+ - null
2971
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
2626
2972
  # S5P L1
2627
2973
  S5P_L1B_IR_SIR:
2628
2974
  productType: L1B_IR_SIR
@@ -2852,7 +3198,7 @@
2852
3198
  }}
2853
3199
  - '$.id'
2854
3200
  search: !plugin
2855
- type: PostJsonSearchWithStacQueryables
3201
+ type: WekeoSearch
2856
3202
  api_endpoint: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search
2857
3203
  timeout: 60
2858
3204
  need_auth: true
@@ -3097,6 +3443,17 @@
3097
3443
  - '{{"tileId": "{tileIdentifier}"}}'
3098
3444
  - '$.null'
3099
3445
  orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:CLMS_HRVPP_VI"}}'
3446
+ EEA_HRL_TCF:
3447
+ productType: EO:EEA:DAT:HRL:TCF
3448
+ metadata_mapping:
3449
+ startTimeFromAscendingNode:
3450
+ - '{{"year": {startTimeFromAscendingNode#to_datetime_dict("string")}["year"]}}'
3451
+ - $.properties.startdate
3452
+ completionTimeFromAscendingNode: $.properties.enddate
3453
+ resolution:
3454
+ - '{{"resolution": "{resolution}"}}'
3455
+ - '{$.id#replace_str(r"^.*_R([0-9]+m)_.*$",r"\1")}'
3456
+ orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:HRL:TCF"}}'
3100
3457
  COP_DEM_GLO30_DGED:
3101
3458
  productType: EO:ESA:DAT:COP-DEM
3102
3459
  providerProductType: DGE_30
@@ -3135,9 +3492,6 @@
3135
3492
  id:
3136
3493
  - '{{"format": "{id#get_group_name((?P<GeoPackage>geoPackage)|(?P<ESRI fgdb>fgdb)|(?P<GeoTiff100mt>raster100m))}"}}'
3137
3494
  - '$.id'
3138
- providerProductType:
3139
- - '{{"product_type": "{providerProductType}"}}'
3140
- - '$.null'
3141
3495
  startTimeFromAscendingNode: '$.properties.startdate'
3142
3496
  completionTimeFromAscendingNode: '$.properties.enddate'
3143
3497
  format:
@@ -3805,6 +4159,7 @@
3805
4159
  - '='
3806
4160
  - '&'
3807
4161
  - ':'
4162
+ - '%'
3808
4163
  pagination:
3809
4164
  next_page_url_tpl: '{url}?{search}&$top={items_per_page}&$skip={skip}&$expand=Attributes&$expand=Assets'
3810
4165
  count_tpl: '&$count=True'
@@ -3834,17 +4189,17 @@
3834
4189
  - "OData.CSC.Intersects(area=geography'{geometry#to_ewkt}')"
3835
4190
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'productType' and att/OData.CSC.StringAttribute/Value eq '{productType}')"
3836
4191
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformShortName' and att/OData.CSC.StringAttribute/Value eq '{platform}')"
3837
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformSerialIdentifier' and att/OData.CSC.StringAttribute/Value eq '{platformSerialIdentifier}')"
4192
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformSerialIdentifier' and att/OData.CSC.StringAttribute/Value eq '{platformSerialIdentifier#replace_str(\"^S[1-3]\", \"\")}')"
3838
4193
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'spatialResolution' and att/OData.CSC.StringAttribute/Value eq '{resolution}')"
3839
4194
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'authority' and att/OData.CSC.StringAttribute/Value eq '{organisationName}')"
3840
4195
  - "Attributes/OData.CSC.IntegerAttribute/any(att:att/Name eq 'orbitNumber' and att/OData.CSC.StringAttribute/Value eq '{orbitNumber}')"
3841
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'orbitDirection' and att/OData.CSC.StringAttribute/Value eq '{orbitDirection}')"
4196
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'orbitDirection' and att/OData.CSC.StringAttribute/Value eq '{orbitDirection#to_upper}')"
3842
4197
  - "Attributes/OData.CSC.DoubleAttribute/any(att:att/Name eq 'cloudCover' and att/OData.CSC.DoubleAttribute/Value le {cloudCover})"
3843
4198
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'operationalMode' and att/OData.CSC.StringAttribute/Value eq '{sensorMode}')"
3844
- - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'polarisationChannels' and att/OData.CSC.StringAttribute/Value eq '{polarizationChannels}')"
3845
4199
  - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'tileId' and att/OData.CSC.StringAttribute/Value eq '{tileIdentifier}')"
3846
4200
  - "ContentDate/Start lt {completionTimeFromAscendingNode#to_iso_utc_datetime}"
3847
4201
  - "ContentDate/End gt {startTimeFromAscendingNode#to_iso_utc_datetime}"
4202
+ - "Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'polarisationChannels' and att/OData.CSC.StringAttribute/Value eq '{polarizationChannels#replace_str('\\+','%26')}')"
3848
4203
  - contains(Name,'{id}')
3849
4204
  discover_metadata:
3850
4205
  auto_discovery: true
@@ -3900,7 +4255,7 @@
3900
4255
  - '$.Attributes.orbitNumber'
3901
4256
  orbitDirection:
3902
4257
  - null
3903
- - '$.Attributes.orbitDirection'
4258
+ - '{$.Attributes.orbitDirection#to_lower}'
3904
4259
  cloudCover:
3905
4260
  - null
3906
4261
  - '$.Attributes.cloudCover'
@@ -3917,7 +4272,7 @@
3917
4272
  - '$.ContentDate.End'
3918
4273
  polarizationChannels:
3919
4274
  - null
3920
- - '$.Attributes.polarisationChannels'
4275
+ - '{$.Attributes.polarisationChannels#replace_str("&","+")}'
3921
4276
  # Custom parameters (not defined in the base document referenced above)
3922
4277
  id:
3923
4278
  - null
@@ -3947,114 +4302,302 @@
3947
4302
  auth: !plugin
3948
4303
  type: AwsAuth
3949
4304
  auth_error_code: 403
4305
+ s3_endpoint: 'https://eodata.cloudferro.com'
3950
4306
  matching_conf:
3951
4307
  s3_endpoint: 'https://eodata.cloudferro.com'
3952
4308
  products:
3953
4309
  # S1
4310
+ S1_AUX_GNSSRD:
4311
+ productType: AUX_GNSSRD
4312
+ collection: SENTINEL-1
4313
+ metadata_mapping:
4314
+ cloudCover: '$.null'
4315
+ platformSerialIdentifier:
4316
+ - null
4317
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4318
+ S1_AUX_MOEORB:
4319
+ productType: AUX_MOEORB
4320
+ collection: SENTINEL-1
4321
+ metadata_mapping:
4322
+ cloudCover: '$.null'
4323
+ platformSerialIdentifier:
4324
+ - null
4325
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4326
+ S1_AUX_POEORB:
4327
+ productType: AUX_POEORB
4328
+ collection: SENTINEL-1
4329
+ metadata_mapping:
4330
+ cloudCover: '$.null'
4331
+ platformSerialIdentifier:
4332
+ - null
4333
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4334
+ S1_AUX_PREORB:
4335
+ productType: AUX_PREORB
4336
+ collection: SENTINEL-1
4337
+ metadata_mapping:
4338
+ cloudCover: '$.null'
4339
+ platformSerialIdentifier:
4340
+ - null
4341
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4342
+ S1_AUX_PROQUA:
4343
+ productType: AUX_PROQUA
4344
+ collection: SENTINEL-1
4345
+ metadata_mapping:
4346
+ cloudCover: '$.null'
4347
+ platformSerialIdentifier:
4348
+ - null
4349
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4350
+ S1_AUX_RESORB:
4351
+ productType: AUX_RESORB
4352
+ collection: SENTINEL-1
4353
+ metadata_mapping:
4354
+ cloudCover: '$.null'
4355
+ platformSerialIdentifier:
4356
+ - null
4357
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
3954
4358
  S1_SAR_RAW:
3955
4359
  productType: RAW
3956
4360
  collection: SENTINEL-1
3957
4361
  metadata_mapping:
3958
4362
  cloudCover: '$.null'
4363
+ platformSerialIdentifier:
4364
+ - null
4365
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
3959
4366
  S1_SAR_GRD:
3960
4367
  productType: GRD
3961
4368
  collection: SENTINEL-1
3962
4369
  metadata_mapping:
3963
4370
  cloudCover: '$.null'
4371
+ platformSerialIdentifier:
4372
+ - null
4373
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
3964
4374
  S1_SAR_SLC:
3965
4375
  productType: SLC
3966
4376
  collection: SENTINEL-1
3967
4377
  metadata_mapping:
3968
4378
  cloudCover: '$.null'
4379
+ platformSerialIdentifier:
4380
+ - null
4381
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
3969
4382
  S1_SAR_OCN:
3970
4383
  productType: OCN
3971
4384
  collection: SENTINEL-1
3972
4385
  metadata_mapping:
3973
4386
  cloudCover: '$.null'
4387
+ platformSerialIdentifier:
4388
+ - null
4389
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4390
+ S1_SAR_L3_IW_MCM:
4391
+ productType: S1SAR_L3_IW_MCM
4392
+ collection: GLOBAL-MOSAICS
4393
+ metadata_mapping:
4394
+ cloudCover: '$.null'
4395
+ platformSerialIdentifier:
4396
+ - null
4397
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
4398
+ S1_SAR_L3_DH_MCM:
4399
+ productType: S1SAR_L3_DH_MCM
4400
+ collection: GLOBAL-MOSAICS
4401
+ metadata_mapping:
4402
+ cloudCover: '$.null'
4403
+ platformSerialIdentifier:
4404
+ - null
4405
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S1)`'
3974
4406
  # S2
3975
4407
  S2_MSI_L1C:
3976
4408
  collection: SENTINEL-2
3977
4409
  productType: S2MSI1C
4410
+ metadata_mapping:
4411
+ platformSerialIdentifier:
4412
+ - null
4413
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S2)`'
3978
4414
  S2_MSI_L2A:
3979
4415
  collection: SENTINEL-2
3980
4416
  productType: S2MSI2A
4417
+ metadata_mapping:
4418
+ platformSerialIdentifier:
4419
+ - null
4420
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S2)`'
3981
4421
  # S3 SRAL
3982
4422
  S3_SRA:
3983
4423
  productType: SR_1_SRA___
3984
4424
  collection: SENTINEL-3
3985
4425
  metadata_mapping:
3986
4426
  cloudCover: '$.null'
4427
+ platformSerialIdentifier:
4428
+ - null
4429
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
3987
4430
  S3_SRA_A:
3988
4431
  productType: SR_1_SRA_A_
3989
4432
  collection: SENTINEL-3
3990
4433
  metadata_mapping:
3991
4434
  cloudCover: '$.null'
4435
+ platformSerialIdentifier:
4436
+ - null
4437
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
3992
4438
  S3_SRA_BS:
3993
4439
  productType: SR_1_SRA_BS
3994
4440
  collection: SENTINEL-3
3995
4441
  metadata_mapping:
3996
4442
  cloudCover: '$.null'
4443
+ platformSerialIdentifier:
4444
+ - null
4445
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
3997
4446
  S3_LAN:
3998
4447
  productType: SR_2_LAN___
3999
4448
  collection: SENTINEL-3
4000
4449
  metadata_mapping:
4001
4450
  cloudCover: '$.null'
4451
+ platformSerialIdentifier:
4452
+ - null
4453
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4454
+ S3_LAN_HY:
4455
+ productType: SR_2_LAN_HY
4456
+ collection: SENTINEL-3
4457
+ metadata_mapping:
4458
+ cloudCover: '$.null'
4459
+ platformSerialIdentifier:
4460
+ - null
4461
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4462
+ S3_LAN_SI:
4463
+ productType: SR_2_LAN_SI
4464
+ collection: SENTINEL-3
4465
+ metadata_mapping:
4466
+ cloudCover: '$.null'
4467
+ platformSerialIdentifier:
4468
+ - null
4469
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4470
+ S3_LAN_LI:
4471
+ productType: SR_2_LAN_LI
4472
+ collection: SENTINEL-3
4473
+ metadata_mapping:
4474
+ cloudCover: '$.null'
4475
+ platformSerialIdentifier:
4476
+ - null
4477
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4002
4478
  S3_WAT:
4003
4479
  productType: SR_2_WAT___
4004
4480
  collection: SENTINEL-3
4005
4481
  metadata_mapping:
4006
4482
  cloudCover: '$.null'
4483
+ platformSerialIdentifier:
4484
+ - null
4485
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4007
4486
  # S3 OLCI
4008
4487
  S3_EFR:
4009
4488
  productType: OL_1_EFR___
4010
4489
  collection: SENTINEL-3
4490
+ metadata_mapping:
4491
+ platformSerialIdentifier:
4492
+ - null
4493
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4011
4494
  S3_ERR:
4012
4495
  productType: OL_1_ERR___
4013
4496
  collection: SENTINEL-3
4497
+ metadata_mapping:
4498
+ platformSerialIdentifier:
4499
+ - null
4500
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4014
4501
  S3_OLCI_L2LRR:
4015
4502
  productType: OL_2_LRR___
4016
4503
  collection: SENTINEL-3
4504
+ metadata_mapping:
4505
+ platformSerialIdentifier:
4506
+ - null
4507
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4017
4508
  S3_OLCI_L2LFR:
4018
4509
  productType: OL_2_LFR___
4019
4510
  collection: SENTINEL-3
4511
+ metadata_mapping:
4512
+ platformSerialIdentifier:
4513
+ - null
4514
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4020
4515
  S3_OLCI_L2WRR:
4021
4516
  productType: OL_2_WRR___
4022
4517
  collection: SENTINEL-3
4518
+ metadata_mapping:
4519
+ platformSerialIdentifier:
4520
+ - null
4521
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4023
4522
  S3_OLCI_L2WFR:
4024
4523
  productType: OL_2_WFR___
4025
4524
  collection: SENTINEL-3
4525
+ metadata_mapping:
4526
+ platformSerialIdentifier:
4527
+ - null
4528
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4026
4529
  # S3 SLSTR
4027
4530
  S3_SLSTR_L1RBT:
4028
4531
  productType: SL_1_RBT___
4029
4532
  collection: SENTINEL-3
4533
+ metadata_mapping:
4534
+ platformSerialIdentifier:
4535
+ - null
4536
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4030
4537
  S3_SLSTR_L2LST:
4031
4538
  productType: SL_2_LST___
4032
4539
  collection: SENTINEL-3
4540
+ metadata_mapping:
4541
+ platformSerialIdentifier:
4542
+ - null
4543
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4033
4544
  S3_SLSTR_L2WST:
4034
4545
  productType: SL_2_WST___
4035
4546
  collection: SENTINEL-3
4547
+ metadata_mapping:
4548
+ platformSerialIdentifier:
4549
+ - null
4550
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4036
4551
  S3_SLSTR_L2AOD:
4037
4552
  productType: SL_2_AOD___
4038
4553
  collection: SENTINEL-3
4554
+ metadata_mapping:
4555
+ platformSerialIdentifier:
4556
+ - null
4557
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4039
4558
  S3_SLSTR_L2FRP:
4040
4559
  productType: SL_2_FRP___
4041
4560
  collection: SENTINEL-3
4561
+ metadata_mapping:
4562
+ platformSerialIdentifier:
4563
+ - null
4564
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4042
4565
  # S3 SY
4043
4566
  S3_SY_AOD:
4044
4567
  productType: SY_2_AOD___
4045
4568
  collection: SENTINEL-3
4569
+ metadata_mapping:
4570
+ platformSerialIdentifier:
4571
+ - null
4572
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4046
4573
  S3_SY_SYN:
4047
4574
  productType: SY_2_SYN___
4048
4575
  collection: SENTINEL-3
4576
+ metadata_mapping:
4577
+ platformSerialIdentifier:
4578
+ - null
4579
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4049
4580
  S3_SY_V10:
4050
4581
  productType: SY_2_V10___
4051
4582
  collection: SENTINEL-3
4583
+ metadata_mapping:
4584
+ platformSerialIdentifier:
4585
+ - null
4586
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4052
4587
  S3_SY_VG1:
4053
4588
  productType: SY_2_VG1___
4054
4589
  collection: SENTINEL-3
4590
+ metadata_mapping:
4591
+ platformSerialIdentifier:
4592
+ - null
4593
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4055
4594
  S3_SY_VGP:
4056
4595
  productType: SY_2_VGP___
4057
4596
  collection: SENTINEL-3
4597
+ metadata_mapping:
4598
+ platformSerialIdentifier:
4599
+ - null
4600
+ - '$.Attributes.platformSerialIdentifier.`sub(/^/, S3)`'
4058
4601
  # S5P L1
4059
4602
  S5P_L1B_IR_SIR:
4060
4603
  productType: L1B_IR_SIR
@@ -4809,7 +5352,7 @@
4809
5352
  - host
4810
5353
  url: https://data.eumetsat.int
4811
5354
  anchor_sentinel: &sentinel_params
4812
- orbitDirection: '$.properties.acquisitionInformation[0].acquisitionParameters.orbitDirection'
5355
+ orbitDirection: '{$.properties.acquisitionInformation[0].acquisitionParameters.orbitDirection#to_lower}'
4813
5356
  relativeOrbitNumber: '$.properties.acquisitionInformation[0].acquisitionParameters.relativeOrbitNumber'
4814
5357
  timeliness: '$.properties.productInformation.timeliness'
4815
5358
  cycleNumber: '$.properties.acquisitionInformation[0].acquisitionParameters.cycleNumber'
@@ -5428,6 +5971,7 @@
5428
5971
  auth: !plugin
5429
5972
  type: AwsAuth
5430
5973
  matching_url: s3://
5974
+ s3_endpoint: https://s3.waw3-1.cloudferro.com
5431
5975
  matching_conf:
5432
5976
  s3_endpoint: https://s3.waw3-1.cloudferro.com
5433
5977
 
@@ -5674,6 +6218,7 @@
5674
6218
  auth: !plugin
5675
6219
  type: AwsAuth
5676
6220
  auth_error_code: 403
6221
+ s3_endpoint: https://s3.datalake.cnes.fr
5677
6222
  matching_conf:
5678
6223
  s3_endpoint: https://s3.datalake.cnes.fr
5679
6224
 
@@ -5939,7 +6484,7 @@
5939
6484
  search: !plugin
5940
6485
  type: StacSearch
5941
6486
  api_endpoint: 'https://fedeo.ceos.org/search'
5942
- ssl_verify: false
6487
+ ssl_verify: true
5943
6488
  timeout: 20
5944
6489
  discover_product_types:
5945
6490
  fetch_url: 'https://fedeo.ceos.org/series/eo:organisationName/CEDA/'