nnpdf 4.1.0__py3-none-any.whl → 4.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. n3fit/backends/keras_backend/MetaModel.py +27 -26
  2. n3fit/backends/keras_backend/callbacks.py +16 -8
  3. n3fit/backends/keras_backend/internal_state.py +13 -2
  4. n3fit/backends/keras_backend/operations.py +26 -26
  5. n3fit/hyper_optimization/hyper_scan.py +3 -9
  6. n3fit/hyper_optimization/penalties.py +11 -8
  7. n3fit/hyper_optimization/rewards.py +65 -34
  8. n3fit/model_gen.py +344 -270
  9. n3fit/model_trainer.py +71 -105
  10. n3fit/performfit.py +2 -7
  11. n3fit/tests/regressions/quickcard_1.json +12 -28
  12. n3fit/tests/regressions/quickcard_3.json +12 -28
  13. n3fit/tests/regressions/quickcard_pol_1.json +10 -26
  14. n3fit/tests/regressions/quickcard_pol_3.json +9 -25
  15. n3fit/tests/regressions/quickcard_qed_1.json +11 -27
  16. n3fit/tests/regressions/quickcard_qed_3.json +11 -27
  17. n3fit/tests/test_hyperopt.py +6 -12
  18. n3fit/tests/test_layers.py +6 -6
  19. n3fit/tests/test_modelgen.py +73 -24
  20. n3fit/tests/test_multireplica.py +52 -16
  21. n3fit/tests/test_penalties.py +7 -8
  22. n3fit/tests/test_preprocessing.py +2 -2
  23. n3fit/tests/test_vpinterface.py +5 -10
  24. n3fit/vpinterface.py +88 -44
  25. {nnpdf-4.1.0.dist-info → nnpdf-4.1.1.dist-info}/METADATA +9 -3
  26. {nnpdf-4.1.0.dist-info → nnpdf-4.1.1.dist-info}/RECORD +105 -67
  27. {nnpdf-4.1.0.dist-info → nnpdf-4.1.1.dist-info}/WHEEL +1 -1
  28. nnpdf_data/_version.py +1 -1
  29. nnpdf_data/commondata/ATLAS_2JET_7TEV_R06/metadata.yaml +16 -5
  30. nnpdf_data/commondata/ATLAS_TTBAR_13P6TEV_TOT/data.yaml +2 -0
  31. nnpdf_data/commondata/ATLAS_TTBAR_13P6TEV_TOT/kinematics.yaml +13 -0
  32. nnpdf_data/commondata/ATLAS_TTBAR_13P6TEV_TOT/metadata.yaml +51 -0
  33. nnpdf_data/commondata/ATLAS_TTBAR_13P6TEV_TOT/uncertainties.yaml +17 -0
  34. nnpdf_data/commondata/ATLAS_TTBAR_5TEV_TOT/data.yaml +2 -0
  35. nnpdf_data/commondata/ATLAS_TTBAR_5TEV_TOT/kinematics.yaml +13 -0
  36. nnpdf_data/commondata/ATLAS_TTBAR_5TEV_TOT/metadata.yaml +52 -0
  37. nnpdf_data/commondata/ATLAS_TTBAR_5TEV_TOT/uncertainties.yaml +22 -0
  38. nnpdf_data/commondata/ATLAS_WPWM_13P6TEV_TOT/data.yaml +3 -0
  39. nnpdf_data/commondata/ATLAS_WPWM_13P6TEV_TOT/kinematics.yaml +17 -0
  40. nnpdf_data/commondata/ATLAS_WPWM_13P6TEV_TOT/metadata.yaml +57 -0
  41. nnpdf_data/commondata/ATLAS_WPWM_13P6TEV_TOT/uncertainties.yaml +8 -0
  42. nnpdf_data/commondata/ATLAS_Z0_13P6TEV_TOT/data.yaml +2 -0
  43. nnpdf_data/commondata/ATLAS_Z0_13P6TEV_TOT/kinematics.yaml +9 -0
  44. nnpdf_data/commondata/ATLAS_Z0_13P6TEV_TOT/metadata.yaml +54 -0
  45. nnpdf_data/commondata/ATLAS_Z0_13P6TEV_TOT/uncertainties.yaml +7 -0
  46. nnpdf_data/commondata/CMS_1JET_8TEV/metadata.yaml +7 -1
  47. nnpdf_data/commondata/CMS_2JET_7TEV/metadata.yaml +16 -19
  48. nnpdf_data/commondata/CMS_TTBAR_13P6TEV_TOT/data.yaml +2 -0
  49. nnpdf_data/commondata/CMS_TTBAR_13P6TEV_TOT/kinematics.yaml +13 -0
  50. nnpdf_data/commondata/CMS_TTBAR_13P6TEV_TOT/metadata.yaml +51 -0
  51. nnpdf_data/commondata/CMS_TTBAR_13P6TEV_TOT/uncertainties.yaml +12 -0
  52. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/data_d2Sig_dmttBar_dyttBar.yaml +17 -0
  53. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/data_dSig_dmttBar.yaml +8 -0
  54. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/data_dSig_dpTt.yaml +8 -0
  55. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/data_dSig_dyt.yaml +11 -0
  56. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/filter.py +260 -0
  57. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/kinematics_d2Sig_dmttBar_dyttBar.yaml +193 -0
  58. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/kinematics_dSig_dmttBar.yaml +57 -0
  59. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/kinematics_dSig_dpTt.yaml +57 -0
  60. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/kinematics_dSig_dyt.yaml +81 -0
  61. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/metadata.yaml +114 -0
  62. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/rawdata/mtt_abs_parton.yaml +828 -0
  63. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/rawdata/mttytt-abs_parton.yaml +1899 -0
  64. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/rawdata/ptt_abs_parton.yaml +828 -0
  65. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/rawdata/submission.yaml +47 -0
  66. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/rawdata/yt_abs_parton.yaml +1179 -0
  67. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/uncertainties_d2Sig_dmttBar_dyttBar.yaml +2282 -0
  68. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/uncertainties_dSig_dmttBar.yaml +1256 -0
  69. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/uncertainties_dSig_dpTt.yaml +1256 -0
  70. nnpdf_data/commondata/CMS_TTBAR_13TEV_2L_138FB-1_DIF/uncertainties_dSig_dyt.yaml +1598 -0
  71. nnpdf_data/commondata/CMS_TTBAR_13TEV_35P9FB-1_TOT/data.yaml +2 -0
  72. nnpdf_data/commondata/CMS_TTBAR_13TEV_35P9FB-1_TOT/kinematics.yaml +13 -0
  73. nnpdf_data/commondata/CMS_TTBAR_13TEV_35P9FB-1_TOT/metadata.yaml +51 -0
  74. nnpdf_data/commondata/CMS_TTBAR_13TEV_35P9FB-1_TOT/uncertainties.yaml +17 -0
  75. nnpdf_data/commondata/CMS_TTBAR_5TEV_TOT/metadata.yaml +1 -1
  76. nnpdf_data/commondata/NNPDF_POS_2P24GEV/metadata.yaml +60 -0
  77. nnpdf_data/commondata/dataset_names.yml +6 -1
  78. nnpdf_data/theory_cards/41000010.yaml +42 -0
  79. nnpdf_data/theory_cards/41000011.yaml +43 -0
  80. nnpdf_data/theory_cards/41000012.yaml +43 -0
  81. nnpdf_data/theory_cards/41000013.yaml +42 -0
  82. nnpdf_data/theory_cards/41000014.yaml +43 -0
  83. nnpdf_data/theory_cards/41000015.yaml +43 -0
  84. validphys/_version.py +1 -1
  85. validphys/config.py +30 -10
  86. validphys/convolution.py +37 -14
  87. validphys/coredata.py +15 -5
  88. validphys/covmats.py +9 -2
  89. validphys/dataplots.py +1 -1
  90. validphys/filters.py +17 -3
  91. validphys/fkparser.py +11 -1
  92. validphys/gridvalues.py +1 -0
  93. validphys/hessian2mc.py +5 -5
  94. validphys/lhaindex.py +5 -0
  95. validphys/loader.py +1 -1
  96. validphys/n3fit_data.py +107 -61
  97. validphys/nnprofile_default.yaml +2 -1
  98. validphys/pineparser.py +12 -2
  99. validphys/scripts/postfit.py +4 -4
  100. validphys/scripts/vp_pdfrename.py +8 -9
  101. validphys/tests/conftest.py +6 -2
  102. validphys/tests/test_hessian2mc.py +7 -5
  103. validphys/utils.py +1 -0
  104. n3fit/tests/regressions/quickcard_pol/filter.yml +0 -80
  105. n3fit/tests/regressions/quickcard_pol/nnfit/input/lockfile.yaml +0 -111
  106. n3fit/tests/regressions/quickcard_pol/nnfit/replica_1/quickcard_pol.exportgrid +0 -572
  107. n3fit/tests/regressions/quickcard_pol/nnfit/replica_1/quickcard_pol.json +0 -71
  108. n3fit/tests/regressions/quickcard_pol/nnfit/replica_3/quickcard_pol.exportgrid +0 -615
  109. n3fit/tests/regressions/quickcard_pol/nnfit/replica_3/quickcard_pol.json +0 -71
  110. n3fit/tests/regressions/weights.weights.h5 +0 -0
  111. n3fit/tests/regressions/weights_pol.weights.h5 +0 -0
  112. n3fit/tests/test +0 -1
  113. nnpdf_data/theory_cards/40000099.yaml +0 -41
  114. nnpdf_data/theory_cards/40000099.yml +0 -41
  115. {nnpdf-4.1.0.dist-info → nnpdf-4.1.1.dist-info}/entry_points.txt +0 -0
  116. {nnpdf-4.1.0.dist-info → nnpdf-4.1.1.dist-info/licenses}/LICENSE +0 -0
@@ -1,6 +1,6 @@
1
1
  """
2
- Tests for the layers of n3fit
3
- This module checks that the layers do what they would do with numpy
2
+ Tests for the layers of n3fit
3
+ This module checks that the layers do what they would do with numpy
4
4
  """
5
5
 
6
6
  import dataclasses
@@ -139,7 +139,7 @@ def test_DIS_basis():
139
139
  reference = np.zeros(FLAVS, dtype=bool)
140
140
  for i in comb:
141
141
  reference[i] = True
142
- assert np.alltrue(result == reference)
142
+ np.testing.assert_allclose(result, reference)
143
143
 
144
144
 
145
145
  def test_DY_basis():
@@ -153,7 +153,7 @@ def test_DY_basis():
153
153
  reference = np.zeros((FLAVS, FLAVS))
154
154
  for i, j in comb:
155
155
  reference[i, j] = True
156
- assert np.alltrue(result == reference)
156
+ np.testing.assert_allclose(result, reference)
157
157
 
158
158
 
159
159
  def test_DIS():
@@ -232,7 +232,7 @@ def test_rotation_flavour():
232
232
  pdf = op.numpy_to_tensor(pdf)
233
233
  rotmat = layers.FlavourToEvolution(flav_info, "FLAVOUR")
234
234
  res_layer = rotmat(pdf)
235
- assert np.alltrue(res_np == res_layer)
235
+ np.testing.assert_allclose(res_np, res_layer)
236
236
 
237
237
 
238
238
  def test_rotation_evol():
@@ -257,7 +257,7 @@ def test_rotation_evol():
257
257
  pdf = op.numpy_to_tensor(pdf)
258
258
  rotmat = layers.FlavourToEvolution(flav_info, "EVOL")
259
259
  res_layer = rotmat(pdf)
260
- assert np.alltrue(res_np == res_layer)
260
+ np.testing.assert_allclose(res_np, res_layer)
261
261
 
262
262
 
263
263
  def test_mask():
@@ -1,50 +1,99 @@
1
1
  """
2
- Test for the model generation
2
+ Test for the model generation
3
3
 
4
- These tests check that the generated NN are as expected
5
- It checks that both the number of layers and the shape
6
- of the weights of the layers are what is expected
4
+ These tests check that the generated NN are as expected
5
+ It checks that both the number of layers and the shape
6
+ of the weights of the layers are what is expected
7
7
  """
8
8
 
9
- from n3fit.backends import NN_PREFIX
10
- from n3fit.model_gen import generate_nn
9
+ from dataclasses import asdict
10
+
11
+ import pytest
12
+
13
+ from n3fit.backends import Input
14
+ from n3fit.model_gen import ReplicaSettings, _generate_nn
15
+ from nnpdf_data.utils import parse_input
11
16
 
12
17
  INSIZE = 16
13
- OUT_SIZES = (4, 3)
14
- BASIS_SIZE = 3
15
-
16
- COMMON_ARGS = {
17
- "nodes_in": INSIZE,
18
- "nodes": OUT_SIZES,
19
- "activations": ["sigmoid", "tanh"],
20
- "initializer_name": "glorot_uniform",
21
- "replica_seeds": [0],
22
- "dropout": 0.0,
23
- "regularizer": None,
24
- "regularizer_args": {},
25
- "last_layer_nodes": BASIS_SIZE,
26
- }
18
+ OUT_SIZES = (4, 7, 3)
19
+ BASIS_SIZE = OUT_SIZES[-1]
20
+
21
+
22
+ def _common_generation(architecture, dropout=0.0):
23
+ """Generate a NN with shared configuration with only
24
+ some free parameters"""
25
+ config = {
26
+ "architecture": architecture,
27
+ "nodes": OUT_SIZES,
28
+ "activations": ["sigmoid", "sigmoid", "tanh"],
29
+ "initializer": "glorot_uniform",
30
+ "seed": 27,
31
+ "dropout_rate": dropout,
32
+ "regularizer": None,
33
+ "regularizer_args": {},
34
+ }
35
+ xin = Input(shape=(None, INSIZE), batch_size=1)
36
+ return _generate_nn(xin, **config)
27
37
 
28
38
 
29
39
  def test_generate_dense_network():
30
- nn = generate_nn("dense", **COMMON_ARGS)
40
+ nn_w_dropout = _common_generation("dense", dropout=0.4)
41
+ nn = _common_generation("dense")
31
42
 
32
43
  # The number of layers should be input layer + len(OUT_SIZES)
33
44
  assert len(nn.layers) == len(OUT_SIZES) + 1
45
+ # And one more with dropout
46
+ assert len(nn_w_dropout.layers) == len(OUT_SIZES) + 2
47
+
34
48
  # Check that the number of parameters is as expected
35
- expected_sizes = [(INSIZE, OUT_SIZES[0]), (OUT_SIZES[0]), (*OUT_SIZES,), (OUT_SIZES[1])]
49
+ expected_sizes = [(INSIZE, OUT_SIZES[0])]
50
+ for i, oz in enumerate(OUT_SIZES[:-1]):
51
+ expected_sizes.append((oz,))
52
+ expected_sizes.append((oz, OUT_SIZES[i + 1]))
53
+ expected_sizes.append((OUT_SIZES[-1],))
36
54
  for weight, esize in zip(nn.weights, expected_sizes):
37
55
  assert weight.shape == esize
38
56
 
39
57
 
40
58
  def test_generate_dense_per_flavour_network():
41
- nn = generate_nn("dense_per_flavour", **COMMON_ARGS).get_layer(f"{NN_PREFIX}_0")
59
+ nn = _common_generation("dense_per_flavour")
42
60
 
43
61
  # The number of layers should be input + BASIS_SIZE*len(OUT_SIZES) + concatenate
44
62
  assert len(nn.layers) == BASIS_SIZE * len(OUT_SIZES) + 2
45
63
  # The shape for this network of denses for flavours will depend on the basis_size
46
64
  expected_sizes = []
47
65
  expected_sizes += BASIS_SIZE * [(INSIZE, OUT_SIZES[0]), (OUT_SIZES[0],)]
48
- expected_sizes += BASIS_SIZE * [(OUT_SIZES[0], 1), (1,)]
66
+ for i, oz in enumerate(OUT_SIZES[1:-1]):
67
+ expected_sizes += BASIS_SIZE * [(OUT_SIZES[i], oz), (oz,)]
68
+ expected_sizes += BASIS_SIZE * [(oz, 1), (1,)]
69
+
49
70
  for weight, esize in zip(nn.weights, expected_sizes):
50
71
  assert weight.shape == esize
72
+
73
+
74
+ def test_replica_settings():
75
+ """Checks that the _ReplicaSettings object works as expected and
76
+ that it matches the input of _generate_nn"""
77
+ config = {
78
+ "seed": 8,
79
+ "nodes": [4, 10],
80
+ "activations": ["linear"] * 2,
81
+ "architecture": "dense",
82
+ "initializer": "glorot_uniform",
83
+ "dropout_rate": 0.4,
84
+ }
85
+
86
+ rsettings = parse_input(config, ReplicaSettings)
87
+
88
+ with pytest.raises(ValueError):
89
+ ctmp = {**config, "regularizer_args": {"some": 4}}
90
+ ReplicaSettings(**ctmp)
91
+
92
+ with pytest.raises(ValueError):
93
+ ctmp = {**config, "nodes": [2]}
94
+ ReplicaSettings(**ctmp)
95
+
96
+ x = Input(shape=(None, 2))
97
+ nn = _generate_nn(x, 0, **asdict(rsettings))
98
+
99
+ nn.layers == (1 + len(rsettings.nodes) + 1 * rsettings.dropout_rate)
@@ -1,28 +1,23 @@
1
1
  import numpy as np
2
2
 
3
- from n3fit.model_gen import generate_pdf_model
3
+ from n3fit.model_gen import ReplicaSettings, generate_pdf_model
4
+
5
+ EPS = 1e-9
6
+ FAKE_FL = [
7
+ {"fl": i, "largex": [0.5, 1.5], "smallx": [1.5, 2.5]}
8
+ for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"]
9
+ ]
4
10
 
5
11
 
6
12
  def test_replica_split():
7
13
  """Check that multi replica pdf and concatenated single output pdfs agree"""
8
14
  num_replicas = 3
9
15
  replica_axis = 1
10
- fake_fl = [
11
- {"fl": i, "largex": [0.5, 1.5], "smallx": [1.5, 2.5]}
12
- for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"]
13
- ]
14
- pdf_model = generate_pdf_model(
15
- nodes=[8],
16
- activations=["linear"],
17
- seed=34,
18
- flav_info=fake_fl,
19
- fitbasis="FLAVOUR",
20
- num_replicas=num_replicas,
21
- )
16
+ rps = num_replicas * [ReplicaSettings(nodes=[8], activations=["linear"], seed=34)]
17
+ pdf_model = generate_pdf_model(rps, flav_info=FAKE_FL, fitbasis="FLAVOUR")
22
18
  rng = np.random.default_rng(seed=34)
23
- eps = 1e-9
24
- pdf_input = np.maximum(rng.random((1, 5, 1)), eps)
25
- int_input = np.maximum(rng.random((1, 2_000, 1)), eps)
19
+ pdf_input = np.maximum(rng.random((1, 5, 1)), EPS)
20
+ int_input = np.maximum(rng.random((1, 2_000, 1)), EPS)
26
21
 
27
22
  fake_input = {
28
23
  'pdf_input': np.sort(pdf_input, axis=1),
@@ -36,3 +31,44 @@ def test_replica_split():
36
31
  output_split_stacked = np.stack(output_split, axis=replica_axis)
37
32
 
38
33
  np.testing.assert_allclose(output_full, output_split_stacked, rtol=1e-5)
34
+
35
+
36
+ def test_multimodel(seed=42, xlen=5):
37
+ """Check that we can run different models, with different settings,
38
+ in one go.
39
+
40
+ This tests runs 3 replicas with 1, 2, and 3 layers respectively.
41
+ """
42
+ nodes = [20, 10, 8]
43
+ activations = ["tanh", "sigmoid", "linear"]
44
+ init_array = ["glorot_normal", "glorot_uniform", "random_uniform"]
45
+
46
+ rps = []
47
+ for i, initialization in enumerate(init_array):
48
+ idx = i + 1
49
+ rps.append(
50
+ ReplicaSettings(
51
+ nodes=nodes[-idx:],
52
+ activations=activations[-idx:],
53
+ seed=seed + idx,
54
+ initializer=initialization,
55
+ )
56
+ )
57
+
58
+ rng = np.random.default_rng(seed=seed)
59
+ pdf_input = np.maximum(rng.random((1, xlen, 1)), EPS)
60
+ int_input = np.maximum(rng.random((1, 2000, 1)), EPS)
61
+ fake_input = {
62
+ 'pdf_input': np.sort(pdf_input, axis=1),
63
+ 'xgrid_integration': np.sort(int_input, axis=1),
64
+ }
65
+
66
+ pdf_model = generate_pdf_model(rps, flav_info=FAKE_FL, fitbasis="FLAVOUR")
67
+ output_full = pdf_model(fake_input)
68
+ # Check that the output size is what we expect
69
+ np.testing.assert_array_equal(output_full.shape, (1, len(rps), xlen, 14))
70
+ # And now check that the split model has the right layers
71
+ single_replicas = pdf_model.split_replicas()
72
+
73
+ for i, model in enumerate(single_replicas):
74
+ len(model.get_layer("all_NNs").weights) == 2 * (i + 1)
@@ -1,10 +1,11 @@
1
1
  """
2
- Test the penalties for n3fit hyperopt
2
+ Test the penalties for n3fit hyperopt
3
3
  """
4
+
4
5
  from types import SimpleNamespace
5
6
 
6
7
  from n3fit.hyper_optimization.penalties import integrability, patience, saturation
7
- from n3fit.model_gen import generate_pdf_model
8
+ from n3fit.model_gen import ReplicaSettings, generate_pdf_model
8
9
 
9
10
 
10
11
  def test_saturation():
@@ -13,9 +14,8 @@ def test_saturation():
13
14
  {"fl": i, "largex": [0, 1], "smallx": [1, 2]}
14
15
  for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"]
15
16
  ]
16
- pdf_model = generate_pdf_model(
17
- nodes=[8], activations=["linear"], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR"
18
- )
17
+ rps = [ReplicaSettings(nodes=[8], activations=["linear"], seed=0)]
18
+ pdf_model = generate_pdf_model(rps, flav_info=fake_fl, fitbasis="FLAVOUR")
19
19
  assert isinstance(saturation(pdf_model, 5)[0], float)
20
20
 
21
21
 
@@ -34,7 +34,6 @@ def test_integrability_numbers():
34
34
  {"fl": i, "largex": [0, 1], "smallx": [1, 2]}
35
35
  for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"]
36
36
  ]
37
- pdf_model = generate_pdf_model(
38
- nodes=[8], activations=["linear"], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR"
39
- )
37
+ rps = [ReplicaSettings(nodes=[8], activations=["linear"], seed=0)]
38
+ pdf_model = generate_pdf_model(rps, flav_info=fake_fl, fitbasis="FLAVOUR")
40
39
  assert isinstance(integrability(pdf_model), float)
@@ -102,8 +102,8 @@ def test_constraint():
102
102
  # Check that now everything satisfies the constraint again
103
103
  for w in prepro.weights:
104
104
  if w.trainable:
105
- assert np.alltrue(w.constraint.min_value <= w.numpy())
106
- assert np.alltrue(w.numpy() <= w.constraint.max_value)
105
+ assert np.all(w.constraint.min_value <= w.numpy())
106
+ assert np.all(w.numpy() <= w.constraint.max_value)
107
107
 
108
108
  # Check that other replicas were not affected
109
109
  for wa, wb in zip(weights_after[1:], weights_before[1:]):
@@ -1,12 +1,12 @@
1
1
  """
2
- Test the n3fit-validphys interface
2
+ Test the n3fit-validphys interface
3
3
  """
4
4
 
5
5
  from hypothesis import example, given, settings
6
6
  from hypothesis.strategies import integers
7
7
  import numpy as np
8
8
 
9
- from n3fit.model_gen import generate_pdf_model
9
+ from n3fit.model_gen import ReplicaSettings, generate_pdf_model
10
10
  from n3fit.vpinterface import N3PDF, compute_arclength, integrability_numbers
11
11
  from validphys.pdfgrids import distance_grids, xplotting_grid
12
12
 
@@ -19,14 +19,9 @@ def generate_n3pdf(layers=1, members=1, name="n3fit"):
19
19
  ]
20
20
  nodes = list(np.random.randint(1, 10, size=layers)) + [8]
21
21
  activations = ["tanh"] * layers + ["linear"]
22
- pdf_model = generate_pdf_model(
23
- nodes=nodes,
24
- activations=activations,
25
- seed=np.random.randint(100),
26
- flav_info=fake_fl,
27
- num_replicas=members,
28
- fitbasis="FLAVOUR",
29
- ).split_replicas()
22
+ seeds = np.random.randint(100, size=members)
23
+ rps = [ReplicaSettings(nodes=nodes, activations=activations, seed=seed) for seed in seeds]
24
+ pdf_model = generate_pdf_model(rps, flav_info=fake_fl, fitbasis="FLAVOUR").split_replicas()
30
25
  return N3PDF(pdf_model, name=name)
31
26
 
32
27
 
n3fit/vpinterface.py CHANGED
@@ -6,32 +6,36 @@ Example
6
6
 
7
7
  >>> import numpy as np
8
8
  >>> from n3fit.vpinterface import N3PDF
9
- >>> from n3fit.model_gen import pdfNN_layer_generator
9
+ >>> from n3fit.model_gen import generate_pdf_model, ReplicaSettings
10
10
  >>> from validphys.pdfgrids import xplotting_grid
11
- >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'cbar', 's', 'sbar']]
11
+ >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 's', 'sbar', 'g']]
12
12
  >>> fake_x = np.linspace(1e-3,0.8,3)
13
- >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl)
14
- >>> n3pdf = N3PDF(pdf_model)
13
+ >>> rps = [ReplicaSettings(nodes=[8], activations=["linear"], seed=4)]*4
14
+ >>> pdf_model = generate_pdf_model(rps, flav_info=fake_fl, fitbasis='FLAVOUR')
15
+ >>> n3pdf = N3PDF(pdf_model.split_replicas())
15
16
  >>> res = xplotting_grid(n3pdf, 1.6, fake_x)
16
17
  >>> res.grid_values.error_members().shape
17
- (1, 8, 3)
18
-
18
+ (4, 8, 3)
19
+ # (nreplicas, flavours, x-grid)
19
20
 
20
21
  """
21
22
 
22
23
  from collections.abc import Iterable
24
+ from dataclasses import dataclass
23
25
  from functools import cached_property
24
26
  import logging
25
27
 
26
28
  import numpy as np
27
- import numpy.linalg as la
29
+ import pandas as pd
30
+ import scipy.linalg as la
28
31
 
29
32
  from validphys.arclength import arc_lengths, integrability_number
33
+ from validphys.calcutils import calc_chi2, calc_phi
34
+ from validphys.convolution import central_predictions, predictions
30
35
  from validphys.core import PDF, MCStats
31
- from validphys.covmats import covmat_from_systematics, sqrt_covmat
36
+ from validphys.covmats import dataset_inputs_covmat_from_systematics
32
37
  from validphys.lhapdfset import LHAPDFSet
33
38
  from validphys.pdfbases import ALL_FLAVOURS, check_basis
34
- from validphys.results import abs_chi2_data, phi_data, results
35
39
 
36
40
  log = logging.getLogger(__name__)
37
41
  # Order of the evolution basis output from n3fit
@@ -53,6 +57,13 @@ EVOL_LIST = [
53
57
  ]
54
58
 
55
59
 
60
+ @dataclass
61
+ class HyperoptMetrics:
62
+ chi2: float
63
+ phi2: float
64
+ logp: float
65
+
66
+
56
67
  class N3Stats(MCStats):
57
68
  """The PDFs from n3fit are MC PDFs
58
69
  however, since there is no grid, the CV has to be computed manually"""
@@ -67,13 +78,14 @@ class N3Stats(MCStats):
67
78
  class N3LHAPDFSet(LHAPDFSet):
68
79
  """Extension of LHAPDFSet using n3fit models"""
69
80
 
70
- def __init__(self, name, pdf_models, Q=1.65):
81
+ def __init__(self, name, pdf_models, Q=1.65, is_t0=False):
71
82
  log.debug("Creating LHAPDF-like n3fit PDF")
72
83
  self._error_type = "replicas"
73
84
  self._name = name
74
85
  self._lhapdf_set = pdf_models
75
86
  self._flavors = None
76
87
  self._fitting_q = Q
88
+ self._is_t0 = is_t0
77
89
  self.basis = check_basis("evolution", EVOL_LIST)["basis"]
78
90
 
79
91
  def xfxQ(self, x, Q, n, fl):
@@ -124,12 +136,12 @@ class N3LHAPDFSet(LHAPDFSet):
124
136
  # Register the grid with the photon
125
137
  self._register_photon(mod_xgrid)
126
138
 
127
- if replica is None or replica == 0:
139
+ if replica is None or replica == 0 or self._is_t0:
128
140
  # We need generate output values for all replicas
129
141
  result = np.concatenate(
130
142
  [m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0
131
143
  )
132
- if replica == 0:
144
+ if replica == 0 or self._is_t0:
133
145
  # We want _only_ the central value
134
146
  result = np.mean(result, axis=0, keepdims=True)
135
147
  else:
@@ -227,6 +239,10 @@ class N3PDF(PDF):
227
239
  """If the function needs an LHAPDF object, return a N3LHAPDFSet"""
228
240
  return self._lhapdf_set
229
241
 
242
+ def load_t0(self):
243
+ """Load the central PDF object"""
244
+ return N3LHAPDFSet(self.name, self._models, Q=self._Q, is_t0=True)
245
+
230
246
  def get_nn_weights(self):
231
247
  """Outputs all weights of the NN as numpy.ndarrays"""
232
248
  return [model.get_weights() for model in self._models]
@@ -326,7 +342,6 @@ def compute_arclength(self, q0=1.65, basis="evolution", flavours=None):
326
342
  """
327
343
  Given the layer with the fit basis computes the arc length
328
344
  using the corresponding validphys action
329
-
330
345
  Parameters
331
346
  ----------
332
347
  pdf_function: function
@@ -337,7 +352,6 @@ def compute_arclength(self, q0=1.65, basis="evolution", flavours=None):
337
352
  basis in which to compute the arc length
338
353
  flavours: list
339
354
  output flavours
340
-
341
355
  Example
342
356
  -------
343
357
  >>> from n3fit.vpinterface import N3PDF, compute_arclength
@@ -353,10 +367,9 @@ def compute_arclength(self, q0=1.65, basis="evolution", flavours=None):
353
367
  return ret.stats.central_value()
354
368
 
355
369
 
356
- def compute_phi(n3pdf, experimental_data):
357
- """Compute phi using validphys functions.
358
-
359
- For more info on how phi is calculated; see Eq.(4.6) of 10.1007/JHEP04(2015)040
370
+ def compute_hyperopt_metrics(n3pdf, experimental_data) -> HyperoptMetrics:
371
+ """Compute the different hyperopt quantities from which one defines
372
+ the hyperopt metric.
360
373
 
361
374
  Parameters
362
375
  ----------
@@ -367,43 +380,74 @@ def compute_phi(n3pdf, experimental_data):
367
380
 
368
381
  Returns
369
382
  -------
370
- sum_phi: float
371
- Sum of phi over all experimental group datasets
383
+ HyperoptMetrics: :class:`n3fit.vpinterface.HyperoptMetrics`
384
+ dataclass holding the values of chi2, phi2 and logp
372
385
 
373
386
  Example
374
387
  -------
375
- >>> from n3fit.vpinterface import N3PDF, compute_phi
376
- >>> from n3fit.model_gen import generate_pdf_model
388
+ >>> from n3fit.vpinterface import N3PDF, compute_hyperopt_metrics
389
+ >>> from n3fit.model_gen import generate_pdf_model, ReplicaSettings
377
390
  >>> from validphys.loader import Loader
378
391
  >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'g', 's', 'sbar']]
379
- >>> pdf_model = generate_pdf_model(nodes=[8], activations=['linear'], seed=0, num_replicas=2, flav_info=fake_fl, fitbasis="FLAVOUR")
392
+ >>> rps = [ReplicaSettings(nodes=[8], activations=["linear"], seed=i) for i in [0,1]]
393
+ >>> pdf_model = generate_pdf_model(rps, flav_info=fake_fl, fitbasis="FLAVOUR")
380
394
  >>> n3pdf = N3PDF(pdf_model.split_replicas())
381
- >>> ds = Loader().check_dataset("NMC_NC_NOTFIXED_P_EM-SIGMARED", theoryid=399, cuts="internal")
395
+ >>> ds = Loader().check_dataset("NMC_NC_NOTFIXED_P_EM-SIGMARED", theoryid=40_000_000, cuts="internal", variant="legacy")
382
396
  >>> data_group_spec = Loader().check_experiment("My DataGroupSpec", [ds])
383
- >>> phi = compute_phi(n3pdf, [data_group_spec])
397
+ >>> hyperopt_losses = compute_hyperopt_metrics(n3pdf, [data_group_spec])
384
398
  """
385
- sum_phi = 0.0
386
- ndat_tot = 0
399
+ exp_cv = []
400
+ th_cvs = []
401
+ th_rep = []
402
+ cds_list = []
403
+
387
404
  # Loop over the list of `DataGroupSpec` objects
388
405
  for datagroupspec in experimental_data:
389
406
  # datagroupspec is an instance of `DataGroupSpec`
390
-
391
407
  # Loop over `DataGroupSpec` datasets
392
408
  for datasetspec in datagroupspec.datasets:
393
409
  # datasetspec is an instance of `DataSetSpec`
394
-
395
- # get covariant matrix for each `DataSetSpec`
396
- covmat = covmat_from_systematics(datasetspec.load_commondata(), datasetspec)
397
-
398
- # get experiment info (`DataResult`) and theory predictions (`ThPredictionsResult`)
399
- res = results(datasetspec, n3pdf, covmat, sqrt_covmat(covmat))
400
-
401
- # calculate standard chi2 (all_chi2) and chi2 using PDF central values (central_chi2)
402
- chi2 = abs_chi2_data(res)
403
-
404
- # calculate phi and store phi**2
405
- phi, ndat = phi_data(chi2)
406
- sum_phi += ndat * phi**2
407
- ndat_tot += ndat
408
-
409
- return np.sqrt(sum_phi / ndat_tot)
410
+ # update list of CommonData and corresponding central values
411
+ cd = datasetspec.load_commondata()
412
+ cds_list.append(cd)
413
+ exp_cv.append(cd.central_values)
414
+ # update list of th pred, for the central value and for each replica
415
+ th_cvs.append(central_predictions(datasetspec, n3pdf))
416
+ th_rep.append(predictions(datasetspec, n3pdf))
417
+
418
+ pred_cvs = pd.concat(th_cvs, axis=0, ignore_index=True)
419
+ pred_rep = pd.concat(th_rep, axis=0, ignore_index=True)
420
+ expr_cvs = pd.concat(exp_cv, axis=0, ignore_index=True)
421
+ diffs = pred_cvs.values.flatten() - expr_cvs.values.flatten()
422
+ diffs_reps = pred_rep.values - expr_cvs.values[:, np.newaxis]
423
+
424
+ exp_cov = dataset_inputs_covmat_from_systematics(cds_list, use_weights_in_covmat=False)
425
+ exp_covmat_col = la.cholesky(exp_cov, lower=True)
426
+
427
+ # If there is only one replica, we don't account for PDF covmat
428
+ if pred_rep.shape[1] == 1:
429
+ total_covmat = exp_cov
430
+ else:
431
+ pdf_cov = np.cov(pred_rep.values)
432
+ assert exp_cov.shape == pdf_cov.shape
433
+ total_covmat = exp_cov + pdf_cov
434
+
435
+ # Compute the log_det
436
+ # Normalize the total covmat to central values of experimental data
437
+ norm_total_covmat = total_covmat / np.outer(
438
+ expr_cvs.values.flatten(), expr_cvs.values.flatten()
439
+ )
440
+ norm_total_covmat_chol = la.cholesky(norm_total_covmat, lower=True)
441
+ log_det_total_cov = 2 * np.sum(np.log(np.diag(norm_total_covmat_chol)))
442
+
443
+ # Compute the chi2
444
+ total_covmat_chol = la.cholesky(total_covmat, lower=True)
445
+ chi2 = calc_chi2(sqrtcov=total_covmat_chol, diffs=diffs)
446
+
447
+ # Compute phi2
448
+ phi2 = calc_phi(sqrtcov=exp_covmat_col, diffs=diffs_reps)
449
+
450
+ ndat = len(diffs)
451
+ logp = -0.5 * (len(diffs) * np.log(2 * np.pi) + log_det_total_cov + chi2)
452
+
453
+ return HyperoptMetrics(chi2=chi2 / ndat, phi2=phi2, logp=-logp / ndat)
@@ -1,8 +1,9 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: nnpdf
3
- Version: 4.1.0
3
+ Version: 4.1.1
4
4
  Summary: An open-source machine learning framework for global analyses of parton distributions.
5
5
  License: GPL-3.0-or-later
6
+ License-File: LICENSE
6
7
  Author: NNPDF Collaboration
7
8
  Requires-Python: >=3.9
8
9
  Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -14,9 +15,11 @@ Classifier: Programming Language :: Python :: 3.10
14
15
  Classifier: Programming Language :: Python :: 3.11
15
16
  Classifier: Programming Language :: Python :: 3.12
16
17
  Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Programming Language :: Python :: 3.14
17
19
  Classifier: Topic :: Scientific/Engineering
18
20
  Classifier: Topic :: Scientific/Engineering :: Physics
19
21
  Provides-Extra: docs
22
+ Provides-Extra: jax
20
23
  Provides-Extra: nolha
21
24
  Provides-Extra: parallelhyperopt
22
25
  Provides-Extra: qed
@@ -26,9 +29,10 @@ Requires-Dist: eko (>=0.15.1,<0.16.0)
26
29
  Requires-Dist: fiatlux ; extra == "qed"
27
30
  Requires-Dist: hyperopt
28
31
  Requires-Dist: hypothesis ; extra == "tests"
32
+ Requires-Dist: jax ; extra == "jax"
29
33
  Requires-Dist: joblib
30
34
  Requires-Dist: keras (>=3.1,<4.0)
31
- Requires-Dist: lhapdf-management (>=0.5,<0.6) ; extra == "nolha"
35
+ Requires-Dist: lhapdf-management (>=0.6,<0.7) ; extra == "nolha"
32
36
  Requires-Dist: matplotlib (>=3.9,<4.0)
33
37
  Requires-Dist: numpy
34
38
  Requires-Dist: packaging
@@ -62,7 +66,9 @@ Description-Content-Type: text/markdown
62
66
  [![Docs](https://github.com/NNPDF/nnpdf/actions/workflows/upload_docs.yml/badge.svg)](https://github.com/NNPDF/nnpdf/actions/workflows/upload_docs.yml)
63
67
  [![Commondata](https://github.com/NNPDF/nnpdf/actions/workflows/check_newcd.yml/badge.svg)](https://github.com/NNPDF/nnpdf/actions/workflows/check_newcd.yml)
64
68
 
69
+ [![EPJC](https://img.shields.io/badge/Eur.Phys.J.C-81%20(2021)%2010-958?color=%231A43BF)](https://link.springer.com/article/10.1140/epjc/s10052-021-09747-9)
65
70
  [![DOI](https://zenodo.org/badge/118135201.svg)](https://zenodo.org/badge/latestdoi/118135201)
71
+ [![HSF](https://hepsoftwarefoundation.org/images/HSF-logo/HSF-Affiliated.svg)](https://hepsoftwarefoundation.org/projects/projects)
66
72
 
67
73
  # NNPDF: An open-source machine learning framework for global analyses of parton distributions
68
74