glean-parser 21.0.1__tar.gz → 22.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. {glean_parser-21.0.1 → glean_parser-22.0.0}/.gitignore +2 -0
  2. {glean_parser-21.0.1 → glean_parser-22.0.0}/PKG-INFO +1 -1
  3. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/go_server.py +33 -0
  4. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/javascript.py +17 -11
  5. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/javascript_server.py +20 -16
  6. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/lint.py +14 -0
  7. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/go_server.jinja2 +40 -0
  8. {glean_parser-21.0.1 → glean_parser-22.0.0}/server_telemetry/server-side-pings.yaml +15 -0
  9. glean_parser-22.0.0/tests/data/go_server_labeled_quantity_metrics.yaml +21 -0
  10. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_metrics_unsupported.yaml +17 -0
  11. glean_parser-22.0.0/tests/data/server_deletion_request_metrics.yaml +21 -0
  12. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_go_server.py +146 -0
  13. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_javascript_server.py +68 -0
  14. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_lint.py +26 -0
  15. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_metrics.py +2 -3
  16. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_python_server.py +4 -3
  17. {glean_parser-21.0.1 → glean_parser-22.0.0}/AUTHORS.md +0 -0
  18. {glean_parser-21.0.1 → glean_parser-22.0.0}/LICENSE +0 -0
  19. {glean_parser-21.0.1 → glean_parser-22.0.0}/README.md +0 -0
  20. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/__init__.py +0 -0
  21. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/__main__.py +0 -0
  22. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/data_review.py +0 -0
  23. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/kotlin.py +0 -0
  24. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/markdown.py +0 -0
  25. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/metrics.py +0 -0
  26. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/parser.py +0 -0
  27. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/pings.py +0 -0
  28. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/python_server.py +0 -0
  29. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/ruby_server.py +0 -0
  30. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/rust.py +0 -0
  31. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/rust_server.py +0 -0
  32. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/rust_sym.py +0 -0
  33. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/schemas/metrics.1-0-0.schema.yaml +0 -0
  34. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/schemas/metrics.2-0-0.schema.yaml +0 -0
  35. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/schemas/pings.1-0-0.schema.yaml +0 -0
  36. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/schemas/pings.2-0-0.schema.yaml +0 -0
  37. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/schemas/tags.1-0-0.schema.yaml +0 -0
  38. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/swift.py +0 -0
  39. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/tags.py +0 -0
  40. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/data_review.jinja2 +0 -0
  41. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/javascript.buildinfo.jinja2 +0 -0
  42. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/javascript.jinja2 +0 -0
  43. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/javascript_server.jinja2 +0 -0
  44. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/kotlin.buildinfo.jinja2 +0 -0
  45. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/kotlin.jinja2 +0 -0
  46. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/markdown.jinja2 +0 -0
  47. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/python_server.jinja2 +0 -0
  48. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/qmldir.jinja2 +0 -0
  49. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/ruby_server.jinja2 +0 -0
  50. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/rust.jinja2 +0 -0
  51. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/rust_server.jinja2 +0 -0
  52. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/rust_sym.jinja2 +0 -0
  53. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/templates/swift.jinja2 +0 -0
  54. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/translate.py +0 -0
  55. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/translation_options.py +0 -0
  56. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/util.py +0 -0
  57. {glean_parser-21.0.1 → glean_parser-22.0.0}/glean_parser/validate_ping.py +0 -0
  58. {glean_parser-21.0.1 → glean_parser-22.0.0}/pyproject.toml +0 -0
  59. {glean_parser-21.0.1 → glean_parser-22.0.0}/server_telemetry/sdk-metrics-compat.yaml +0 -0
  60. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/conftest.py +0 -0
  61. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/all_metrics.yaml +0 -0
  62. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/all_pings.yaml +0 -0
  63. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/attribution.yaml +0 -0
  64. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/bad_attribution.yamlx +0 -0
  65. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/bad_ping.yamlx +0 -0
  66. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/categories.yaml +0 -0
  67. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/core.yaml +0 -0
  68. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/custom_ping_no_event_metrics.yaml +0 -0
  69. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/custom_ping_no_event_pings.yaml +0 -0
  70. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/dual_labeled.yaml +0 -0
  71. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/dual_labeled_invalid.yaml +0 -0
  72. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/duplicate_labeled.yaml +0 -0
  73. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/duplicate_send_in_ping.yaml +0 -0
  74. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/empty.yaml +0 -0
  75. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/event_key_ordering.yaml +0 -0
  76. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/events_data_sensitivity.yaml +0 -0
  77. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/events_with_types.yaml +0 -0
  78. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/fxa-server-metrics.yaml +0 -0
  79. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/fxa-server-pings.yaml +0 -0
  80. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_custom_ping_only_metrics.yaml +0 -0
  81. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_custom_ping_only_pings.yaml +0 -0
  82. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_events_and_custom_ping_metrics.yaml +0 -0
  83. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_events_and_custom_ping_pings.yaml +0 -0
  84. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_events_only_metrics.yaml +0 -0
  85. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/go_server_labeled_boolean_metrics.yaml +0 -0
  86. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/invalid-ping-names.yaml +0 -0
  87. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/invalid.yamlx +0 -0
  88. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/jwe.yaml +0 -0
  89. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/metric-with-tags.yaml +0 -0
  90. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/mixed-expirations.yaml +0 -0
  91. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/name_too_similar.yaml +0 -0
  92. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/object.yaml +0 -0
  93. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/old_event_api.yamlx +0 -0
  94. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/ordering.yaml +0 -0
  95. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/pings.yaml +0 -0
  96. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/python_server_metrics_unsupported.yaml +0 -0
  97. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rate.yaml +0 -0
  98. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/redefined_category.yamlx +0 -0
  99. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/redefined_metric.yamlx +0 -0
  100. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/redefined_ping.yamlx +0 -0
  101. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/reserved_categories.yamlx +0 -0
  102. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/ruby_server_metrics_unsupported.yaml +0 -0
  103. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/ruby_server_pings_unsupported.yaml +0 -0
  104. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rust_server_custom_ping_only_metrics.yaml +0 -0
  105. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rust_server_custom_ping_only_pings.yaml +0 -0
  106. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rust_server_events_and_custom_ping_metrics.yaml +0 -0
  107. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rust_server_events_and_custom_ping_pings.yaml +0 -0
  108. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rust_server_events_only_metrics.yaml +0 -0
  109. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/rust_server_metrics_unsupported.yaml +0 -0
  110. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/same_name_different_category.yaml +0 -0
  111. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/schema-violation.yaml +0 -0
  112. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/send_if_empty_with_metrics.yaml +0 -0
  113. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_custom_ping_only_compare.go +0 -0
  114. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_custom_ping_only_compare.rs +0 -0
  115. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_events_and_custom_ping_compare.go +0 -0
  116. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_events_and_custom_ping_compare.rs +0 -0
  117. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_events_compare.rb +0 -0
  118. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_events_only_compare.go +0 -0
  119. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_events_only_compare.rs +0 -0
  120. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_metrics_no_events_no_pings.yaml +0 -0
  121. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_metrics_with_event.yaml +0 -0
  122. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/server_pings.yaml +0 -0
  123. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/single_labeled.yaml +0 -0
  124. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/smaller.yaml +0 -0
  125. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/tags.yaml +0 -0
  126. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/telemetry_mirror.yaml +0 -0
  127. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/text.yaml +0 -0
  128. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/text_invalid.yaml +0 -0
  129. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/unknown_ping_used.yaml +0 -0
  130. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/wrong_key.yamlx +0 -0
  131. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/data/yaml_nits.yamlx +0 -0
  132. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/detekt.yml +0 -0
  133. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-go/test.go.tmpl +0 -0
  134. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-go/test_publisher.go.tmpl +0 -0
  135. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-js/package.json +0 -0
  136. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-js/test.js.tmpl +0 -0
  137. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-py/test.py +0 -0
  138. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-rb/test.rb.tmpl +0 -0
  139. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test-rs/test.rs.tmpl +0 -0
  140. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_cli.py +0 -0
  141. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_go_server_pubsub.py +0 -0
  142. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_javascript.py +0 -0
  143. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_kotlin.py +0 -0
  144. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_markdown.py +0 -0
  145. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_parser.py +0 -0
  146. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_pings.py +0 -0
  147. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_ruby_server.py +0 -0
  148. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_rust.py +0 -0
  149. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_rust_server.py +0 -0
  150. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_rust_sym.py +0 -0
  151. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_swift.py +0 -0
  152. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_tags.py +0 -0
  153. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_translate.py +0 -0
  154. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_utils.py +0 -0
  155. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/test_validate_ping.py +0 -0
  156. {glean_parser-21.0.1 → glean_parser-22.0.0}/tests/util.py +0 -0
@@ -106,5 +106,7 @@ ENV/
106
106
 
107
107
  .vscode/
108
108
 
109
+ *.swp
110
+
109
111
  detekt*.jar
110
112
  ktlint
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: glean-parser
3
- Version: 21.0.1
3
+ Version: 22.0.0
4
4
  Summary: Parser tools for Mozilla's Glean telemetry
5
5
  Project-URL: Homepage, https://mozilla.github.io/glean
6
6
  Project-URL: Repository, https://github.com/mozilla/glean_parser
@@ -45,6 +45,7 @@ SUPPORTED_METRIC_TYPES = [
45
45
  "boolean",
46
46
  "labeled_boolean", # static labels only; dynamic labels are not supported
47
47
  "string_list",
48
+ "labeled_quantity",
48
49
  ]
49
50
 
50
51
 
@@ -83,6 +84,8 @@ def generate_metric_type(metric_type: str) -> str:
83
84
  return "time.Time"
84
85
  elif metric_type == "string_list":
85
86
  return "[]string"
87
+ elif metric_type == "labeled_quantity":
88
+ return "map[string]int64"
86
89
  else:
87
90
  print("❌ Unable to generate Go type from metric type: " + metric_type)
88
91
  exit
@@ -115,6 +118,27 @@ def validate_labeled_boolean(metric: metrics.Metric) -> bool:
115
118
  return True
116
119
 
117
120
 
121
+ def validate_labeled_quantity(metric: metrics.Metric) -> bool:
122
+ """
123
+ Validate that a labeled_quantity metric does not have static labels defined.
124
+
125
+ The Go server outputter does not support static labels. It generates a Go map[string]int64
126
+ Dynamic labels are supported.
127
+
128
+ Returns:
129
+ bool: True if valid, False otherwise
130
+ """
131
+ if getattr(metric, "ordered_labels", None):
132
+ print(
133
+ "❌ Ignoring labeled_quantity metric with static labels: "
134
+ + f"{metric.name}."
135
+ + " Do not define labels in metrics.yaml to use this metric type."
136
+ )
137
+ return False
138
+
139
+ return True
140
+
141
+
118
142
  def output_go(
119
143
  objs: metrics.ObjectTree,
120
144
  output_dir: Path,
@@ -155,6 +179,8 @@ def output_go(
155
179
  # unique list of labeled_boolean metrics used in any ping
156
180
  labeled_boolean_metrics: List[metrics.Metric] = []
157
181
 
182
+ has_labeled_quantity: bool = False
183
+
158
184
  # Go through all metrics in objs and build a map of
159
185
  # ping->list of metric categories->list of metrics
160
186
  # for easier processing in the template.
@@ -177,6 +203,12 @@ def output_go(
177
203
  ):
178
204
  continue
179
205
 
206
+ # Validate labeled_quantity metrics
207
+ if metric.type == "labeled_quantity":
208
+ if not validate_labeled_quantity(metric):
209
+ continue
210
+ has_labeled_quantity = True
211
+
180
212
  for ping in metric.send_in_pings:
181
213
  if metric.type == "event" and metric not in event_metrics:
182
214
  event_metrics.append(metric)
@@ -209,6 +241,7 @@ def output_go(
209
241
  pings=ping_to_metrics,
210
242
  events=event_metrics,
211
243
  labeled_booleans=labeled_boolean_metrics,
244
+ has_labeled_quantity=has_labeled_quantity,
212
245
  transport=transport,
213
246
  )
214
247
  )
@@ -206,17 +206,23 @@ def output(
206
206
  filename = util.camelize(category_key) + extension
207
207
  filepath = output_dir / filename
208
208
 
209
- types = set(
210
- [
211
- # This takes care of the regular metric type imports
212
- # as well as the labeled metric subtype imports,
213
- # thus the removal of the `labeled_` substring.
214
- #
215
- # The actual LabeledMetricType import is conditioned after
216
- # the `has_labeled_metrics` boolean.
217
- obj.type if not obj.type.startswith("labeled_") else obj.type[8:]
218
- for obj in category_val.values()
219
- ]
209
+ types = sorted(
210
+ list(
211
+ set(
212
+ [
213
+ # This takes care of the regular metric type imports
214
+ # as well as the labeled metric subtype imports,
215
+ # thus the removal of the `labeled_` substring.
216
+ #
217
+ # The actual LabeledMetricType import is conditioned after
218
+ # the `has_labeled_metrics` boolean.
219
+ obj.type
220
+ if not obj.type.startswith("labeled_")
221
+ else obj.type[8:]
222
+ for obj in category_val.values()
223
+ ]
224
+ )
225
+ )
220
226
  )
221
227
  has_labeled_metrics = any(
222
228
  getattr(metric, "labeled", False) for metric in category_val.values()
@@ -41,6 +41,10 @@ from . import util
41
41
  # and might require changes to the template.
42
42
  SUPPORTED_METRIC_TYPES = ["string", "event"]
43
43
 
44
+ # Pings defined in the glean-server library (server_telemetry/server-side-pings.yaml)
45
+ # are available to all server apps without needing a local pings.yaml definition.
46
+ KNOWN_SERVER_PINGS = {"events", "server-deletion-request"}
47
+
44
48
 
45
49
  def event_class_name(
46
50
  ping_name: str, metrics_by_type: Dict[str, List[metrics.Metric]]
@@ -164,24 +168,24 @@ def output(
164
168
  + " parser doesn't generate individual metric files. Make sure to pass all"
165
169
  + " your ping and metric definitions in a single invocation of the parser."
166
170
  )
167
- if "pings" not in objs:
168
- # If events are meant to be sent in custom pings, we need to make sure they
169
- # are defined. Otherwise we won't have destination tables defined and
170
- # submissions won't pass validation at ingestion.
171
- if event_metric_exists:
172
- if "events" not in ping_to_metrics:
173
- # Event metrics can be sent in standard `events` ping
174
- # or in custom pings.
175
- print(
176
- "❌ "
177
- + PING_METRIC_ERROR_MSG
178
- + "\n You need to either send your event metrics in standard"
179
- + " `events` ping or define a custom one."
180
- )
181
- return
182
- else:
171
+ # If custom pings are targeted, they must be explicitly defined in a
172
+ # pings.yaml - otherwise destination tables aren't set up and submissions
173
+ # fail ingestion. Pings from the glean-server library are exempt: they're
174
+ # centrally defined and available to all server apps.
175
+ if "pings" not in objs and not all(
176
+ p in KNOWN_SERVER_PINGS for p in ping_to_metrics
177
+ ):
178
+ if not event_metric_exists:
183
179
  print("❌ No ping definition found." + PING_METRIC_ERROR_MSG)
184
180
  return
181
+ if "events" not in ping_to_metrics:
182
+ print(
183
+ "❌ "
184
+ + PING_METRIC_ERROR_MSG
185
+ + "\n You need to either send your event metrics in standard"
186
+ + " `events` ping or define a custom one."
187
+ )
188
+ return
185
189
 
186
190
  if not ping_to_metrics:
187
191
  print("❌ No pings with metrics found." + PING_METRIC_ERROR_MSG)
@@ -369,6 +369,19 @@ def check_redundant_ping(
369
369
  yield ("The word 'custom' is redundant.")
370
370
 
371
371
 
372
+ def check_ohttp_and_info(
373
+ ping: pings.Ping, parser_config: Dict[str, Any]
374
+ ) -> LintGenerator:
375
+ """
376
+ Ensure pings with uploader_capabilities containing "ohttp" have metadata.include_info_sections: false
377
+ """
378
+ if "ohttp" in ping.uploader_capabilities and (
379
+ "include_info_sections" not in ping.metadata
380
+ or ping.metadata["include_info_sections"]
381
+ ):
382
+ yield ("OHTTP requires `metadata.include_info_sections: false`.")
383
+
384
+
372
385
  def check_unknown_ping(
373
386
  check_name: str,
374
387
  check_type: CheckType,
@@ -478,6 +491,7 @@ PING_CHECKS: Dict[
478
491
  "BUG_NUMBER": (check_bug_number, CheckType.error),
479
492
  "TAGS_REQUIRED": (check_tags_required, CheckType.error),
480
493
  "REDUNDANT_PING": (check_redundant_ping, CheckType.error),
494
+ "NO_INFO_FOR_OHTTP": (check_ohttp_and_info, CheckType.error),
481
495
  }
482
496
 
483
497
  ALL_OBJECT_CHECKS: Dict[
@@ -21,6 +21,9 @@ import (
21
21
  {% endif %}
22
22
  "fmt"
23
23
  "io"
24
+ {% if has_labeled_quantity %}
25
+ "sort"
26
+ {% endif %}
24
27
  {% if include_logging %}
25
28
  "strconv"
26
29
  {% endif %}
@@ -185,6 +188,39 @@ func createPingInfo() pingInfo {
185
188
  }
186
189
  }
187
190
 
191
+ {% if has_labeled_quantity %}
192
+ // maxLabelLength is the maximum label length, in bytes, accepted by the
193
+ // Glean ingestion schema. Longer labels cause the entire ping to be rejected.
194
+ const maxLabelLength = 111
195
+
196
+ // otherLabel is the label Glean uses for values recorded under an invalid label.
197
+ // Monitoring can be added for any recorded metrics with the label "__other__"
198
+ const otherLabel = "__other__"
199
+
200
+ // sanitizeLabeledQuantity returns a copy of m with any label longer than
201
+ // maxLabelLength recorded under otherLabel, matching the Glean client SDKs.
202
+ // Labels are processed in sorted order so the value kept under otherLabel is
203
+ // deterministic when several labels are invalid. A nil map is returned as an
204
+ // empty map so it serializes as {} rather than null.
205
+ func sanitizeLabeledQuantity(m map[string]int64) map[string]int64 {
206
+ labels := make([]string, 0, len(m))
207
+ for label := range m {
208
+ labels = append(labels, label)
209
+ }
210
+ sort.Strings(labels)
211
+
212
+ out := make(map[string]int64, len(m))
213
+ for _, label := range labels {
214
+ value := m[label]
215
+ if len(label) > maxLabelLength {
216
+ label = otherLabel
217
+ }
218
+ out[label] = value
219
+ }
220
+ return out
221
+ }
222
+
223
+ {% endif %}
188
224
  // resolveDocumentID returns supplied if non-empty, otherwise a fresh UUID.
189
225
  // Set DocumentID on the ping params struct to share an ID across transports
190
226
  // (e.g., dual-write verification during a logging->pubsub migration).
@@ -346,6 +382,10 @@ func (p {{ ping|ping_type_name }}) buildPayload(appDisplayVersion, appChannel st
346
382
  }
347
383
  {% endfor %}
348
384
  {% endif %}
385
+ {% for metric in metrics_by_type['labeled_quantity'] %}
386
+ // Remap invalid labels to __other__ and ensure nil serializes as {}, not null
387
+ p.{{ metric|metric_argument_name }} = sanitizeLabeledQuantity(p.{{ metric|metric_argument_name }})
388
+ {% endfor %}
349
389
  metrics := metrics{
350
390
  {% for metric_type, metrics in metrics_by_type.items() %}
351
391
  {% if metric_type != 'event' %}
@@ -21,3 +21,18 @@ events:
21
21
  - https://bugzilla.mozilla.org/show_bug.cgi?id=1512938#c3
22
22
  notification_emails:
23
23
  - glean-team@mozilla.com
24
+
25
+ server-deletion-request:
26
+ description: |
27
+ This ping is submitted when a server application needs to request
28
+ deletion of a user's telemetry data from the warehouse.
29
+ Unlike the SDK's `deletion-request` ping, server applications have no
30
+ Glean-managed `client_id`, so this ping carries application-defined
31
+ identifier metrics that the pipeline uses to find the data to delete.
32
+ include_client_id: false
33
+ bugs:
34
+ - https://mozilla-hub.atlassian.net/browse/DENG-4079
35
+ data_reviews:
36
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1587095
37
+ notification_emails:
38
+ - glean-team@mozilla.com
@@ -0,0 +1,21 @@
1
+ # Any copyright is dedicated to the Public Domain.
2
+ # https://creativecommons.org/publicdomain/zero/1.0/
3
+
4
+ ---
5
+ $schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
6
+
7
+ telemetry:
8
+ test_labeled_quantity:
9
+ type: labeled_quantity
10
+ description: >
11
+ Labeled Quantity metric
12
+ bugs:
13
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1234567
14
+ data_reviews:
15
+ - https://example.com/review
16
+ notification_emails:
17
+ - telemetry@example.com
18
+ lifetime: application
19
+ expires: never
20
+ send_in_pings:
21
+ - events
@@ -36,6 +36,23 @@ metric:
36
36
  send_in_pings:
37
37
  - events
38
38
 
39
+ labeled_quantity:
40
+ type: labeled_quantity
41
+ description: >
42
+ labeled quantity
43
+ bugs:
44
+ - TBD
45
+ data_reviews:
46
+ - TBD
47
+ notification_emails:
48
+ - CHANGE-ME@example.com
49
+ lifetime: application
50
+ expires: never
51
+ send_in_pings:
52
+ - events
53
+ labels:
54
+ - sample_feature
55
+
39
56
  object:
40
57
  type: object
41
58
  description: >
@@ -0,0 +1,21 @@
1
+ # Any copyright is dedicated to the Public Domain.
2
+ # https://creativecommons.org/publicdomain/zero/1.0/
3
+
4
+ ---
5
+ $schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
6
+
7
+ account:
8
+ user_id:
9
+ type: string
10
+ description: |
11
+ The firefox/mozilla account id
12
+ bugs:
13
+ - TBD
14
+ data_reviews:
15
+ - TBD
16
+ notification_emails:
17
+ - fake@fake.com
18
+ expires: never
19
+ lifetime: application
20
+ send_in_pings:
21
+ - server-deletion-request
@@ -85,6 +85,38 @@ def test_parser_go_server_labeled_boolean(tmp_path):
85
85
  assert "TelemetryFeatureFlags TelemetryFeatureFlags" in content
86
86
 
87
87
 
88
+ def test_parser_go_server_labeled_quantity_with_labels(tmp_path, capsys):
89
+ """Test that labeled_quantity with static labels is rejected."""
90
+ translate.translate(
91
+ [
92
+ ROOT / "data" / "go_server_metrics_unsupported.yaml",
93
+ ],
94
+ "go_server",
95
+ tmp_path,
96
+ )
97
+ captured = capsys.readouterr()
98
+ assert "Ignoring labeled_quantity metric with static labels" in captured.out
99
+
100
+
101
+ def test_parser_go_server_labeled_quantity(tmp_path):
102
+ """Test that labeled_quantity metrics generate proper map[string]int64."""
103
+ translate.translate(
104
+ ROOT / "data" / "go_server_labeled_quantity_metrics.yaml",
105
+ "go_server",
106
+ tmp_path,
107
+ )
108
+
109
+ assert set(x.name for x in tmp_path.iterdir()) == set(["server_events.go"])
110
+
111
+ # Read generated file and verify struct is created
112
+ with (tmp_path / "server_events.go").open("r", encoding="utf-8") as fd:
113
+ content = fd.read()
114
+
115
+ # Check that it's used in the ping struct
116
+ assert "TelemetryTestLabeledQuantity map[string]int64" in content
117
+ assert "func sanitizeLabeledQuantity" in content
118
+
119
+
88
120
  def test_parser_go_server_events_only(tmp_path):
89
121
  """Test that parser works for definitions that only use events ping"""
90
122
  translate.translate(
@@ -504,6 +536,120 @@ def test_run_logging_labeled_boolean_omitted(tmp_path):
504
536
  assert feature_flags == {}
505
537
 
506
538
 
539
+ @pytest.mark.go_dependency
540
+ def test_run_logging_labeled_quantity(tmp_path):
541
+ glean_module_path = tmp_path / "glean"
542
+
543
+ translate.translate(
544
+ [
545
+ ROOT / "data" / "go_server_labeled_quantity_metrics.yaml",
546
+ ],
547
+ "go_server",
548
+ glean_module_path,
549
+ )
550
+
551
+ code = f"""
552
+ _ = time.Now() // satisfy Go's unused import check for "time"
553
+ logger.RecordEventsPing(
554
+ glean.RequestInfo{{
555
+ UserAgent: "glean-test/1.0",
556
+ IpAddress: "127.0.0.1",
557
+ }},
558
+ glean.EventsPing{{
559
+ TelemetryTestLabeledQuantity: map[string]int64{{
560
+ "tile": 5,
561
+ "{"a" * 111}": 9,
562
+ "{"b" * 112}": 7,
563
+ }},
564
+ }},
565
+ )
566
+ """
567
+
568
+ logged_output = run_logger(tmp_path, code)
569
+ logged_output = json.loads(logged_output)
570
+ fields = logged_output["Fields"]
571
+ payload_str = fields["payload"]
572
+ payload = json.loads(payload_str)
573
+
574
+ assert "glean-server-event" == logged_output["Type"]
575
+ assert "glean.test" == fields["document_namespace"]
576
+ assert "events" == fields["document_type"]
577
+
578
+ # Validate payload against Glean schema
579
+ schema_url = (
580
+ "https://raw.githubusercontent.com/mozilla-services/"
581
+ "mozilla-pipeline-schemas/main/"
582
+ "schemas/glean/glean/glean.1.schema.json"
583
+ )
584
+
585
+ input = io.StringIO(payload_str)
586
+ output = io.StringIO()
587
+ assert validate_ping.validate_ping(input, output, schema_url=schema_url) == 0, (
588
+ output.getvalue()
589
+ )
590
+
591
+ # Labels over 111 bytes are recorded under __other__; 111 bytes is kept
592
+ value = payload["metrics"]["labeled_quantity"]["telemetry.test_labeled_quantity"]
593
+ assert value == {"tile": 5, "a" * 111: 9, "__other__": 7}
594
+
595
+
596
+ @pytest.mark.go_dependency
597
+ def test_run_logging_labeled_quantity_omitted(tmp_path):
598
+ """Test that completely omitted labeled_quantity metrics serialize correctly."""
599
+ glean_module_path = tmp_path / "glean"
600
+
601
+ translate.translate(
602
+ [
603
+ ROOT / "data" / "go_server_labeled_quantity_metrics.yaml",
604
+ ],
605
+ "go_server",
606
+ glean_module_path,
607
+ )
608
+
609
+ # This code logs an events ping without setting the labeled_quantity metric at all, not even to nil.
610
+ code = """
611
+ _ = time.Now() // satisfy Go's unused import check for "time"
612
+ logger.RecordEventsPing(
613
+ glean.RequestInfo{
614
+ UserAgent: "glean-test/1.0",
615
+ IpAddress: "127.0.0.1",
616
+ },
617
+ glean.EventsPing{
618
+ // TelemetryTestLabeledQuantity completely omitted (zero value)
619
+ },
620
+ )
621
+ """
622
+
623
+ logged_output = run_logger(tmp_path, code)
624
+ logged_output = json.loads(logged_output)
625
+ fields = logged_output["Fields"]
626
+ payload_str = fields["payload"]
627
+ payload = json.loads(payload_str)
628
+
629
+ assert "glean-server-event" == logged_output["Type"]
630
+ assert "glean.test" == fields["document_namespace"]
631
+ assert "events" == fields["document_type"]
632
+
633
+ # Validate payload against Glean schema
634
+ schema_url = (
635
+ "https://raw.githubusercontent.com/mozilla-services/"
636
+ "mozilla-pipeline-schemas/main/"
637
+ "schemas/glean/glean/glean.1.schema.json"
638
+ )
639
+
640
+ input = io.StringIO(payload_str)
641
+ output = io.StringIO()
642
+ validation_result = validate_ping.validate_ping(
643
+ input, output, schema_url=schema_url
644
+ )
645
+
646
+ assert validation_result == 0, f"Validation failed. Output: {output.getvalue()}"
647
+
648
+ # Check how labeled_quantity is serialized when omitted
649
+ labeled_quantity_metrics = payload["metrics"]["labeled_quantity"]
650
+ assert labeled_quantity_metrics["telemetry.test_labeled_quantity"] == {}
651
+
652
+
507
653
  @pytest.mark.go_dependency
508
654
  def test_run_logging_nil_string_list(tmp_path):
509
655
  """Test that nil string_list metrics serialize as empty arrays, not null."""
@@ -44,6 +44,19 @@ def test_parser_js_server_metrics_no_ping(tmp_path):
44
44
  assert all(False for _ in tmp_path.iterdir())
45
45
 
46
46
 
47
+ def test_parser_js_server_event_metrics_custom_ping_no_pings_file(tmp_path):
48
+ """Test that no files are generated when event metrics target only a
49
+ custom ping and that ping isn't defined. Event metrics are allowed
50
+ without a pings.yaml only when they go in the standard `events` ping."""
51
+ translate.translate(
52
+ ROOT / "data" / "go_server_custom_ping_only_metrics.yaml",
53
+ "javascript_server",
54
+ tmp_path,
55
+ )
56
+
57
+ assert all(False for _ in tmp_path.iterdir())
58
+
59
+
47
60
  def test_parser_js_server(tmp_path):
48
61
  """Test that no files are generated if only metric definitions
49
62
  are provided without pings."""
@@ -143,6 +156,61 @@ eventLogger.record({ user_agent: "glean-test/1.0", event_name: "testing" });
143
156
  )
144
157
 
145
158
 
159
+ @pytest.mark.parametrize(
160
+ "outputter,ext,extra_assertions",
161
+ [
162
+ (
163
+ "javascript_server",
164
+ "js",
165
+ ["'account.user_id'", "document_type: 'server-deletion-request'"],
166
+ ),
167
+ ("typescript_server", "ts", ["account_user_id: string"]),
168
+ ],
169
+ )
170
+ def test_parser_server_deletion_request_standalone(
171
+ tmp_path, outputter, ext, extra_assertions
172
+ ):
173
+ """Test that server-deletion-request ping code is generated when metrics
174
+ target it, even without a pings.yaml file."""
175
+ translate.translate(
176
+ ROOT / "data" / "server_deletion_request_metrics.yaml",
177
+ outputter,
178
+ tmp_path,
179
+ )
180
+
181
+ assert set(x.name for x in tmp_path.iterdir()) == {f"server_events.{ext}"}
182
+
183
+ content = (tmp_path / f"server_events.{ext}").read_text(encoding="utf-8")
184
+ assert "ServerDeletionRequestServerEvent" in content
185
+ assert "createServerDeletionRequestEvent" in content
186
+ for expected in extra_assertions:
187
+ assert expected in content
188
+
189
+
190
+ def test_parser_js_server_deletion_request_with_other_pings(tmp_path):
191
+ """Test that server-deletion-request ping is generated alongside regular
192
+ pings when metrics target both."""
193
+ translate.translate(
194
+ [
195
+ ROOT / "data" / "fxa-server-pings.yaml",
196
+ ROOT / "data" / "fxa-server-metrics.yaml",
197
+ ROOT / "data" / "server_deletion_request_metrics.yaml",
198
+ ],
199
+ "javascript_server",
200
+ tmp_path,
201
+ )
202
+
203
+ assert set(x.name for x in tmp_path.iterdir()) == set(["server_events.js"])
204
+
205
+ with (tmp_path / "server_events.js").open("r", encoding="utf-8") as fd:
206
+ content = fd.read()
207
+ # Both ping classes should be generated
208
+ assert "AccountsEventsServerEvent" in content
209
+ assert "ServerDeletionRequestServerEvent" in content
210
+ assert "createAccountsEventsEvent" in content
211
+ assert "createServerDeletionRequestEvent" in content
212
+
213
+
146
214
  @pytest.mark.node_dependency
147
215
  def test_logging_events_ping_with_event_metrics(tmp_path):
148
216
  translate.translate(
@@ -966,3 +966,29 @@ def test_events_on_metrics_ping(content, num_nits):
966
966
  assert len(nits) == num_nits
967
967
  if num_nits > 0:
968
968
  assert set(["EVENT_ON_NON_EVENTS_PING"]) == set(v.check_name for v in nits)
969
+
970
+
971
+ def test_no_info_for_ohttp():
972
+ """
973
+ Test that pings must exclude *_info sections if they require OHTTP.
974
+ """
975
+ content = {
976
+ "uno": {"uploader_capabilities": ["ohttp"]},
977
+ "dos": {
978
+ "uploader_capabilities": ["ohttp"],
979
+ "metadata": {"include_info_sections": True},
980
+ },
981
+ "tres": {
982
+ "uploader_capabilities": ["ohttp"],
983
+ "metadata": {"include_info_sections": False},
984
+ },
985
+ }
986
+
987
+ content = util.add_required_ping(content)
988
+ all_pings = parser.parse_objects([content])
989
+ errs = list(all_pings)
990
+ assert len(errs) == 0
991
+
992
+ nits = lint.lint_metrics(all_pings.value)
993
+ assert len(nits) == 2
994
+ assert set(["NO_INFO_FOR_OHTTP"]) == set(v.check_name for v in nits)
@@ -3,12 +3,11 @@
3
3
  # Any copyright is dedicated to the Public Domain.
4
4
  # http://creativecommons.org/publicdomain/zero/1.0/
5
5
 
6
- import datetime
7
-
8
6
  import pytest
9
7
 
10
8
  from glean_parser import parser
11
9
  from glean_parser import metrics
10
+ from glean_parser import util
12
11
 
13
12
 
14
13
  def test_metrics_match_schema():
@@ -52,7 +51,7 @@ def test_expires():
52
51
  """
53
52
  for date, expired in [
54
53
  ("2018-06-10", True),
55
- (datetime.datetime.now(datetime.timezone.utc).date().isoformat(), True),
54
+ (util.now_epoch().date().isoformat(), True),
56
55
  ("3000-01-01", False),
57
56
  ]:
58
57
  m = metrics.Boolean(
@@ -9,6 +9,7 @@ import json
9
9
 
10
10
  import shutil
11
11
  import subprocess
12
+ import sys
12
13
 
13
14
 
14
15
  from glean_parser import translate
@@ -48,9 +49,9 @@ def test_logging(tmp_path):
48
49
  shutil.copy(ROOT / "test-py" / "test.py", tmp_path)
49
50
 
50
51
  # run test script
51
- logged_output = subprocess.check_output(["python", "test.py"], cwd=tmp_path).decode(
52
- "utf-8"
53
- )
52
+ logged_output = subprocess.check_output(
53
+ [sys.executable, "test.py"], cwd=tmp_path
54
+ ).decode("utf-8")
54
55
 
55
56
  for log_line in logged_output.splitlines():
56
57
  json_line = json.loads(log_line)
File without changes