glean-parser 22.0.1__tar.gz → 22.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. {glean_parser-22.0.1 → glean_parser-22.1.0}/PKG-INFO +2 -3
  2. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/__main__.py +1 -0
  3. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/data_review.py +4 -1
  4. glean_parser-22.1.0/glean_parser/flows.py +177 -0
  5. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/kotlin.py +17 -0
  6. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/lint.py +75 -4
  7. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/markdown.py +28 -0
  8. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/metrics.py +2 -1
  9. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/parser.py +123 -10
  10. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/pings.py +11 -1
  11. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/rust.py +3 -1
  12. glean_parser-22.1.0/glean_parser/schemas/flows.1-0-0.schema.yaml +178 -0
  13. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/swift.py +3 -1
  14. glean_parser-22.1.0/glean_parser/templates/kotlin.flows.jinja2 +42 -0
  15. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/markdown.jinja2 +29 -1
  16. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/rust.jinja2 +31 -0
  17. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/rust_sym.jinja2 +1 -0
  18. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/swift.jinja2 +23 -0
  19. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/translate.py +13 -1
  20. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/util.py +2 -19
  21. {glean_parser-22.0.1 → glean_parser-22.1.0}/pyproject.toml +0 -1
  22. glean_parser-22.1.0/tests/data/flows.yaml +40 -0
  23. glean_parser-22.1.0/tests/data/flows_metrics.yaml +25 -0
  24. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/object.yaml +7 -7
  25. glean_parser-22.1.0/tests/data/reserved_pings.yamlx +27 -0
  26. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/unknown_ping_used.yaml +2 -0
  27. glean_parser-22.1.0/tests/test_flows.py +378 -0
  28. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_lint.py +1 -1
  29. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_parser.py +32 -9
  30. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_python_server.py +2 -0
  31. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_rust.py +21 -5
  32. {glean_parser-22.0.1 → glean_parser-22.1.0}/.gitignore +0 -0
  33. {glean_parser-22.0.1 → glean_parser-22.1.0}/AUTHORS.md +0 -0
  34. {glean_parser-22.0.1 → glean_parser-22.1.0}/LICENSE +0 -0
  35. {glean_parser-22.0.1 → glean_parser-22.1.0}/README.md +0 -0
  36. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/__init__.py +0 -0
  37. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/go_server.py +0 -0
  38. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/javascript.py +0 -0
  39. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/javascript_server.py +0 -0
  40. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/python_server.py +0 -0
  41. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/ruby_server.py +0 -0
  42. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/rust_server.py +0 -0
  43. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/rust_sym.py +0 -0
  44. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/schemas/metrics.1-0-0.schema.yaml +0 -0
  45. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/schemas/metrics.2-0-0.schema.yaml +0 -0
  46. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/schemas/pings.1-0-0.schema.yaml +0 -0
  47. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/schemas/pings.2-0-0.schema.yaml +0 -0
  48. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/schemas/tags.1-0-0.schema.yaml +0 -0
  49. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/tags.py +0 -0
  50. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/data_review.jinja2 +0 -0
  51. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/go_server.jinja2 +0 -0
  52. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/javascript.buildinfo.jinja2 +0 -0
  53. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/javascript.jinja2 +0 -0
  54. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/javascript_server.jinja2 +0 -0
  55. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/kotlin.buildinfo.jinja2 +0 -0
  56. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/kotlin.jinja2 +0 -0
  57. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/python_server.jinja2 +0 -0
  58. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/qmldir.jinja2 +0 -0
  59. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/ruby_server.jinja2 +0 -0
  60. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/templates/rust_server.jinja2 +0 -0
  61. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/translation_options.py +0 -0
  62. {glean_parser-22.0.1 → glean_parser-22.1.0}/glean_parser/validate_ping.py +0 -0
  63. {glean_parser-22.0.1 → glean_parser-22.1.0}/server_telemetry/sdk-metrics-compat.yaml +0 -0
  64. {glean_parser-22.0.1 → glean_parser-22.1.0}/server_telemetry/server-side-pings.yaml +0 -0
  65. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/conftest.py +0 -0
  66. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/all_metrics.yaml +0 -0
  67. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/all_pings.yaml +0 -0
  68. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/attribution.yaml +0 -0
  69. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/bad_attribution.yamlx +0 -0
  70. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/bad_ping.yamlx +0 -0
  71. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/categories.yaml +0 -0
  72. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/core.yaml +0 -0
  73. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/custom_ping_no_event_metrics.yaml +0 -0
  74. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/custom_ping_no_event_pings.yaml +0 -0
  75. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/dual_labeled.yaml +0 -0
  76. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/dual_labeled_invalid.yaml +0 -0
  77. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/duplicate_labeled.yaml +0 -0
  78. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/duplicate_send_in_ping.yaml +0 -0
  79. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/empty.yaml +0 -0
  80. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/event_key_ordering.yaml +0 -0
  81. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/events_data_sensitivity.yaml +0 -0
  82. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/events_with_types.yaml +0 -0
  83. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/fxa-server-metrics.yaml +0 -0
  84. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/fxa-server-pings.yaml +0 -0
  85. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_custom_ping_only_metrics.yaml +0 -0
  86. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_custom_ping_only_pings.yaml +0 -0
  87. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_events_and_custom_ping_metrics.yaml +0 -0
  88. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_events_and_custom_ping_pings.yaml +0 -0
  89. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_events_only_metrics.yaml +0 -0
  90. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_labeled_boolean_metrics.yaml +0 -0
  91. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_labeled_quantity_metrics.yaml +0 -0
  92. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/go_server_metrics_unsupported.yaml +0 -0
  93. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/invalid-ping-names.yaml +0 -0
  94. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/invalid.yamlx +0 -0
  95. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/jwe.yaml +0 -0
  96. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/metric-with-tags.yaml +0 -0
  97. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/mixed-expirations.yaml +0 -0
  98. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/name_too_similar.yaml +0 -0
  99. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/old_event_api.yamlx +0 -0
  100. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/ordering.yaml +0 -0
  101. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/pings.yaml +0 -0
  102. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/python_server_metrics_unsupported.yaml +0 -0
  103. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rate.yaml +0 -0
  104. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/redefined_category.yamlx +0 -0
  105. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/redefined_metric.yamlx +0 -0
  106. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/redefined_ping.yamlx +0 -0
  107. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/reserved_categories.yamlx +0 -0
  108. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/ruby_server_metrics_unsupported.yaml +0 -0
  109. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/ruby_server_pings_unsupported.yaml +0 -0
  110. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rust_server_custom_ping_only_metrics.yaml +0 -0
  111. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rust_server_custom_ping_only_pings.yaml +0 -0
  112. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rust_server_events_and_custom_ping_metrics.yaml +0 -0
  113. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rust_server_events_and_custom_ping_pings.yaml +0 -0
  114. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rust_server_events_only_metrics.yaml +0 -0
  115. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/rust_server_metrics_unsupported.yaml +0 -0
  116. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/same_name_different_category.yaml +0 -0
  117. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/schema-violation.yaml +0 -0
  118. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/send_if_empty_with_metrics.yaml +0 -0
  119. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_custom_ping_only_compare.go +0 -0
  120. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_custom_ping_only_compare.rs +0 -0
  121. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_deletion_request_metrics.yaml +0 -0
  122. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_events_and_custom_ping_compare.go +0 -0
  123. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_events_and_custom_ping_compare.rs +0 -0
  124. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_events_compare.rb +0 -0
  125. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_events_only_compare.go +0 -0
  126. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_events_only_compare.rs +0 -0
  127. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_metrics_no_events_no_pings.yaml +0 -0
  128. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_metrics_with_event.yaml +0 -0
  129. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/server_pings.yaml +0 -0
  130. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/single_labeled.yaml +0 -0
  131. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/smaller.yaml +0 -0
  132. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/tags.yaml +0 -0
  133. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/telemetry_mirror.yaml +0 -0
  134. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/text.yaml +0 -0
  135. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/text_invalid.yaml +0 -0
  136. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/wrong_key.yamlx +0 -0
  137. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/data/yaml_nits.yamlx +0 -0
  138. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/detekt.yml +0 -0
  139. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-go/test.go.tmpl +0 -0
  140. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-go/test_publisher.go.tmpl +0 -0
  141. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-js/package.json +0 -0
  142. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-js/test.js.tmpl +0 -0
  143. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-py/test.py +0 -0
  144. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-rb/test.rb.tmpl +0 -0
  145. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test-rs/test.rs.tmpl +0 -0
  146. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_cli.py +0 -0
  147. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_go_server.py +0 -0
  148. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_go_server_pubsub.py +0 -0
  149. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_javascript.py +0 -0
  150. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_javascript_server.py +0 -0
  151. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_kotlin.py +0 -0
  152. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_markdown.py +0 -0
  153. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_metrics.py +0 -0
  154. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_pings.py +0 -0
  155. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_ruby_server.py +0 -0
  156. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_rust_server.py +0 -0
  157. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_rust_sym.py +0 -0
  158. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_swift.py +0 -0
  159. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_tags.py +0 -0
  160. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_translate.py +0 -0
  161. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_utils.py +0 -0
  162. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/test_validate_ping.py +0 -0
  163. {glean_parser-22.0.1 → glean_parser-22.1.0}/tests/util.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: glean-parser
3
- Version: 22.0.1
3
+ Version: 22.1.0
4
4
  Summary: Parser tools for Mozilla's Glean telemetry
5
5
  Project-URL: Homepage, https://mozilla.github.io/glean
6
6
  Project-URL: Repository, https://github.com/mozilla/glean_parser
@@ -19,7 +19,6 @@ Classifier: Programming Language :: Python :: 3.12
19
19
  Classifier: Programming Language :: Python :: 3.13
20
20
  Requires-Python: >=3.9
21
21
  Requires-Dist: click>=7
22
- Requires-Dist: diskcache>=4
23
22
  Requires-Dist: jinja2>=2.10.1
24
23
  Requires-Dist: jsonschema>=3.0.2
25
24
  Requires-Dist: platformdirs>=2.4.0
@@ -233,6 +233,7 @@ def dump(input, allow_reserved, allow_missing_files, require_tags):
233
233
  metrics = {
234
234
  metric.identifier(): metric.serialize()
235
235
  for category, probes in results.value.items()
236
+ if not glean_parser.flows.flows_in({category: probes})
236
237
  for probe_name, metric in probes.items()
237
238
  }
238
239
 
@@ -13,6 +13,7 @@ from typing import Sequence
13
13
  import re
14
14
 
15
15
 
16
+ from . import flows
16
17
  from . import parser
17
18
  from . import util
18
19
 
@@ -47,7 +48,9 @@ def generate(
47
48
  durations = set()
48
49
  responsible_emails = set()
49
50
  filtered_metrics = list()
50
- for metrics in all_objects.value.values():
51
+ for category_key, metrics in all_objects.value.items():
52
+ if flows.flows_in({category_key: metrics}):
53
+ continue
51
54
  for metric in metrics.values():
52
55
  if not any([len(reobj.findall(bug)) == 1 for bug in metric.bugs]):
53
56
  continue
@@ -0,0 +1,177 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ # This Source Code Form is subject to the terms of the Mozilla Public
4
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
5
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
+
7
+ """
8
+ Classes for each of the high-level flow types.
9
+ """
10
+
11
+ from typing import Any, Dict, List, Mapping, Optional, Set
12
+
13
+ from . import util
14
+
15
+
16
+ # Milliseconds per `auto_close_timeout` unit.
17
+ TIMEOUT_UNIT_MS = {
18
+ "s": 1000,
19
+ "m": 60 * 1000,
20
+ "h": 60 * 60 * 1000,
21
+ "d": 24 * 60 * 60 * 1000,
22
+ }
23
+
24
+ # The roles an event can play in a flow, in the order they are listed in
25
+ # `flows.yaml`.
26
+ ROLES = ("starts_on", "ends_on", "includes")
27
+
28
+
29
+ class Flow:
30
+ def __init__(
31
+ self,
32
+ name: str,
33
+ description: str,
34
+ bugs: List[str],
35
+ data_reviews: List[str],
36
+ notification_emails: List[str],
37
+ starts_on: List[str],
38
+ ends_on: Optional[List[str]] = None,
39
+ includes: Optional[List[str]] = None,
40
+ auto_close_timeout: Optional[str] = None,
41
+ disabled: bool = False,
42
+ defined_in: Optional[Dict] = None,
43
+ no_lint: Optional[List[str]] = None,
44
+ _validated: bool = False,
45
+ ):
46
+ # Avoid cyclical import
47
+ from . import parser
48
+
49
+ self.name = name
50
+ self.description = description
51
+ self.bugs = bugs
52
+ self.data_reviews = data_reviews
53
+ self.notification_emails = notification_emails
54
+ self.starts_on = starts_on
55
+ self.ends_on = ends_on if ends_on is not None else []
56
+ self.includes = includes if includes is not None else []
57
+ self.auto_close_timeout = auto_close_timeout
58
+ self.disabled = disabled
59
+ self.defined_in = defined_in
60
+ if no_lint is None:
61
+ no_lint = []
62
+ self.no_lint = no_lint
63
+
64
+ # The events this flow names that are not defined in any of the files
65
+ # given to the parser, and are assumed to be defined elsewhere. Filled
66
+ # in once every file has been parsed.
67
+ self.unresolved: Set[str] = set()
68
+
69
+ # _validated indicates whether this flow has already been jsonschema
70
+ # validated (but not any of the Python-level validation).
71
+ if not _validated:
72
+ data: Dict[str, util.JSONType] = {
73
+ "$schema": parser.FLOWS_ID,
74
+ self.name: self._serialize_input(),
75
+ }
76
+ for error in parser.validate(data):
77
+ raise ValueError(error)
78
+
79
+ if not self.starts_on:
80
+ raise ValueError("A flow must have at least one event in `starts_on`.")
81
+
82
+ # An event may play at most one role in a flow.
83
+ seen: Dict[str, str] = {}
84
+ for role in ROLES:
85
+ for event in getattr(self, role):
86
+ if event in seen:
87
+ raise ValueError(
88
+ f"Event '{event}' is in both `{seen[event]}` and `{role}`. "
89
+ "An event may appear in at most one of a flow's lists."
90
+ )
91
+ seen[event] = role
92
+
93
+ @property
94
+ def type(self) -> str:
95
+ return "flow"
96
+
97
+ @property
98
+ def auto_close_timeout_ms(self) -> Optional[int]:
99
+ """
100
+ The `auto_close_timeout`, in milliseconds, or `None` if unset.
101
+ """
102
+ if self.auto_close_timeout is None:
103
+ return None
104
+ value, unit = self.auto_close_timeout[:-1], self.auto_close_timeout[-1]
105
+ return int(value) * TIMEOUT_UNIT_MS[unit]
106
+
107
+ def events(self) -> Dict[str, str]:
108
+ """
109
+ Every event this flow names, mapped to its role: `starts_on`,
110
+ `ends_on`, or `includes`.
111
+ """
112
+ return {event: role for role in ROLES for event in getattr(self, role)}
113
+
114
+ def _serialize_input(self) -> Dict[str, util.JSONType]:
115
+ d = self.serialize()
116
+ modified_dict = util.remove_output_params(d, "defined_in")
117
+ modified_dict = util.remove_output_params(modified_dict, "unresolved")
118
+ return modified_dict
119
+
120
+ def serialize(self) -> Dict[str, util.JSONType]:
121
+ """
122
+ Serialize the flow back to JSON object model.
123
+ """
124
+ d = self.__dict__.copy()
125
+ del d["name"]
126
+ d["unresolved"] = sorted(self.unresolved)
127
+ if d["auto_close_timeout"] is None:
128
+ del d["auto_close_timeout"]
129
+ return d
130
+
131
+
132
+ def flows_in(objs: Mapping[str, Any]) -> Dict[str, "Flow"]:
133
+ """
134
+ Returns the flows in a parsed object tree, keyed by name.
135
+
136
+ Flows are stored under the `flows` key, which is also a valid metric
137
+ category name. The key holds flows only if it holds `Flow` objects; the
138
+ parser rejects a tree that mixes the two.
139
+ """
140
+ candidate = objs.get("flows")
141
+ if not candidate:
142
+ return {}
143
+ if not all(isinstance(obj, Flow) for obj in candidate.values()):
144
+ return {}
145
+ return candidate
146
+
147
+
148
+ # The SDK's name for each role in a flow.
149
+ _REGISTRATION_ROLES = {"starts_on": "Start", "ends_on": "End", "includes": "Member"}
150
+
151
+
152
+ def registration_data(flows: Mapping[str, "Flow"]) -> List[Dict[str, Any]]:
153
+ """
154
+ The flow registration data the Glean SDK loads at initialization, in a
155
+ form the code generators can render directly.
156
+
157
+ Flows are emitted as authored, flow to events, in name order. Each event
158
+ carries its role and whether it was resolved; the SDK builds the reverse
159
+ index itself.
160
+ """
161
+ return [
162
+ {
163
+ "name": flow.name,
164
+ "description": flow.description,
165
+ "events": [
166
+ {
167
+ "identifier": event,
168
+ "role": _REGISTRATION_ROLES[role],
169
+ "unresolved": event in flow.unresolved,
170
+ }
171
+ for event, role in sorted(flow.events().items())
172
+ ],
173
+ "auto_close_timeout_ms": flow.auto_close_timeout_ms,
174
+ "disabled": flow.disabled,
175
+ }
176
+ for _, flow in sorted(flows.items())
177
+ ]
@@ -14,6 +14,7 @@ from pathlib import Path
14
14
  from typing import Any, Dict, List, Optional, Union # noqa
15
15
 
16
16
  from . import __version__
17
+ from . import flows
17
18
  from . import metrics
18
19
  from . import pings
19
20
  from . import util
@@ -231,6 +232,22 @@ def output_kotlin(
231
232
  )
232
233
  fd.write("\n")
233
234
 
235
+ # Write out the flow registration data, only if there are any flows. The
236
+ # filename starts with "Glean" so it can never clash with a metric category.
237
+ flow_data = flows.registration_data(options.get("flows", {}))
238
+ if flow_data:
239
+ flows_template = util.get_jinja2_template("kotlin.flows.jinja2")
240
+ with (output_dir / "GleanFlows.kt").open("w", encoding="utf-8") as fd:
241
+ fd.write(
242
+ flows_template.render(
243
+ parser_version=__version__,
244
+ namespace=namespace,
245
+ glean_namespace=glean_namespace,
246
+ flows=flow_data,
247
+ )
248
+ )
249
+ fd.write("\n")
250
+
234
251
  template = util.get_jinja2_template(
235
252
  "kotlin.jinja2",
236
253
  filters=(
@@ -20,6 +20,7 @@ from typing import (
20
20
  ) # noqa
21
21
 
22
22
 
23
+ from . import flows
23
24
  from . import metrics
24
25
  from . import parser
25
26
  from . import pings
@@ -494,6 +495,44 @@ PING_CHECKS: Dict[
494
495
  "NO_INFO_FOR_OHTTP": (check_ohttp_and_info, CheckType.error),
495
496
  }
496
497
 
498
+
499
+ def check_empty_ends_on(
500
+ flow: flows.Flow, parser_config: Dict[str, Any]
501
+ ) -> LintGenerator:
502
+ if not flow.ends_on:
503
+ yield (
504
+ "The flow has no `ends_on` events, so once an instance of it opens, it "
505
+ "can only close by timing out or when its session ends, and can never "
506
+ "record a conclusive outcome. Until it closes, recording a `starts_on` "
507
+ "event again does not open a new instance: if the user starts the task "
508
+ "over, their new run is counted as part of the instance that is still "
509
+ "open."
510
+ )
511
+
512
+
513
+ def check_unresolved_flow_event(
514
+ flow: flows.Flow, parser_config: Dict[str, Any]
515
+ ) -> LintGenerator:
516
+ roles = flow.events()
517
+ for event in sorted(flow.unresolved):
518
+ yield (
519
+ f"'{event}' in `{roles[event]}` is not defined in the metrics given to "
520
+ "glean_parser, so it is assumed to be defined elsewhere, such as in a "
521
+ "library built separately. Its existence, type, and `in_session` setting "
522
+ "were not checked. If it is a typo, it will never match; a test that "
523
+ "records it and checks its `flows` entry will catch that."
524
+ )
525
+
526
+
527
+ # The checks that operate on individual flows:
528
+ # {NAME: (function, is_error)}
529
+ FLOW_CHECKS: Dict[
530
+ str, Tuple[Callable[[flows.Flow, dict], LintGenerator], CheckType]
531
+ ] = {
532
+ "EMPTY_ENDS_ON": (check_empty_ends_on, CheckType.warning),
533
+ "UNRESOLVED_FLOW_EVENT": (check_unresolved_flow_event, CheckType.warning),
534
+ }
535
+
497
536
  ALL_OBJECT_CHECKS: Dict[
498
537
  str,
499
538
  Tuple[
@@ -546,7 +585,7 @@ def _lint_item_tags(
546
585
 
547
586
 
548
587
  def _lint_pings(
549
- category: Dict[str, Union[metrics.Metric, pings.Ping, tags.Tag]],
588
+ category: Dict[str, Union[metrics.Metric, pings.Ping, tags.Tag, flows.Flow]],
550
589
  parser_config: Dict[str, Any],
551
590
  valid_tag_names: List[str],
552
591
  ) -> List[GlinterNit]:
@@ -578,17 +617,44 @@ def _lint_pings(
578
617
  return nits
579
618
 
580
619
 
620
+ def _lint_flows(
621
+ category: Dict[str, flows.Flow], parser_config: Dict[str, Any]
622
+ ) -> List[GlinterNit]:
623
+ nits: List[GlinterNit] = []
624
+
625
+ for flow_name, flow in sorted(category.items()):
626
+ for check_name, (check_func, check_type) in FLOW_CHECKS.items():
627
+ if check_name in flow.no_lint:
628
+ continue
629
+ nits.extend(
630
+ GlinterNit(check_name, flow_name, msg, check_type)
631
+ for msg in check_func(flow, parser_config)
632
+ )
633
+ return nits
634
+
635
+
581
636
  def _lint_all_objects(
582
- objects: Dict[str, Dict[str, Union[metrics.Metric, pings.Ping, tags.Tag]]],
637
+ objects: Dict[
638
+ str, Dict[str, Union[metrics.Metric, pings.Ping, tags.Tag, flows.Flow]]
639
+ ],
583
640
  parser_config: Dict[str, Any],
584
641
  ) -> List[GlinterNit]:
585
642
  nits: List[GlinterNit] = []
586
643
 
587
- pings = objects.get("pings")
644
+ pings = {}
645
+ metrics = {}
646
+
647
+ for key, item in objects.items():
648
+ if key == "pings":
649
+ pings.update(item)
650
+ elif key == "tags":
651
+ pass
652
+ else:
653
+ metrics.update(item)
654
+
588
655
  if not pings:
589
656
  return []
590
657
 
591
- metrics = objects.get("all_metrics")
592
658
  if not metrics:
593
659
  return []
594
660
 
@@ -644,6 +710,11 @@ def lint_metrics(
644
710
  # currently we have no linting for tags
645
711
  continue
646
712
 
713
+ category_flows = flows.flows_in({category_name: category})
714
+ if category_flows:
715
+ nits.extend(_lint_flows(category_flows, parser_config))
716
+ continue
717
+
647
718
  # The information for whether there's duplicate metrics found within the
648
719
  # same YAML document is on the category value, which is not presently linted
649
720
  # (we lint only its metrics).
@@ -14,6 +14,7 @@ from urllib.parse import urlsplit, parse_qs
14
14
 
15
15
 
16
16
  from . import __version__
17
+ from . import flows
17
18
  from . import metrics
18
19
  from . import pings
19
20
  from . import util
@@ -172,6 +173,20 @@ def data_sensitivity_numbers(
172
173
  return ", ".join(str(x.value) for x in data_sensitivity)
173
174
 
174
175
 
176
+ def timeout_desc(timeout_ms: Optional[int]) -> str:
177
+ """
178
+ A readable description of a flow's `auto_close_timeout`.
179
+ """
180
+ if timeout_ms is None:
181
+ return "none: an instance stays open until it exits or its session ends"
182
+ for unit, ms in (("day", 86400000), ("hour", 3600000), ("minute", 60000)):
183
+ if timeout_ms % ms == 0:
184
+ count = timeout_ms // ms
185
+ return f"{count} {unit}{'s' if count != 1 else ''}"
186
+ count = timeout_ms // 1000
187
+ return f"{count} second{'s' if count != 1 else ''}"
188
+
189
+
175
190
  def output_markdown(
176
191
  objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
177
192
  ) -> None:
@@ -234,6 +249,16 @@ def output_markdown(
234
249
  metrics_by_pings[ping_name], key=lambda x: x.identifier()
235
250
  )
236
251
 
252
+ # The flows each event belongs to, so the event's row can link to them.
253
+ flow_data = flows.registration_data(options.get("flows", {}))
254
+ role_labels = {"Start": "entry", "End": "exit", "Member": "member"}
255
+ flows_by_event: Dict[str, List[Tuple[str, str]]] = defaultdict(list)
256
+ for flow in flow_data:
257
+ for event in flow["events"]:
258
+ flows_by_event[event["identifier"]].append(
259
+ (flow["name"], role_labels[event["role"]])
260
+ )
261
+
237
262
  project_title = options.get("project_title", "this project")
238
263
  introduction_extra = options.get("introduction_extra")
239
264
 
@@ -254,6 +279,8 @@ def output_markdown(
254
279
  lambda x: ping_include_client_id(x, custom_pings_cache),
255
280
  ),
256
281
  ("data_sensitivity_numbers", data_sensitivity_numbers),
282
+ ("event_flows", lambda x: flows_by_event.get(x.identifier(), [])),
283
+ ("timeout_desc", timeout_desc),
257
284
  ),
258
285
  )
259
286
 
@@ -265,6 +292,7 @@ def output_markdown(
265
292
  template.render(
266
293
  parser_version=__version__,
267
294
  metrics_by_pings=metrics_by_pings,
295
+ flows=flow_data,
268
296
  project_title=project_title,
269
297
  introduction_extra=introduction_extra,
270
298
  )
@@ -12,6 +12,7 @@ import enum
12
12
  from typing import Any, Dict, List, Optional, Type, Union # noqa
13
13
 
14
14
 
15
+ from . import flows
15
16
  from . import pings
16
17
  from . import tags
17
18
  from . import util
@@ -629,4 +630,4 @@ class DualLabeledCounter(Metric):
629
630
  return d
630
631
 
631
632
 
632
- ObjectTree = Dict[str, Dict[str, Union[Metric, pings.Ping, tags.Tag]]]
633
+ ObjectTree = Dict[str, Dict[str, Union[Metric, pings.Ping, tags.Tag, flows.Flow]]]
@@ -16,6 +16,7 @@ from typing import Any, cast, Dict, Generator, Iterable, Optional, Set, Tuple, U
16
16
  import jsonschema # type: ignore
17
17
  from jsonschema.exceptions import ValidationError # type: ignore
18
18
 
19
+ from .flows import Flow, flows_in
19
20
  from .metrics import Metric, ObjectTree
20
21
  from .pings import Ping, RESERVED_PING_NAMES
21
22
  from .tags import Tag
@@ -29,6 +30,7 @@ SCHEMAS_DIR = ROOT_DIR / "schemas"
29
30
  METRICS_ID = "moz://mozilla.org/schemas/glean/metrics/2-0-0"
30
31
  PINGS_ID = "moz://mozilla.org/schemas/glean/pings/2-0-0"
31
32
  TAGS_ID = "moz://mozilla.org/schemas/glean/tags/1-0-0"
33
+ FLOWS_ID = "moz://mozilla.org/schemas/glean/flows/1-0-0"
32
34
 
33
35
 
34
36
  def _update_validator(validator):
@@ -56,7 +58,7 @@ def _load_file(
56
58
  filepath: Path, parser_config: Dict[str, Any]
57
59
  ) -> Generator[str, None, Tuple[Dict[str, util.JSONType], Optional[str]]]:
58
60
  """
59
- Load a metrics.yaml or pings.yaml format file.
61
+ Load a metrics.yaml, pings.yaml, tags.yaml, or flows.yaml format file.
60
62
 
61
63
  If the `filepath` does not exist, raises `FileNotFoundError`, unless
62
64
  `parser_config["allow_missing_files"]` is `True`.
@@ -92,12 +94,13 @@ def _load_file(
92
94
  except IndexError:
93
95
  filetype = None
94
96
 
95
- if filetype not in ("metrics", "pings", "tags"):
97
+ if filetype not in ("metrics", "pings", "tags", "flows"):
96
98
  filetype = None
97
99
 
98
- for error in validate(content, filepath):
99
- content = {}
100
- yield error
100
+ if not parser_config.get("skip_schema_validation", False):
101
+ for error in validate(content, filepath):
102
+ content = {}
103
+ yield error
101
104
 
102
105
  return content, filetype
103
106
 
@@ -428,6 +431,104 @@ def _instantiate_tags(
428
431
  sources[tag_key] = filepath
429
432
 
430
433
 
434
+ def _instantiate_flows(
435
+ all_objects: ObjectTree,
436
+ sources: Dict[Any, Path],
437
+ content: Dict[str, util.JSONType],
438
+ filepath: Path,
439
+ config: Dict[str, Any],
440
+ ) -> Generator[str, None, None]:
441
+ """
442
+ Load a list of flows.yaml files, convert the JSON information into Flow
443
+ objects.
444
+ """
445
+ global_no_lint = content.get("no_lint", [])
446
+ assert isinstance(global_no_lint, list)
447
+
448
+ for flow_key, flow_val in sorted(content.items()):
449
+ if flow_key.startswith("$"):
450
+ continue
451
+ if flow_key == "no_lint":
452
+ continue
453
+ if not isinstance(flow_val, dict):
454
+ raise TypeError(f"Invalid content for flow {flow_key}")
455
+ flow_val["name"] = flow_key
456
+ try:
457
+ flow_obj = Flow(
458
+ defined_in=getattr(flow_val, "defined_in", None),
459
+ _validated=True,
460
+ **flow_val,
461
+ )
462
+ except Exception as e:
463
+ yield util.format_error(filepath, f"On flow '{flow_key}'", str(e))
464
+ continue
465
+
466
+ flow_obj.no_lint = sorted(set(flow_obj.no_lint + global_no_lint))
467
+ if isinstance(filepath, Path) and flow_obj.defined_in is not None:
468
+ flow_obj.defined_in["filepath"] = str(filepath)
469
+
470
+ # Flow names share no namespace with pings or tags, so key their
471
+ # sources separately.
472
+ already_seen = sources.get(("flow", flow_key))
473
+ if already_seen is not None:
474
+ # Definitions are never merged: a library must not be able to
475
+ # redefine an application's flow by declaring one of the same name.
476
+ yield util.format_error(
477
+ filepath,
478
+ "",
479
+ f"Duplicate flow name '{flow_key}' already defined in '{already_seen}'",
480
+ )
481
+ else:
482
+ all_objects.setdefault("flows", {})[flow_key] = flow_obj
483
+ sources[("flow", flow_key)] = filepath
484
+
485
+
486
+ def _resolve_flow_events(all_objects: ObjectTree) -> Generator[str, None, None]:
487
+ """
488
+ Check every event the flows name against the metrics that were parsed.
489
+
490
+ An event defined in the files given to the parser must be an `in_session`
491
+ event metric. An event that isn't defined in any of them is assumed to be
492
+ defined elsewhere, such as in a library built separately, and is recorded
493
+ on the flow as unresolved. The linter warns about those.
494
+ """
495
+ stored = all_objects.get("flows", {})
496
+ if not any(isinstance(obj, Flow) for obj in stored.values()):
497
+ return
498
+
499
+ # A metric category named `flows` shares the key flows are stored under.
500
+ if any(isinstance(obj, Metric) for obj in stored.values()):
501
+ yield util.format_error(
502
+ "",
503
+ "",
504
+ "A metrics category named 'flows' can't be used together with flows.yaml.",
505
+ )
506
+ return
507
+
508
+ for flow in flows_in(all_objects).values():
509
+ filepath = (flow.defined_in or {}).get("filepath", "")
510
+ for event, role in flow.events().items():
511
+ category, _, name = event.rpartition(".")
512
+ metric = all_objects.get(category, {}).get(name)
513
+ if not isinstance(metric, Metric):
514
+ flow.unresolved.add(event)
515
+ continue
516
+ if metric.type != "event":
517
+ yield util.format_error(
518
+ filepath,
519
+ f"On flow '{flow.name}'",
520
+ f"'{event}' in `{role}` is a {metric.type} metric. "
521
+ "Flows may only name event metrics.",
522
+ )
523
+ elif not metric.in_session:
524
+ yield util.format_error(
525
+ filepath,
526
+ f"On flow '{flow.name}'",
527
+ f"'{event}' in `{role}` is not `in_session`. "
528
+ "Flows live inside sessions, so they may only name in-session events.",
529
+ )
530
+
531
+
431
532
  def _preprocess_objects(objs: ObjectTree, config: Dict[str, Any]) -> ObjectTree:
432
533
  """
433
534
  Preprocess the object tree to better set defaults.
@@ -456,8 +557,9 @@ def parse_objects(
456
557
  filepaths: Iterable[Path], config: Optional[Dict[str, Any]] = None
457
558
  ) -> Generator[str, None, ObjectTree]:
458
559
  """
459
- Parse one or more metrics.yaml and/or pings.yaml files, returning a tree of
460
- `metrics.Metric`, `pings.Ping`, and `tags.Tag` instances.
560
+ Parse one or more metrics.yaml, pings.yaml, tags.yaml, and/or flows.yaml
561
+ files, returning a tree of `metrics.Metric`, `pings.Ping`, `tags.Tag`, and
562
+ `flows.Flow` instances.
461
563
 
462
564
  The result is a generator over any errors. If there are no errors, the
463
565
  actual metrics can be obtained from `result.value`. For example::
@@ -469,8 +571,9 @@ def parse_objects(
469
571
 
470
572
  The result value is a dictionary of category names to categories, where
471
573
  each category is a dictionary from metric name to `metrics.Metric`
472
- instances. There are also the special categories `pings` and `tags`
473
- containing all of the `pings.Ping` and `tags.Tag` instances, respectively.
574
+ instances. There are also the special categories `pings`, `tags`, and
575
+ `flows` containing all of the `pings.Ping`, `tags.Tag`, and `flows.Flow`
576
+ instances, respectively.
474
577
 
475
578
  :param filepaths: list of Path objects to metrics.yaml, pings.yaml, and/or
476
579
  tags.yaml files
@@ -486,6 +589,10 @@ def parse_objects(
486
589
  the input `filepaths` do not exist.
487
590
  - `interesting`: Contains an array of interesting metrics/ping files.
488
591
  Probes not included in these files will be marked as disabled.
592
+ - `skip_schema_validation`: Don't validate the input files against the
593
+ schema, including checking that `$schema` is a known schema. This is
594
+ only safe when they have been validated elsewhere: invalid input may
595
+ raise exceptions rather than produce errors.
489
596
  """
490
597
  if config is None:
491
598
  config = {}
@@ -507,6 +614,12 @@ def parse_objects(
507
614
  yield from _instantiate_tags(
508
615
  all_objects, sources, content, filepath, config
509
616
  )
617
+ elif filetype == "flows":
618
+ yield from _instantiate_flows(
619
+ all_objects, sources, content, filepath, config
620
+ )
621
+
622
+ yield from _resolve_flow_events(all_objects)
510
623
 
511
624
  if config.get("interesting"):
512
625
  # We're configured to disable probes not included in the interesting list.
@@ -533,7 +646,7 @@ def parse_objects(
533
646
  interesting_metrics_dict[category_key][metric_key] = metric_val
534
647
 
535
648
  for category_key, category_val in all_objects.items():
536
- if category_key == "tags":
649
+ if category_key == "tags" or flows_in({category_key: category_val}):
537
650
  continue
538
651
 
539
652
  for metric_key, metric_val in sorted(category_val.items()):
@@ -14,7 +14,17 @@ from typing import Dict, List, Optional
14
14
  from . import util
15
15
 
16
16
 
17
- RESERVED_PING_NAMES = ["baseline", "metrics", "events", "deletion-request", "default"]
17
+ RESERVED_PING_NAMES = [
18
+ "baseline",
19
+ "metrics",
20
+ "events",
21
+ "deletion-request",
22
+ "default",
23
+ "server-deletion-request",
24
+ "tags",
25
+ "pings",
26
+ "health",
27
+ ]
18
28
 
19
29
 
20
30
  class Ping: