glean-parser 17.0.1__tar.gz → 17.2.0__tar.gz

glean_parser-17.0.1/docs/readme.md → glean_parser-17.2.0/PKG-INFO

@@ -1,3 +1,32 @@
+Metadata-Version: 2.4
+Name: glean-parser
+Version: 17.2.0
+Summary: Parser tools for Mozilla's Glean telemetry
+Project-URL: Homepage, https://mozilla.github.io/glean
+Project-URL: Repository, https://github.com/mozilla/glean_parser
+Project-URL: Changelog, https://github.com/mozilla/glean_parser/blob/main/CHANGELOG.md
+Author-email: The Glean Team <glean-team@mozilla.com>
+License-File: AUTHORS.md
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.8
+Requires-Dist: click>=7
+Requires-Dist: diskcache>=4
+Requires-Dist: jinja2>=2.10.1
+Requires-Dist: jsonschema>=3.0.2
+Requires-Dist: platformdirs>=2.4.0
+Requires-Dist: pyyaml>=5.3.1
+Description-Content-Type: text/markdown
+
 # Glean Parser
 
 Parser tools for Mozilla's Glean telemetry.
@@ -16,17 +45,7 @@ code for various integrations, linting and coverage testing.
 
 ## Requirements
 
--   Python 3.8 (or later)
-
-The following library requirements are installed automatically when
-`glean_parser` is installed by `pip`.
-
--   Click
--   diskcache
--   Jinja2
--   jsonschema
--   platformdirs
--   PyYAML
+- Python 3.8 (or later)
 
 ## Usage
 

{glean_parser-17.0.1 → glean_parser-17.2.0}/README.md

@@ -16,17 +16,7 @@ code for various integrations, linting and coverage testing.
 
 ## Requirements
 
--   Python 3.8 (or later)
-
-The following library requirements are installed automatically when
-`glean_parser` is installed by `pip`.
-
--   Click
--   diskcache
--   Jinja2
--   jsonschema
--   platformdirs
--   PyYAML
+- Python 3.8 (or later)
 
 ## Usage
 

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/lint.py

@@ -134,7 +134,6 @@ def check_unit_in_name(
 
     time_unit = getattr(metric, "time_unit", None)
     memory_unit = getattr(metric, "memory_unit", None)
-    unit = getattr(metric, "unit", None)
 
     if time_unit is not None:
         if (
@@ -175,14 +174,6 @@ def check_unit_in_name(
                 "Confirm the unit is correct and only include memory_unit."
             )
 
-    elif unit is not None:
-        if unit_in_name == unit:
-            yield (
-                f"Suffix '{unit_in_name}' is redundant with unit param "
-                f"'{unit}'. "
-                "Only include unit."
-            )
-
 
 def check_category_generic(
     category_name: str, metrics: Iterable[metrics.Metric]
@@ -581,6 +572,20 @@ def lint_metrics(
 
     nits.extend(_lint_all_objects(objs, parser_config))
 
+    # The information for whether there's duplicate categories found within the
+    # same YAML document is on the objs value, which is not presently linted
+    # (we pull out its metrics, pings, and tags and lint those).
+    # So we perform that custom work here.
+    if getattr(objs, "duplicate", None):
+        nits.append(
+            GlinterNit(
+                "REDEFINED_CATEGORY",
+                getattr(objs, "duplicate", ""),
+                f"Category redefined {objs.duplicate}",  # type: ignore[attr-defined]
+                CheckType.error,
+            )
+        )
+
     for category_name, category in sorted(list(objs.items())):
         if category_name == "pings":
             nits.extend(_lint_pings(category, parser_config, valid_tag_names))
@@ -590,6 +595,20 @@ def lint_metrics(
             # currently we have no linting for tags
             continue
 
+        # The information for whether there's duplicate metrics found within the
+        # same YAML document is on the category value, which is not presently linted
+        # (we lint only its metrics).
+        # So we perform that custom work here.
+        if getattr(objs[category_name], "duplicate", None):
+            nits.append(
+                GlinterNit(
+                    "REDEFINED_METRIC",
+                    category_name,
+                    f"Metric redefined {getattr(objs[category_name], 'duplicate', '')}",
+                    CheckType.error,
+                )
+            )
+
         # Make sure the category has only Metrics, not Pings or Tags
         category_metrics = dict(
             (name, metric)

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/metrics.py

@@ -416,18 +416,23 @@ class LabeledString(Labeled, String):
 class LabeledCounter(Labeled, Counter):
     typename = "labeled_counter"
 
+
 class LabeledCustomDistribution(Labeled, CustomDistribution):
     typename = "labeled_custom_distribution"
 
+
 class LabeledMemoryDistribution(Labeled, MemoryDistribution):
     typename = "labeled_memory_distribution"
 
+
 class LabeledTimingDistribution(Labeled, TimingDistribution):
     typename = "labeled_timing_distribution"
 
+
 class LabeledQuantity(Labeled, Quantity):
     typename = "labeled_quantity"
 
+
 class Rate(Metric):
     typename = "rate"
 
@@ -527,4 +532,55 @@ class Object(Metric):
         return structure
 
 
+class DualLabeledCounter(Metric):
+    typename = "dual_labeled_counter"
+    dual_labeled = True
+
+    def __init__(self, *args, **kwargs):
+        dual_labels = kwargs.pop("dual_labels", None)
+        if not dual_labels:
+            raise ValueError(
+                "`dual_labeled_counter` is missing required parameter `dual_labels`"
+            )
+        k = dual_labels.get("key", None)
+        if not k:
+            raise ValueError("`dual_labels` is missing required parameter `key`")
+        c = dual_labels.get("category", None)
+        if not c:
+            raise ValueError("`dual_labels` is missing required parameter `categories`")
+        keys = k.get("labels", None)
+        if keys is not None:
+            if not isinstance(keys, list) or not all(isinstance(k, str) for k in keys):
+                raise ValueError("key `labels` must be a list of strings")
+            self.ordered_keys = keys
+            self.keys = set([CowString(key) for key in keys])
+        else:
+            self.ordered_keys = None
+            self.keys = None
+        categories = c.get("labels", None)
+        if categories is not None:
+            if not isinstance(categories, list) or not all(
+                isinstance(c, str) for c in categories
+            ):
+                raise ValueError("category `labels` must be a list of strings")
+            self.ordered_categories = categories
+            self.categories = set([CowString(category) for category in categories])
+        else:
+            self.ordered_categories = None
+            self.categories = None
+        super().__init__(*args, **kwargs)
+
+    def serialize(self) -> Dict[str, util.JSONType]:
+        """
+        Serialize the metric back to JSON object model.
+        """
+        d = super().serialize()
+        d["keys"] = self.ordered_keys
+        d["categories"] = self.ordered_categories
+        del d["ordered_keys"]
+        del d["ordered_categories"]
+        del d["dual_labeled"]
+        return d
+
+
 ObjectTree = Dict[str, Dict[str, Union[Metric, pings.Ping, tags.Tag]]]

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/parser.py

@@ -182,25 +182,42 @@ def _instantiate_metrics(
     global_tags = content.get("$tags", [])
     assert isinstance(global_tags, list)
 
+    # Duplicate category names end up on the root content.
+    if not hasattr(all_objects, "duplicate"):
+        setattr(all_objects, "duplicate", getattr(content, "duplicate", None))
+
     for category_key, category_val in sorted(content.items()):
         if category_key.startswith("$"):
             continue
         if category_key == "no_lint":
             continue
         if not config.get("allow_reserved") and category_key.split(".")[0] == "glean":
-            yield util.format_error(
-                filepath,
-                f"For category '{category_key}'",
-                "Categories beginning with 'glean' are reserved for "
-                "Glean internal use.",
-            )
-            continue
+            if category_key not in ("glean.attribution", "glean.distribution"):
+                yield util.format_error(
+                    filepath,
+                    f"For category '{category_key}'",
+                    "Categories beginning with 'glean' are reserved for "
+                    "Glean internal use.",
+                )
+                continue
         all_objects.setdefault(category_key, DictWrapper())
 
         if not isinstance(category_val, dict):
             raise TypeError(f"Invalid content for {category_key}")
 
         for metric_key, metric_val in sorted(category_val.items()):
+            if (
+                not config.get("allow_reserved")
+                and category_key in ("glean.attribution", "glean.distribution")
+                and metric_key != "ext"
+            ):
+                yield util.format_error(
+                    filepath,
+                    f"For {category_key}.{metric_key}",
+                    f"May only use semi-reserved category {category_key} with metric name 'ext'",
+                    metric_val.defined_in["line"],
+                )
+                continue
             try:
                 metric_obj = Metric.make_metric(
                     category_key, metric_key, metric_val, validated=True, config=config
@@ -214,18 +231,28 @@ def _instantiate_metrics(
                 )
                 metric_obj = None
             else:
-                if (
-                    not config.get("allow_reserved")
-                    and "all-pings" in metric_obj.send_in_pings
-                ):
-                    yield util.format_error(
-                        filepath,
-                        f"On instance {category_key}.{metric_key}",
-                        'Only internal metrics may specify "all-pings" '
-                        'in "send_in_pings"',
-                        metric_val.defined_in["line"],
-                    )
-                    metric_obj = None
+                if not config.get("allow_reserved"):
+                    if "all-pings" in metric_obj.send_in_pings:
+                        yield util.format_error(
+                            filepath,
+                            f"On instance {category_key}.{metric_key}",
+                            'Only internal metrics may specify "all-pings" '
+                            'in "send_in_pings"',
+                            metric_val.defined_in["line"],
+                        )
+                        metric_obj = None
+                    elif (
+                        metric_obj.identifier()
+                        in ("glean.attribution.ext", "glean.distribution.ext")
+                        and metric_obj.type != "object"
+                    ):
+                        yield util.format_error(
+                            filepath,
+                            f"On instance {category_key}.{metric_key}",
+                            "Extended attribution/distribution metrics must be of type 'object'",
+                            metric_val.defined_in["line"],
+                        )
+                        metric_obj = None
 
             if metric_obj is not None:
                 metric_obj.no_lint = sorted(set(metric_obj.no_lint + global_no_lint))
@@ -250,6 +277,12 @@ def _instantiate_metrics(
                     metric_obj.defined_in["line"],
                 )
             else:
+                if not hasattr(all_objects[category_key], "duplicate"):
+                    setattr(
+                        all_objects[category_key],
+                        "duplicate",
+                        getattr(content[category_key], "duplicate", None),
+                    )
                 all_objects[category_key][metric_key] = metric_obj
                 sources[(category_key, metric_key)] = filepath
 
@@ -269,6 +302,15 @@ def _instantiate_pings(
     assert isinstance(global_no_lint, list)
     ping_schedule_reverse_map: Dict[str, Set[str]] = dict()
 
+    # Duplicate ping names end up on the root content.
+    duplicate_ping = getattr(content, "duplicate", None)
+    if duplicate_ping:
+        yield util.format_error(
+            filepath,
+            "",
+            f"Duplicate ping named '{duplicate_ping}'.",
+        )
+
     for ping_key, ping_val in sorted(content.items()):
         if ping_key.startswith("$"):
             continue
@@ -321,8 +363,7 @@ def _instantiate_pings(
             yield util.format_error(
                 filepath,
                 "",
-                f"Duplicate ping name '{ping_key}' "
-                f"already defined in '{already_seen}'",
+                f"Duplicate ping name '{ping_key}' already defined in '{already_seen}'",
             )
         else:
             all_objects.setdefault("pings", {})[ping_key] = ping_obj
@@ -380,8 +421,7 @@ def _instantiate_tags(
             yield util.format_error(
                 filepath,
                 "",
-                f"Duplicate tag name '{tag_key}' "
-                f"already defined in '{already_seen}'",
+                f"Duplicate tag name '{tag_key}' already defined in '{already_seen}'",
             )
         else:
             all_objects.setdefault("tags", {})[tag_key] = tag_obj
@@ -481,16 +521,16 @@ def parse_objects(
                 raise TypeError(f"Invalid content for {filepath}")
 
             for category_key, category_val in sorted(content.items()):
-              if category_key.startswith("$"):
-                  continue
+                if category_key.startswith("$"):
+                    continue
 
-              interesting_metrics_dict.setdefault(category_key, DictWrapper())
+                interesting_metrics_dict.setdefault(category_key, DictWrapper())
 
-              if not isinstance(category_val, dict):
-                  raise TypeError(f"Invalid category_val for {category_key}")
+                if not isinstance(category_val, dict):
+                    raise TypeError(f"Invalid category_val for {category_key}")
 
-              for metric_key, metric_val in sorted(category_val.items()):
-                  interesting_metrics_dict[category_key][metric_key] = metric_val
+                for metric_key, metric_val in sorted(category_val.items()):
+                    interesting_metrics_dict[category_key][metric_key] = metric_val
 
         for category_key, category_val in all_objects.items():
             if category_key == "tags":

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/rust.py

@@ -101,6 +101,8 @@ def type_name(obj):
 
     if getattr(obj, "labeled", False):
         return "LabeledMetric<{}>".format(class_name(obj.type))
+    if getattr(obj, "dual_labeled_counter", False):
+        return "DualLabeledCounterMetric"
     generate_enums = getattr(obj, "_generate_enums", [])  # Extra Keys? Reasons?
     if len(generate_enums):
         generic = None

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/schemas/metrics.2-0-0.schema.yaml

@@ -52,7 +52,16 @@ definitions:
   labeled_metric_id:
     type: string
     pattern: "^[ -~]+$"
-    maxLength: 71  # Note: this should be category + metric + 1
+    maxLength: 111  # Note: this should be category + metric + 1
+
+  optional_labels:
+    anyOf:
+      - type: array
+        uniqueItems: true
+        items:
+          $ref: "#/definitions/labeled_metric_id"
+        maxItems: 4096
+      - type: "null"
 
   metric:
     description: |
@@ -121,14 +130,20 @@ definitions:
               metric types include:
 
                 `labeled_boolean`, `labeled_string`, `labeled_counter`,
-                `labeled_custom_distribution`, `labeled_memory_distribution`,
-                `labeled_timing_distribution`, `labeled_quantity`.
+                `dual_labeled_counter`, `labeled_custom_distribution`,
+                `labeled_memory_distribution`, `labeled_timing_distribution`,
+                `labeled_quantity`.
 
             - `text`: Record long text data.
 
             - `object`: Record structured data based on a pre-defined schema
               Additional properties: `structure`.
 
+            - `dual_labeled_counter`: A counter with two label dimensions.
+              This metric type is used to record a counter with two different
+              label axes, such as a key and a category. Additional properties:
+              `dual_labels`.
+
         type: string
         enum:
           - event
@@ -155,6 +170,7 @@ definitions:
           - rate
           - text
           - object
+          - dual_labeled_counter
 
       description:
         title: Description
@@ -370,13 +386,51 @@ definitions:
           labels in the special label `__other__`.
 
           Valid with any of the labeled metric types.
-        anyOf:
-          - type: array
-            uniqueItems: true
-            items:
-              $ref: "#/definitions/labeled_metric_id"
-            maxItems: 4096
-          - type: "null"
+        $ref: "#/definitions/optional_labels"
+
+      dual_labels:
+        type: object
+        description: Defines the two label dimensions for a dual-labeled metric.
+        properties:
+          key:
+            type: object
+            description: The primary label dimension.
+            properties:
+              description:
+                type: string
+                description: |
+                  Human-readable description of the key dimension (first label
+                  axis).
+              labels:
+                description: |
+                  Optional list of statically defined label values for the key
+                  dimension.
+                $ref: "#/definitions/optional_labels"
+            required:
+              - description
+            additionalProperties: false
+          category:
+            type: object
+            description: The secondary label dimension.
+            properties:
+              description:
+                type: string
+                description: |
+                  Human-readable description of the category dimension (second
+                  label axis).
+              labels:
+                description: |
+                  Optional list of statically defined label values for the
+                  category dimension.
+                $ref: "#/definitions/optional_labels"
+            required:
+              - description
+            additionalProperties: false
+        required:
+          - key
+          - category
+        additionalProperties: false
+
 
       extra_keys:
         title: Extra keys
@@ -565,7 +619,7 @@ definitions:
           Use is limited to Firefox Desktop only.
           Has no effect when used with non-FOG outputters.
           See FOG's documentation on mirroring for details -
-          https://firefox-source-docs.mozilla.org/toolkit/components/glean/mirroring.html
+          https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/gifft.html
         type: string
         minLength: 6
 

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/templates/rust.jinja2

@@ -89,7 +89,7 @@ CommonMetricData {
     disabled: {{ obj.is_disabled()|rust }},
     ..Default::default()
 }
-{% endmacro %}
+{%- endmacro -%}
 {% for category in categories %}
 {% if category.contains_pings %}
 {% for obj in category.objs.values() %}
@@ -150,7 +150,7 @@ pub mod {{ category.name|snake_case }} {
         {%- for arg_name in extra_metric_args if not obj.labeled and obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%}
             , {{ obj[arg_name]|rust }}
         {%- endfor -%}
-        {{ ", " if obj.labeled else ")\n" }}
+        {{ ", " if obj.labeled or obj.dual_labeled else ")\n" }}
         {%- if obj.labeled -%}
         {%- if obj.labels -%}
         Some({{ obj.labels|rust }})
@@ -158,6 +158,19 @@ pub mod {{ category.name|snake_case }} {
         None
         {%- endif -%})
         {% endif %}
+        {%- if obj.dual_labeled -%}
+        {%- if obj.keys -%}
+        Some({{ obj.keys|rust }})
+        {%- else -%}
+        None
+        {%- endif -%}
+        {{ ", " }}
+        {%- if obj.categories -%}
+        Some({{ obj.categories|rust }})
+        {%- else -%}
+        None
+        {%- endif -%})
+        {% endif %}
     });
     {% endfor %}
 }

{glean_parser-17.0.1 → glean_parser-17.2.0}/glean_parser/util.py

@@ -88,7 +88,15 @@ def yaml_load(stream):
 
     def _construct_mapping_adding_line(loader, node):
         loader.flatten_mapping(node)
-        mapping = DictWrapper(loader.construct_pairs(node))
+        pairs = loader.construct_pairs(node)
+
+        # Redefinition of a key might be a mistake if that key is a metric name.
+        mapping = DictWrapper()
+        for pair in pairs:
+            if pair[0] in mapping:
+                mapping.duplicate = pair[0]
+            mapping[pair[0]] = pair[1]
+
         mapping.defined_in = {"line": node.start_mark.line}
         return mapping
 

glean_parser-17.2.0/pyproject.toml

@@ -0,0 +1,76 @@
+[project]
+name = "glean-parser"
+dynamic = ["version"]
+authors = [
+    { name = "The Glean Team", email = "glean-team@mozilla.com" }
+]
+description = "Parser tools for Mozilla's Glean telemetry"
+readme = "README.md"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+requires-python = ">=3.8"
+dependencies = [
+    "Click>=7",
+    "diskcache>=4",
+    "Jinja2>=2.10.1",
+    "jsonschema>=3.0.2",
+    "platformdirs>=2.4.0",
+    "PyYAML>=5.3.1",
+]
+
+[project.urls]
+Homepage = "https://mozilla.github.io/glean"
+Repository = "https://github.com/mozilla/glean_parser"
+Changelog = "https://github.com/mozilla/glean_parser/blob/main/CHANGELOG.md"
+
+[project.scripts]
+glean_parser = "glean_parser.__main__:main_wrapper"
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "coverage>=7.6.1",
+    "mypy>=1.14.1",
+    "pip-licenses>=4.5.1",
+    "pytest>=8.3.4",
+    "recommonmark>=0.7.1",
+    "ruff>=0.9.3",
+    "sphinx>=7.1.2",
+    "types-pyyaml>=6.0.12.20241230",
+    "yamllint>=1.35.1",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/glean_parser",
+  "/server_telemetry",
+  "/tests",
+]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+version_scheme = "no-guess-dev"
+
+[tool.pytest.ini_options]
+markers = [
+    "web_dependency: mark a test that requires a web connection.",
+    "node_dependency: mark a test that requires node.",
+    "ruby_dependency: mark a test that requires ruby.",
+    "go_dependency: mark a test that requires go.",
+    "rust_dependency: mark a test that requires rust.",
+]

{glean_parser-17.0.1 → glean_parser-17.2.0}/tests/data/all_metrics.yaml

@@ -27,6 +27,21 @@ all_metrics:
       - label_a
       - label_b
 
+  dual_labeled_counter:
+    <<: *defaults
+    type: dual_labeled_counter
+    dual_labels:
+      key:
+        description: The key for the dual labeled counter
+        labels:
+          - key1
+          - key2
+      category:
+        description: The category for the dual labeled counter
+        labels:
+          - category1
+          - category2
+
   bool:
     <<: *defaults
     type: boolean