glean-parser 10.0.2__tar.gz → 11.0.0__tar.gz

{glean_parser-10.0.2 → glean_parser-11.0.0}/.circleci/config.yml

@@ -37,15 +37,18 @@ commands:
         default: "requirements_dev.txt"
     steps:
       - run:
-          name: install
+          name: Install required native dependencies
           command: |
             pip install --progress-bar off --user -U -r <<parameters.requirements-file>>
             sudo apt update -q
             sudo apt upgrade -q
-            sudo apt install openjdk-11-jdk-headless
+            sudo apt install \
+                --yes --no-install-recommends \
+                openjdk-11-jdk-headless \
+                ruby
             make install-kotlin-linters
       - run:
-          name: install
+          name: Install glean_parser
           # Set CC to something that isn't a working compiler so we
           # can detect if any of the dependencies require a compiler
           # to be installed. We can't count on a working compiler
@@ -55,7 +58,7 @@ commands:
           command: CC=broken_compiler pip install . --user
       - run:
           name: test
-          command: make test
+          command: make test-full
 
   lint:
     parameters:
@@ -92,23 +95,16 @@ commands:
             pip-licenses --fail-on 'GNU General Public License v3 (GPLv3)'
 
 jobs:
-  build-37:
-    docker:
-      - image: cimg/python:3.7
-    steps:
-      - test-start
-      - test-python-version
-
   build-38:
     docker:
-      - image: cimg/python:3.8
+      - image: cimg/python:3.8-node
     steps:
       - test-start
       - test-python-version
 
   build-38-min:
     docker:
-      - image: cimg/python:3.8
+      - image: cimg/python:3.8-node
     steps:
       - test-start
       - test-min-requirements
@@ -116,14 +112,14 @@ jobs:
 
   build-39:
     docker:
-      - image: cimg/python:3.9
+      - image: cimg/python:3.9-node
     steps:
       - test-start
       - test-python-version
 
   build-310:
     docker:
-      - image: cimg/python:3.10
+      - image: cimg/python:3.10-node
     steps:
       - test-start
       - test-python-version
@@ -136,16 +132,30 @@ jobs:
           root: docs/_build
           paths: html
 
+  build-311:
+    docker:
+      - image: cimg/python:3.11-node
+    steps:
+      - test-start
+      - test-python-version
+
+  build-312:
+    docker:
+      - image: cimg/python:3.12-node
+    steps:
+      - test-start
+      - test-python-version
+
   lint:
     docker:
-      - image: cimg/python:3.9
+      - image: cimg/python:3.12
     steps:
       - test-start
       - lint
 
   license-check:
     docker:
-      - image: cimg/python:3.9
+      - image: cimg/python:3.12
     steps:
       - test-start
       - license-check
@@ -161,7 +171,7 @@ jobs:
       - attach_workspace:
           at: docs/_build
       - run:
-          name: install
+          name: install gh-pages
           command: |
             npm install -g --silent gh-pages@2.0.1
             git config user.email "glean-ci@nowhere.com"
@@ -173,7 +183,7 @@ jobs:
 
   pypi-deploy:
     docker:
-      - image: cimg/python:3.7
+      - image: cimg/python:3.8
     steps:
       - checkout
       - run:
@@ -185,7 +195,7 @@ jobs:
           command: |
             pip install --upgrade --user pip
       - run:
-          name: install
+          name: install Python dependencies
           command: |
             pip install --user -U -r requirements_dev.txt
       - run:
@@ -207,10 +217,6 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - build-37:
-          filters:
-            tags:
-              only: /.*/
       - build-38:
           filters:
             tags:
@@ -227,6 +233,14 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - build-311:
+          filters:
+            tags:
+              only: /.*/
+      - build-312:
+          filters:
+            tags:
+              only: /.*/
       - docs-deploy:
           requires:
             - build-310
@@ -235,7 +249,7 @@ workflows:
               only: main
       - pypi-deploy:
           requires:
-            - build-37
+            - build-38
           filters:
             branches:
               ignore: /.*/

{glean_parser-10.0.2 → glean_parser-11.0.0}/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## Unreleased
 
+## 11.0.0
+
+- Add updated logging logic for Ruby Server ([#642](https://github.com/mozilla/glean_parser/pull/642))
+- Add support for event metric type in server-side JavaScript outputter ([DENG-1736](https://mozilla-hub.atlassian.net/browse/DENG-1736))
+- BREAKING CHANGE: Dropped support for Python 3.7 ([#638](https://github.com/mozilla/glean_parser/pull/638))
+- Add official support for Python 3.11+ ([#638](https://github.com/mozilla/glean_parser/pull/638))
+
+## 10.0.3
+
+- Warn about empty or TODO-tagged data reviews in the list ([#634](https://github.com/mozilla/glean_parser/pull/634))
+- Allow `unit` field on all metrics, but warn for all but quantity and custom distribution ([#636](https://github.com/mozilla/glean_parser/pull/636))
+
 ## 10.0.2
 
 - Allow `unit` field for string again, but warn about it in the linter ([#634](https://github.com/mozilla/glean_parser/pull/634))

{glean_parser-10.0.2 → glean_parser-11.0.0}/CONTRIBUTING.md

@@ -116,7 +116,7 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put your
    new functionality into a function with a docstring, and describe
    public-facing features in the docs.
-3. The pull request should work for Python 3.7+ (The CI system
+3. The pull request should work for Python 3.8+ (The CI system
    will take care of testing all of these Python versions).
 4. The pull request should update the changelog in `CHANGELOG.md`.
 

{glean_parser-10.0.2 → glean_parser-11.0.0}/Makefile

@@ -44,7 +44,10 @@ fmt: ## autoformat files
 	python3 -m black glean_parser tests setup.py
 
 test: ## run tests quickly with the default Python
-	py.test --run-web-tests
+	py.test
+
+test-full:  ## run tests, including those with additional dependencies
+	py.test --run-web-tests --run-node-tests --run-ruby-tests
 
 coverage: ## check code coverage quickly with the default Python
 	coverage run --source glean_parser -m pytest

{glean_parser-10.0.2 → glean_parser-11.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: glean_parser
-Version: 10.0.2
+Version: 11.0.0
 Summary: Parser tools for Mozilla's Glean telemetry
 Home-page: https://github.com/mozilla/glean_parser
 Author: The Glean Team
@@ -10,11 +10,11 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.md
@@ -37,7 +37,7 @@ code for various integrations, linting and coverage testing.
 
 ## Requirements
 
--   Python 3.7 (or later)
+-   Python 3.8 (or later)
 
 The following library requirements are installed automatically when
 `glean_parser` is installed by `pip`.
@@ -73,6 +73,18 @@ $ glean_parser check < ping.json
 
 ## Unreleased
 
+## 11.0.0
+
+- Add updated logging logic for Ruby Server ([#642](https://github.com/mozilla/glean_parser/pull/642))
+- Add support for event metric type in server-side JavaScript outputter ([DENG-1736](https://mozilla-hub.atlassian.net/browse/DENG-1736))
+- BREAKING CHANGE: Dropped support for Python 3.7 ([#638](https://github.com/mozilla/glean_parser/pull/638))
+- Add official support for Python 3.11+ ([#638](https://github.com/mozilla/glean_parser/pull/638))
+
+## 10.0.3
+
+- Warn about empty or TODO-tagged data reviews in the list ([#634](https://github.com/mozilla/glean_parser/pull/634))
+- Allow `unit` field on all metrics, but warn for all but quantity and custom distribution ([#636](https://github.com/mozilla/glean_parser/pull/636))
+
 ## 10.0.2
 
 - Allow `unit` field for string again, but warn about it in the linter ([#634](https://github.com/mozilla/glean_parser/pull/634))

{glean_parser-10.0.2 → glean_parser-11.0.0}/README.md

@@ -16,7 +16,7 @@ code for various integrations, linting and coverage testing.
 
 ## Requirements
 
--   Python 3.7 (or later)
+-   Python 3.8 (or later)
 
 The following library requirements are installed automatically when
 `glean_parser` is installed by `pip`.

{glean_parser-10.0.2 → glean_parser-11.0.0}/docs/contributing.md

@@ -116,7 +116,7 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put your
    new functionality into a function with a docstring, and describe
    public-facing features in the docs.
-3. The pull request should work for Python 3.7+ (The CI system
+3. The pull request should work for Python 3.8+ (The CI system
    will take care of testing all of these Python versions).
 4. The pull request should update the changelog in `CHANGELOG.md`.
 

{glean_parser-10.0.2 → glean_parser-11.0.0}/docs/history.md

@@ -2,6 +2,18 @@
 
 ## Unreleased
 
+## 11.0.0
+
+- Add updated logging logic for Ruby Server ([#642](https://github.com/mozilla/glean_parser/pull/642))
+- Add support for event metric type in server-side JavaScript outputter ([DENG-1736](https://mozilla-hub.atlassian.net/browse/DENG-1736))
+- BREAKING CHANGE: Dropped support for Python 3.7 ([#638](https://github.com/mozilla/glean_parser/pull/638))
+- Add official support for Python 3.11+ ([#638](https://github.com/mozilla/glean_parser/pull/638))
+
+## 10.0.3
+
+- Warn about empty or TODO-tagged data reviews in the list ([#634](https://github.com/mozilla/glean_parser/pull/634))
+- Allow `unit` field on all metrics, but warn for all but quantity and custom distribution ([#636](https://github.com/mozilla/glean_parser/pull/636))
+
 ## 10.0.2
 
 - Allow `unit` field for string again, but warn about it in the linter ([#634](https://github.com/mozilla/glean_parser/pull/634))

{glean_parser-10.0.2 → glean_parser-11.0.0}/docs/readme.md

@@ -16,7 +16,7 @@ code for various integrations, linting and coverage testing.
 
 ## Requirements
 
--   Python 3.7 (or later)
+-   Python 3.8 (or later)
 
 The following library requirements are installed automatically when
 `glean_parser` is installed by `pip`.

{glean_parser-10.0.2 → glean_parser-11.0.0}/glean_parser/javascript_server.py

@@ -9,14 +9,24 @@ Outputter to generate server Javascript code for collecting events.
 
 This outputter is different from the rest of the outputters in that the code it
 generates does not use the Glean SDK. It is meant to be used to collect events
-using "events as pings" pattern in server-side environments. In these environments
-SDK assumptions to measurement window and connectivity don't hold.
+in server-side environments. In these environments SDK assumptions to measurement
+window and connectivity don't hold.
 Generated code takes care of assembling pings with metrics, serializing to messages
 conforming to Glean schema, and logging with mozlog. Then it's the role of the ingestion
 pipeline to pick the messages up and process.
 
 Warning: this outputter supports limited set of metrics,
 see `SUPPORTED_METRIC_TYPES` below.
+
+There are two patterns for event structure supported in this environment:
+* Events as `Event` metric type, where we generate a single class per ping with
+  `record{event_name}` method for each event metric. This is recommended to use for new
+  applications as it allows to fully leverage standard Data Platform tools
+  post-ingestion.
+* Custom pings-as-events, where for each ping we generate a class with a single `record`
+  method, usually with an `event_name` string metric.
+
+Therefore, unlike in other outputters, here we don't generate classes for each metric.
 """
 from collections import defaultdict
 from pathlib import Path
@@ -28,11 +38,14 @@ from . import util
 
 # Adding a metric here will require updating the `generate_js_metric_type` function
 # and might require changes to the template.
-SUPPORTED_METRIC_TYPES = ["string"]
+SUPPORTED_METRIC_TYPES = ["string", "event"]
 
 
-def event_class_name(pingName: str) -> str:
-    return util.Camelize(pingName) + "ServerEvent"
+def event_class_name(ping_name: str, event_metric_exists: bool) -> str:
+    # For compatibility with FxA codebase we don't want to add "Logger" suffix
+    # when custom pings without event metrics are used.
+    suffix = "Logger" if event_metric_exists else ""
+    return util.Camelize(ping_name) + "ServerEvent" + suffix
 
 
 def generate_metric_name(metric: metrics.Metric) -> str:
@@ -47,12 +60,20 @@ def generate_js_metric_type(metric: metrics.Metric) -> str:
     return metric.type
 
 
-def generate_metric_argument_description(metric: metrics.Metric) -> str:
-    return metric.description.replace("\n", " ").rstrip()
+def generate_ping_factory_method(ping: str, event_metric_exists: bool) -> str:
+    # `ServerEventLogger` better describes role of the class that this factory
+    # method generates, but for compatibility with existing FxA codebase
+    # we use `Event` suffix if no event metrics are defined.
+    suffix = "ServerEventLogger" if event_metric_exists else "Event"
+    return f"create{util.Camelize(ping)}{suffix}"
+
 
+def generate_event_metric_record_function_name(metric: metrics.Metric) -> str:
+    return f"record{util.Camelize(metric.category)}{util.Camelize(metric.name)}"
 
-def generate_ping_factory_method(ping: str) -> str:
-    return f"create{util.Camelize(ping)}Event"
+
+def clean_string(s: str) -> str:
+    return s.replace("\n", " ").rstrip()
 
 
 def output(
@@ -79,24 +100,16 @@ def output(
             ("metric_name", generate_metric_name),
             ("metric_argument_name", generate_metric_argument_name),
             ("js_metric_type", generate_js_metric_type),
-            ("metric_argument_description", generate_metric_argument_description),
             ("factory_method", generate_ping_factory_method),
+            (
+                "event_metric_record_function_name",
+                generate_event_metric_record_function_name,
+            ),
+            ("clean_string", clean_string),
         ),
     )
 
-    # In this environment we don't use a concept of measurement window for collecting
-    # metrics. Only "events as pings" are supported.
-    # For each ping we generate code which contains all the logic for assembling it
-    # with metrics, serializing, and submitting. Therefore we don't generate classes for
-    # each metric as in standard outputters.
-    PING_METRIC_ERROR_MSG = (
-        " Server-side environment is simplified and this"
-        + " parser doesn't generate individual metric files. Make sure to pass all"
-        + " your ping and metric definitions in a single invocation of the parser."
-    )
-    if "pings" not in objs:
-        print("❌ No ping definition found." + PING_METRIC_ERROR_MSG)
-        return
+    event_metric_exists = False
 
     # Go through all metrics in objs and build a map of
     # ping->list of metric categories->list of metrics
@@ -113,11 +126,39 @@ def output(
                         + " metric type."
                     )
                     continue
+                if metric.type == "event":
+                    # This is used in the template - generated code is slightly
+                    # different when event metric type is used.
+                    event_metric_exists = True
                 for ping in metric.send_in_pings:
                     metrics_by_type = ping_to_metrics[ping]
                     metrics_list = metrics_by_type.setdefault(metric.type, [])
                     metrics_list.append(metric)
 
+    PING_METRIC_ERROR_MSG = (
+        " Server-side environment is simplified and this"
+        + " parser doesn't generate individual metric files. Make sure to pass all"
+        + " your ping and metric definitions in a single invocation of the parser."
+    )
+    if "pings" not in objs:
+        # If events are meant to be sent in custom pings, we need to make sure they
+        # are defined. Otherwise we won't have destination tables defined and
+        # submissions won't pass validation at ingestion.
+        if event_metric_exists:
+            if "events" not in ping_to_metrics:
+                # Event metrics can be sent in standard `events` ping
+                # or in custom pings.
+                print(
+                    "❌ "
+                    + PING_METRIC_ERROR_MSG
+                    + "\n You need to either send your event metrics in standard"
+                    + " `events` ping or define a custom one."
+                )
+                return
+        else:
+            print("❌ No ping definition found." + PING_METRIC_ERROR_MSG)
+            return
+
     if not ping_to_metrics:
         print("❌ No pings with metrics found." + PING_METRIC_ERROR_MSG)
         return
@@ -129,6 +170,7 @@ def output(
             template.render(
                 parser_version=__version__,
                 pings=ping_to_metrics,
+                event_metric_exists=event_metric_exists,
                 lang=lang,
             )
         )

{glean_parser-10.0.2 → glean_parser-11.0.0}/glean_parser/lint.py

@@ -303,13 +303,30 @@ def check_metric_on_events_lifetime(
         )
 
 
-def check_unit_on_string(
+def check_unexpected_unit(
     metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
-    """`unit` was allowed on all metrics and recently disallowed.
-    To unbreak things we had to add it back to `string` metrics, but now warn"""
-    if isinstance(metric, metrics.String) and metric.unit:
-        yield "The `unit` property is not allowed for string metrics."
+    """
+    `unit` was allowed on all metrics and recently disallowed.
+    We now warn about its use on all but quantity and custom distribution
+    metrics.
+    """
+    allowed_types = [metrics.Quantity, metrics.CustomDistribution]
+    if not any([isinstance(metric, ty) for ty in allowed_types]) and metric.unit:
+        yield (
+            "The `unit` property is only allowed for quantity "
+            + "and custom distribution metrics."
+        )
+
+
+def check_empty_datareview(
+    metric: metrics.Metric, parser_config: Dict[str, Any]
+) -> LintGenerator:
+    disallowed_datareview = ["", "todo"]
+    data_reviews = [dr.lower() in disallowed_datareview for dr in metric.data_reviews]
+
+    if any(data_reviews):
+        yield "List of data reviews should not contain empty strings or TODO markers."
 
 
 def check_redundant_ping(
@@ -393,7 +410,8 @@ METRIC_CHECKS: Dict[
     "EXPIRED": (check_expired_metric, CheckType.warning),
     "OLD_EVENT_API": (check_old_event_api, CheckType.warning),
     "METRIC_ON_EVENTS_LIFETIME": (check_metric_on_events_lifetime, CheckType.error),
-    "UNIT_ON_STRING": (check_unit_on_string, CheckType.warning),
+    "UNEXPECTED_UNIT": (check_unexpected_unit, CheckType.warning),
+    "EMPTY_DATAREVIEW": (check_empty_datareview, CheckType.warning),
 }
 
 

{glean_parser-10.0.2 → glean_parser-11.0.0}/glean_parser/metrics.py

@@ -55,6 +55,7 @@ class Metric:
         disabled: bool = False,
         lifetime: str = "ping",
         send_in_pings: Optional[List[str]] = None,
+        unit: Optional[str] = None,
         gecko_datapoint: str = "",
         no_lint: Optional[List[str]] = None,
         data_sensitivity: Optional[List[str]] = None,
@@ -85,6 +86,7 @@ class Metric:
         if send_in_pings is None:
             send_in_pings = ["default"]
         self.send_in_pings = send_in_pings
+        self.unit = unit
         self.gecko_datapoint = gecko_datapoint
         if no_lint is None:
             no_lint = []
@@ -175,6 +177,8 @@ class Metric:
                 d[key] = [x.name for x in val]
         del d["name"]
         del d["category"]
+        if not d["unit"]:
+            d.pop("unit")
         d.pop("_config", None)
         d.pop("_generate_enums", None)
         return d
@@ -222,11 +226,6 @@ class Boolean(Metric):
 class String(Metric):
     typename = "string"
 
-    def __init__(self, *args, **kwargs):
-        # Deprecated. Kept here to unbreak stuff.
-        self.unit = kwargs.pop("unit", None)
-        Metric.__init__(self, *args, **kwargs)
-
 
 class StringList(Metric):
     typename = "string_list"
@@ -239,10 +238,6 @@ class Counter(Metric):
 class Quantity(Metric):
     typename = "quantity"
 
-    def __init__(self, *args, **kwargs):
-        self.unit = kwargs.pop("unit")
-        Metric.__init__(self, *args, **kwargs)
-
 
 class TimeUnit(enum.Enum):
     nanosecond = 0
@@ -302,7 +297,6 @@ class CustomDistribution(Metric):
         self.histogram_type = getattr(
             HistogramType, kwargs.pop("histogram_type", "exponential")
         )
-        self.unit = kwargs.pop("unit", None)
         super().__init__(*args, **kwargs)