glean-parser 19.2.0__tar.gz → 20.0.1__tar.gz

{glean_parser-19.2.0 → glean_parser-20.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glean-parser
-Version: 19.2.0
+Version: 20.0.1
 Summary: Parser tools for Mozilla's Glean telemetry
 Project-URL: Homepage, https://mozilla.github.io/glean
 Project-URL: Repository, https://github.com/mozilla/glean_parser

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/go_server.py

@@ -13,9 +13,11 @@ in server-side environments. In these environments SDK assumptions to measuremen
 window and connectivity don't hold.
 
 Generated code takes care of assembling pings with metrics, and serializing to messages
-conforming to Glean schema. Two transport modes are supported:
-- Cloud Logging (go_server): Logs to stdout in MozLog format for ingestion via GCP log routing
-- Pub/Sub (go_server_pubsub): Publishes directly to GCP Pub/Sub topics
+conforming to Glean schema. The transport is selected with the `transport` option
+on the `go_server` outputter (`-s transport=...`):
+- `logging` (default): Logs to stdout in MozLog format for ingestion via GCP log routing
+- `pubsub`: Builds messages for direct publishing to GCP Pub/Sub topics
+- `combined`: Emits both in a single file (sharing common types) for gradual migration
 
 Warning: this outputter supports limited set of metrics,
 see `SUPPORTED_METRIC_TYPES` below.
@@ -128,8 +130,9 @@ def output_go(
     :param objects: A tree of objects (metrics and pings) as returned from
         `parser.parse_objects`.
     :param output_dir: Path to an output directory to write to.
-    :param transport: Transport mode - either "logging" (Cloud Logging) or
-        "pubsub" (Pub/Sub direct publishing). Default is "logging".
+    :param transport: Transport mode - one of "logging" (Cloud Logging, the
+        default), "pubsub" (Pub/Sub direct publishing), or "combined" (both in
+        a single file, sharing common types).
     """
 
     template = util.get_jinja2_template(
@@ -211,29 +214,36 @@ def output_go(
         )
 
 
-def output_go_logger(
-    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
-) -> None:
-    """
-    Given a tree of objects, output Go code using Cloud Logging transport.
-
-    :param objects: A tree of objects (metrics and pings) as returned from
-        `parser.parse_objects`.
-    :param output_dir: Path to an output directory to write to.
-    :param options: options dictionary (currently unused for Go).
-    """
-    output_go(objs, output_dir, options, transport="logging")
+def _resolve_transport(default: str, options: Optional[Dict[str, Any]]) -> str:
+    if not options:
+        return default
+    transport = options.get("transport")
+    if transport is None:
+        return default
+    if transport not in ("logging", "pubsub", "combined"):
+        raise ValueError(
+            f"Invalid transport '{transport}'."
+            " Must be one of: logging, pubsub, combined."
+        )
+    return transport
 
 
-def output_go_pubsub(
+def output_go_logger(
     objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
 ) -> None:
     """
-    Given a tree of objects, output Go code using Pub/Sub transport.
+    Given a tree of objects, output server Go code.
+
+    Defaults to the Cloud Logging transport; pass `-s transport=pubsub` for
+    direct Pub/Sub publishing, or `-s transport=combined` to emit both in a
+    single file (sharing common types) for a gradual logging->pubsub migration.
 
     :param objects: A tree of objects (metrics and pings) as returned from
         `parser.parse_objects`.
     :param output_dir: Path to an output directory to write to.
-    :param options: options dictionary (currently unused for Go).
+    :param options: options dictionary. Supports `transport` key with values
+        `logging` (default), `pubsub`, or `combined`.
     """
-    output_go(objs, output_dir, options, transport="pubsub")
+    output_go(
+        objs, output_dir, options, transport=_resolve_transport("logging", options)
+    )

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/metrics.py

@@ -61,6 +61,7 @@ class Metric:
         data_sensitivity: Optional[List[str]] = None,
         defined_in: Optional[Dict] = None,
         telemetry_mirror: Optional[str] = None,
+        in_session: bool = False,
         _config: Optional[Dict[str, Any]] = None,
         _validated: bool = False,
     ):
@@ -98,6 +99,8 @@ class Metric:
         self.defined_in = defined_in
         if telemetry_mirror is not None:
             self.telemetry_mirror = telemetry_mirror
+        if not hasattr(self, "in_session"):
+            self.in_session = in_session
 
         # _validated indicates whether this metric has already been jsonschema
         # validated (but not any of the Python-level validation).
@@ -312,6 +315,7 @@ class Event(Metric):
 
     def __init__(self, *args, **kwargs):
         self.extra_keys = kwargs.pop("extra_keys", {})
+        self.in_session = kwargs.pop("in_session", True)
         self.validate_extra_keys(self.extra_keys, kwargs.get("_config", {}))
         super().__init__(*args, **kwargs)
         self._generate_enums = [("allowed_extra_keys_with_types", "Extra")]

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/schemas/metrics.2-0-0.schema.yaml

@@ -627,6 +627,19 @@ definitions:
           so glean_parser can find it.
         type: string
 
+      in_session:
+        title: In session
+        description: |
+          Whether this metric participates in session sampling.
+          When `true`, the metric is subject to session-level sampling and
+          session metadata is attached to it.
+          When `false`, the metric bypasses session sampling entirely.
+
+          Only valid when `type`_ is `event`.
+          Defaults to `true` when `type`_ is `event`, `false` otherwise.
+        type: boolean
+        default: false
+
       structure:
         title: A subset of a JSON schema definition
         description: |
@@ -773,6 +786,18 @@ additionalProperties:
               description: |
                 `denominator_metric` is only allowed for `rate`.
               maxLength: 0
+      - if:
+          not:
+            properties:
+              type:
+                const: event
+        then:
+          properties:
+            in_session:
+              description: |
+                `in_session: true` is only allowed for `event` metrics.
+                Only event metrics support session sampling.
+              const: false
       -
         if:
           properties:

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/templates/go_server.jinja2

@@ -1,4 +1,6 @@
 {# The final Go code is autogenerated, but this template is not. Please file bugs! #}
+{% set include_pubsub = transport in ["pubsub", "combined"] %}
+{% set include_logging = transport in ["logging", "combined"] %}
 package glean
 
 // This Source Code Form is subject to the terms of the Mozilla Public
@@ -9,30 +11,31 @@ package glean
 
 // required imports
 import (
-{% if transport == "pubsub" %}
+{% if include_pubsub %}
     "bytes"
     "compress/gzip"
+{% endif %}
     "encoding/json"
-    "fmt"
-    "io"
-    "sync"
-    "time"
-
-    pubsub "cloud.google.com/go/pubsub/v2"
-    "github.com/google/uuid"
-{% else %}
-    "encoding/json"
+{% if include_logging %}
     "errors"
+{% endif %}
     "fmt"
     "io"
+{% if include_logging %}
     "strconv"
+{% endif %}
+{% if include_pubsub %}
+    "sync"
+{% endif %}
     "time"
 
-    "github.com/google/uuid"
+{% if include_pubsub %}
+    pubsub "cloud.google.com/go/pubsub/v2"
 {% endif %}
+    "github.com/google/uuid"
 )
 
-{% if transport == "pubsub" %}
+{% if include_pubsub %}
 // GleanEventsBuilder constructs Pub/Sub messages carrying Glean pings.
 // Builder is stateless beyond its app-identity fields; callers own the
 // pubsub.Client, the pubsub.Publisher, batching settings, retries, shutdown sequencing, and any
@@ -74,7 +77,8 @@ func compressPayload(data []byte) ([]byte, error) {
     gzipPool.Put(gz)
     return buf.Bytes(), nil
 }
-{% else %}
+{% endif %}
+{% if include_logging %}
 // log type string used to identify logs to process in the Moz Data Pipeline
 var gleanEventMozlogType string = "glean-server-event"
 
@@ -120,7 +124,7 @@ type pingInfo struct {
     EndTime   string `json:"end_time"`
 }
 
-{% if transport == "logging" %}
+{% if include_logging %}
 type ping struct {
     DocumentNamespace string `json:"document_namespace"`
     DocumentType      string `json:"document_type"`
@@ -148,7 +152,7 @@ type gleanEvent struct {
     Extra     map[string]string `json:"extra"`
 }
 
-{% if transport == "logging" %}
+{% if include_logging %}
 type logEnvelope struct {
     Timestamp string
     Logger    string
@@ -157,12 +161,8 @@ type logEnvelope struct {
 }
 {% endif %}
 
-{% if transport == "pubsub" %}
-func (g GleanEventsBuilder) createClientInfo() clientInfo {
-{% else %}
-func (g GleanEventsLogger) createClientInfo() clientInfo {
-{% endif %}
-    // Fields with default values are required in the Glean schema, but not used in server context
+// Fields with default values are required in the Glean schema, but not used in server context
+func createClientInfo(appDisplayVersion, appChannel string) clientInfo {
     return clientInfo{
         TelemetrySDKBuild: "glean_parser v{{ parser_version }}",
         FirstRunDate:      "Unknown",
@@ -170,8 +170,8 @@ func (g GleanEventsLogger) createClientInfo() clientInfo {
         OSVersion:         "Unknown",
         Architecture:      "Unknown",
         AppBuild:          "Unknown",
-        AppDisplayVersion: g.AppDisplayVersion,
-        AppChannel:        g.AppChannel,
+        AppDisplayVersion: appDisplayVersion,
+        AppChannel:        appChannel,
     }
 }
 
@@ -185,14 +185,23 @@ func createPingInfo() pingInfo {
     }
 }
 
-{% if transport == "logging" %}
-func (g GleanEventsLogger) createPing(documentType string, config RequestInfo, payload pingPayload) (ping, error) {
-    payloadJson, err := json.Marshal(payload)
+// resolveDocumentID returns supplied if non-empty, otherwise a fresh UUID.
+// Set DocumentID on the ping params struct to share an ID across transports
+// (e.g., dual-write verification during a logging->pubsub migration).
+func resolveDocumentID(supplied string) (string, error) {
+    if supplied != "" {
+        return supplied, nil
+    }
+    id, err := uuid.NewRandom()
     if err != nil {
-        return ping{}, err
+        return "", err
     }
+    return id.String(), nil
+}
 
-    documentID, err := uuid.NewRandom()
+{% if include_logging %}
+func (g GleanEventsLogger) createPing(documentType, documentID string, config RequestInfo, payload pingPayload) (ping, error) {
+    payloadJson, err := json.Marshal(payload)
     if err != nil {
         return ping{}, err
     }
@@ -201,7 +210,7 @@ func (g GleanEventsLogger) createPing(documentType string, config RequestInfo, p
         DocumentNamespace: g.AppID,
         DocumentType:      documentType,
         DocumentVersion:   "1",
-        DocumentID:        documentID.String(),
+        DocumentID:        documentID,
         UserAgent:         config.UserAgent,
         IpAddress:         config.IpAddress,
         Payload:           string(payloadJson),
@@ -209,25 +218,18 @@ func (g GleanEventsLogger) createPing(documentType string, config RequestInfo, p
 }
 
 // method called by each ping-specific record method.
-// construct the ping, wrap it in the envelope, and print to stdout
+// wrap the payload in the envelope and print to stdout
 func (g GleanEventsLogger) record(
     documentType string,
+    documentID string,
     requestInfo RequestInfo,
-    metrics metrics,
-    events []gleanEvent,
+    payload pingPayload,
 ) error {
     if g.Writer == nil {
         return errors.New("writer not specified")
     }
 
-    telemetryPayload := pingPayload{
-        ClientInfo: g.createClientInfo(),
-        PingInfo:   createPingInfo(),
-        Metrics:    metrics,
-        Events:     events,
-    }
-
-    ping, err := g.createPing(documentType, requestInfo, telemetryPayload)
+    ping, err := g.createPing(documentType, documentID, requestInfo, payload)
     if err != nil {
         return err
     }
@@ -330,25 +332,17 @@ type {{ ping|ping_type_name }} struct {
     {% if metrics_by_type['event'] %}
     Event {{ ping|ping_events_type_name }} // valid event for this ping
     {% endif %}
+    DocumentID string // optional; used as document_id if non-empty, otherwise a fresh UUID is generated
 }
 
-{% if transport == "pubsub" %}
-// Build{{ ping|ping_type_name }}Message constructs a Pub/Sub message carrying
-// the given `{{ ping }}` ping. The caller publishes the returned message
-// (e.g., topic.Publish(ctx, msg)) and owns batching, retries, shutdown
-// sequencing, and any publish-result metrics.
-//
-// Wire-format contract: see
-// docs/architecture/decoder_service_specification.md in mozilla/gcp-ingestion.
-func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}Message(
-    requestInfo RequestInfo,
-    params {{ ping|ping_type_name }},
-) (*pubsub.Message, error) {
+// buildPayload assembles the inner pingPayload for `{{ ping }}`. Shared by
+// the logging and pub/sub code paths.
+func (p {{ ping|ping_type_name }}) buildPayload(appDisplayVersion, appChannel string) pingPayload {
     {% if metrics_by_type['string_list'] %}
     {% for metric in metrics_by_type['string_list'] %}
     // Ensure nil string_list metrics serialize as empty arrays, not null
-    if params.{{ metric|metric_argument_name }} == nil {
-        params.{{ metric|metric_argument_name }} = []string{}
+    if p.{{ metric|metric_argument_name }} == nil {
+        p.{{ metric|metric_argument_name }} = []string{}
     }
     {% endfor %}
     {% endif %}
@@ -358,9 +352,9 @@ func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}Message(
         "{{ metric_type }}": {
         {% for metric in metrics %}
             {% if metric_type == 'datetime' %}
-            "{{ metric|metric_name }}": params.{{ metric|metric_argument_name }}.Format("2006-01-02T15:04:05.000Z"),
+            "{{ metric|metric_name }}": p.{{ metric|metric_argument_name }}.Format("2006-01-02T15:04:05.000Z"),
             {% else %}
-            "{{ metric|metric_name }}": params.{{ metric|metric_argument_name }},
+            "{{ metric|metric_name }}": p.{{ metric|metric_argument_name }},
             {% endif %}
         {% endfor %}
         },
@@ -370,17 +364,32 @@ func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}Message(
 
     events := []gleanEvent{}
     {% if metrics_by_type['event'] %}
-    if params.Event != nil {
-        events = append(events, params.Event.gleanEvent())
+    if p.Event != nil {
+        events = append(events, p.Event.gleanEvent())
     }
     {% endif %}
 
-    payload := pingPayload{
-        ClientInfo: g.createClientInfo(),
+    return pingPayload{
+        ClientInfo: createClientInfo(appDisplayVersion, appChannel),
         PingInfo:   createPingInfo(),
         Metrics:    metrics,
         Events:     events,
     }
+}
+
+{% if include_pubsub %}
+// Build{{ ping|ping_type_name }}Message constructs a Pub/Sub message carrying
+// the given `{{ ping }}` ping. The caller publishes the returned message
+// (e.g., topic.Publish(ctx, msg)) and owns batching, retries, shutdown
+// sequencing, and any publish-result metrics.
+//
+// Wire-format contract: see
+// docs/architecture/decoder_service_specification.md in mozilla/gcp-ingestion.
+func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}Message(
+    requestInfo RequestInfo,
+    params {{ ping|ping_type_name }},
+) (*pubsub.Message, error) {
+    payload := params.buildPayload(g.AppDisplayVersion, g.AppChannel)
 
     payloadJSON, err := json.Marshal(payload)
     if err != nil {
@@ -391,16 +400,16 @@ func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}Message(
         return nil, fmt.Errorf("compress payload: %w", err)
     }
 
-    documentID, err := uuid.NewRandom()
+    documentID, err := resolveDocumentID(params.DocumentID)
     if err != nil {
-        return nil, fmt.Errorf("generate document_id: %w", err)
+        return nil, fmt.Errorf("resolve document_id: %w", err)
     }
 
     attributes := map[string]string{
         "document_namespace": g.AppID,
         "document_type":      "{{ ping }}",
         "document_version":   "1",
-        "document_id":        documentID.String(),
+        "document_id":        documentID,
     }
     // Skip empty optional attributes; the decoder treats missing and empty
     // the same, and Pub/Sub charges for attribute bytes. The publisher does
@@ -426,43 +435,19 @@ func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}MessageWithoutUserInfo
 ) (*pubsub.Message, error) {
     return g.Build{{ ping|ping_type_name }}Message(defaultRequestInfo, params)
 }
-{% else %}
+{% endif %}
+{% if include_logging %}
 // Record and submit `{{ ping }}` ping
 func (g GleanEventsLogger) Record{{ ping|ping_type_name }}(
     requestInfo RequestInfo,
     params {{ ping|ping_type_name }},
 ) error {
-    {% if metrics_by_type['string_list'] %}
-    {% for metric in metrics_by_type['string_list'] %}
-    // Ensure nil string_list metrics serialize as empty arrays, not null
-    if params.{{ metric|metric_argument_name }} == nil {
-        params.{{ metric|metric_argument_name }} = []string{}
-    }
-    {% endfor %}
-    {% endif %}
-    metrics := metrics{
-    {% for metric_type, metrics in metrics_by_type.items() %}
-    {% if metric_type != 'event' %}
-        "{{ metric_type }}": {
-        {% for metric in metrics %}
-            {% if metric_type == 'datetime' %}
-            "{{ metric|metric_name }}": params.{{ metric|metric_argument_name }}.Format("2006-01-02T15:04:05.000Z"),
-            {% else %}
-            "{{ metric|metric_name }}": params.{{ metric|metric_argument_name }},
-            {% endif %}
-        {% endfor %}
-        },
-    {% endif %}
-    {% endfor %}
-    }
-
-    events := []gleanEvent{}
-    {% if metrics_by_type['event'] %}
-    if params.Event != nil {
-        events = append(events, params.Event.gleanEvent())
+    documentID, err := resolveDocumentID(params.DocumentID)
+    if err != nil {
+        return fmt.Errorf("resolve document_id: %w", err)
     }
-    {% endif %}
-    return g.record("{{ ping }}", requestInfo, metrics, events)
+    payload := params.buildPayload(g.AppDisplayVersion, g.AppChannel)
+    return g.record("{{ ping }}", documentID, requestInfo, payload)
 }
 
 // Record and submit `{{ ping }}` ping omitting user request info

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/templates/rust.jinja2

@@ -102,6 +102,7 @@ CommonMetricData {
     send_in_pings: {{ obj.send_in_pings|rust }},
     lifetime: {{ obj.lifetime|rust }},
     disabled: {{ obj.is_disabled()|rust }},
+    in_session: {{ obj.in_session|rust }},
     ..Default::default()
 }
 {%- endmacro -%}

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/translate.py

@@ -58,7 +58,6 @@ class Outputter:
 
 OUTPUTTERS = {
     "go_server": Outputter(go_server.output_go_logger, []),
-    "go_server_pubsub": Outputter(go_server.output_go_pubsub, []),
     "javascript": Outputter(javascript.output_javascript, []),
     "typescript": Outputter(javascript.output_typescript, []),
     "javascript_server": Outputter(javascript_server.output_javascript, []),

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/translation_options.py

@@ -45,6 +45,11 @@ JavaScript:
 Markdown:
 - `project_title`: The project's title.
 
+Go server (`go_server`):
+- `transport`: Transport mode. One of `logging` (Cloud Logging, the default),
+        `pubsub` (Pub/Sub direct publishing), or `combined` (emit both in a
+        single file, sharing common types, for a gradual migration).
+
 (press q to exit)"""
 
     if value:

{glean_parser-19.2.0 → glean_parser-20.0.1}/glean_parser/util.py

@@ -515,6 +515,7 @@ common_metric_args = [
     "send_in_pings",
     "lifetime",
     "disabled",
+    "in_session",
 ]
 
 

{glean_parser-19.2.0 → glean_parser-20.0.1}/tests/data/all_metrics.yaml

@@ -107,6 +107,7 @@ all_metrics:
   event:
     <<: *defaults
     type: event
+    in_session: true
     extra_keys:
       source:
         description: Source of this event