glean-parser 19.1.0__tar.gz → 19.2.0__tar.gz

{glean_parser-19.1.0 → glean_parser-19.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glean-parser
-Version: 19.1.0
+Version: 19.2.0
 Summary: Parser tools for Mozilla's Glean telemetry
 Project-URL: Homepage, https://mozilla.github.io/glean
 Project-URL: Repository, https://github.com/mozilla/glean_parser

{glean_parser-19.1.0 → glean_parser-19.2.0}/glean_parser/go_server.py

@@ -11,14 +11,17 @@ This outputter is different from the rest of the outputters in that the code it
 generates does not use the Glean SDK. It is meant to be used to collect events
 in server-side environments. In these environments SDK assumptions to measurement
 window and connectivity don't hold.
+
 Generated code takes care of assembling pings with metrics, and serializing to messages
-conforming to Glean schema.
+conforming to Glean schema. Two transport modes are supported:
+- Cloud Logging (go_server): Logs to stdout in MozLog format for ingestion via GCP log routing
+- Pub/Sub (go_server_pubsub): Publishes directly to GCP Pub/Sub topics
 
 Warning: this outputter supports limited set of metrics,
 see `SUPPORTED_METRIC_TYPES` below.
 
 Generated code creates two methods for each ping (`RecordPingX` and `RecordPingXWithoutUserInfo`)
-that are used for submitting (logging) them.
+that are used for submitting events.
 If pings have `event` metrics assigned, they can be passed to these methods.
 """
 
@@ -111,7 +114,10 @@ def validate_labeled_boolean(metric: metrics.Metric) -> bool:
 
 
 def output_go(
-    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]]
+    objs: metrics.ObjectTree,
+    output_dir: Path,
+    options: Optional[Dict[str, Any]],
+    transport: str = "logging",
 ) -> None:
     """
     Given a tree of objects, output Go code to `output_dir`.
@@ -122,6 +128,8 @@ def output_go(
     :param objects: A tree of objects (metrics and pings) as returned from
         `parser.parse_objects`.
     :param output_dir: Path to an output directory to write to.
+    :param transport: Transport mode - either "logging" (Cloud Logging) or
+        "pubsub" (Pub/Sub direct publishing). Default is "logging".
     """
 
     template = util.get_jinja2_template(
@@ -198,5 +206,34 @@ def output_go(
                 pings=ping_to_metrics,
                 events=event_metrics,
                 labeled_booleans=labeled_boolean_metrics,
+                transport=transport,
             )
         )
+
+
+def output_go_logger(
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
+) -> None:
+    """
+    Given a tree of objects, output Go code using Cloud Logging transport.
+
+    :param objects: A tree of objects (metrics and pings) as returned from
+        `parser.parse_objects`.
+    :param output_dir: Path to an output directory to write to.
+    :param options: options dictionary (currently unused for Go).
+    """
+    output_go(objs, output_dir, options, transport="logging")
+
+
+def output_go_pubsub(
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
+) -> None:
+    """
+    Given a tree of objects, output Go code using Pub/Sub transport.
+
+    :param objects: A tree of objects (metrics and pings) as returned from
+        `parser.parse_objects`.
+    :param output_dir: Path to an output directory to write to.
+    :param options: options dictionary (currently unused for Go).
+    """
+    output_go(objs, output_dir, options, transport="pubsub")

{glean_parser-19.1.0 → glean_parser-19.2.0}/glean_parser/templates/go_server.jinja2

@@ -9,6 +9,18 @@ package glean
 
 // required imports
 import (
+{% if transport == "pubsub" %}
+    "bytes"
+    "compress/gzip"
+    "encoding/json"
+    "fmt"
+    "io"
+    "sync"
+    "time"
+
+    pubsub "cloud.google.com/go/pubsub/v2"
+    "github.com/google/uuid"
+{% else %}
     "encoding/json"
     "errors"
     "fmt"
@@ -17,8 +29,52 @@ import (
     "time"
 
     "github.com/google/uuid"
+{% endif %}
 )
 
+{% if transport == "pubsub" %}
+// GleanEventsBuilder constructs Pub/Sub messages carrying Glean pings.
+// Builder is stateless beyond its app-identity fields; callers own the
+// pubsub.Client, the pubsub.Publisher, batching settings, retries, shutdown sequencing, and any
+// publish-result metrics
+type GleanEventsBuilder struct {
+    AppID             string // Application ID to identify application per Glean standards
+    AppDisplayVersion string // Version of application emitting the event
+    AppChannel        string // Channel to differentiate logs from prod/beta/staging/devel
+}
+
+// gzipPool reuses gzip.Writer instances across calls. gzip.NewWriter()
+// allocates ~800 KB internally; pooling + Reset() drops per-message
+// allocations to near zero at high publish volumes.
+var gzipPool = sync.Pool{
+    New: func() interface{} {
+        return gzip.NewWriter(io.Discard)
+    },
+}
+
+// compressPayload gzips data using the package-level writer pool. The
+// returned slice is caller-owned; only the *gzip.Writer is returned to the
+// pool.
+func compressPayload(data []byte) ([]byte, error) {
+    var buf bytes.Buffer
+    // Optimistic preallocation: gzip on small JSON typically achieves ~2x ratio.
+    buf.Grow(len(data) / 2)
+
+    gz := gzipPool.Get().(*gzip.Writer)
+    gz.Reset(&buf)
+
+    if _, err := gz.Write(data); err != nil {
+        gzipPool.Put(gz)
+        return nil, fmt.Errorf("gzip write failed: %w", err)
+    }
+    if err := gz.Close(); err != nil {
+        gzipPool.Put(gz)
+        return nil, fmt.Errorf("gzip close failed: %w", err)
+    }
+    gzipPool.Put(gz)
+    return buf.Bytes(), nil
+}
+{% else %}
 // log type string used to identify logs to process in the Moz Data Pipeline
 var gleanEventMozlogType string = "glean-server-event"
 
@@ -32,6 +88,7 @@ type GleanEventsLogger struct {
     AppChannel        string // Channel to differentiate logs from prod/beta/staging/devel
     Writer            io.Writer // Writer to output to. Normal operation expects os.Stdout
 }
+{% endif %}
 
 // exported type for public method parameters
 type RequestInfo struct {
@@ -63,6 +120,7 @@ type pingInfo struct {
     EndTime   string `json:"end_time"`
 }
 
+{% if transport == "logging" %}
 type ping struct {
     DocumentNamespace string `json:"document_namespace"`
     DocumentType      string `json:"document_type"`
@@ -72,6 +130,7 @@ type ping struct {
     IpAddress         string `json:"ip_address,omitempty"`
     Payload           string `json:"payload"`
 }
+{% endif %}
 
 type metrics map[string]map[string]any
 
@@ -89,14 +148,20 @@ type gleanEvent struct {
     Extra     map[string]string `json:"extra"`
 }
 
+{% if transport == "logging" %}
 type logEnvelope struct {
     Timestamp string
     Logger    string
     Type      string
     Fields    ping
 }
+{% endif %}
 
+{% if transport == "pubsub" %}
+func (g GleanEventsBuilder) createClientInfo() clientInfo {
+{% else %}
 func (g GleanEventsLogger) createClientInfo() clientInfo {
+{% endif %}
     // Fields with default values are required in the Glean schema, but not used in server context
     return clientInfo{
         TelemetrySDKBuild: "glean_parser v{{ parser_version }}",
@@ -120,6 +185,7 @@ func createPingInfo() pingInfo {
     }
 }
 
+{% if transport == "logging" %}
 func (g GleanEventsLogger) createPing(documentType string, config RequestInfo, payload pingPayload) (ping, error) {
     payloadJson, err := json.Marshal(payload)
     if err != nil {
@@ -180,6 +246,7 @@ func (g GleanEventsLogger) record(
     fmt.Fprintln(g.Writer, string(envelopeJson))
     return nil
 }
+{% endif %}
 {# if any ping has an event metric, create methods and types for them #}
 {% if events %}
 
@@ -265,6 +332,101 @@ type {{ ping|ping_type_name }} struct {
     {% endif %}
 }
 
+{% if transport == "pubsub" %}
+// Build{{ ping|ping_type_name }}Message constructs a Pub/Sub message carrying
+// the given `{{ ping }}` ping. The caller publishes the returned message
+// (e.g., topic.Publish(ctx, msg)) and owns batching, retries, shutdown
+// sequencing, and any publish-result metrics.
+//
+// Wire-format contract: see
+// docs/architecture/decoder_service_specification.md in mozilla/gcp-ingestion.
+func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}Message(
+    requestInfo RequestInfo,
+    params {{ ping|ping_type_name }},
+) (*pubsub.Message, error) {
+    {% if metrics_by_type['string_list'] %}
+    {% for metric in metrics_by_type['string_list'] %}
+    // Ensure nil string_list metrics serialize as empty arrays, not null
+    if params.{{ metric|metric_argument_name }} == nil {
+        params.{{ metric|metric_argument_name }} = []string{}
+    }
+    {% endfor %}
+    {% endif %}
+    metrics := metrics{
+    {% for metric_type, metrics in metrics_by_type.items() %}
+    {% if metric_type != 'event' %}
+        "{{ metric_type }}": {
+        {% for metric in metrics %}
+            {% if metric_type == 'datetime' %}
+            "{{ metric|metric_name }}": params.{{ metric|metric_argument_name }}.Format("2006-01-02T15:04:05.000Z"),
+            {% else %}
+            "{{ metric|metric_name }}": params.{{ metric|metric_argument_name }},
+            {% endif %}
+        {% endfor %}
+        },
+    {% endif %}
+    {% endfor %}
+    }
+
+    events := []gleanEvent{}
+    {% if metrics_by_type['event'] %}
+    if params.Event != nil {
+        events = append(events, params.Event.gleanEvent())
+    }
+    {% endif %}
+
+    payload := pingPayload{
+        ClientInfo: g.createClientInfo(),
+        PingInfo:   createPingInfo(),
+        Metrics:    metrics,
+        Events:     events,
+    }
+
+    payloadJSON, err := json.Marshal(payload)
+    if err != nil {
+        return nil, fmt.Errorf("marshal ping payload: %w", err)
+    }
+    compressed, err := compressPayload(payloadJSON)
+    if err != nil {
+        return nil, fmt.Errorf("compress payload: %w", err)
+    }
+
+    documentID, err := uuid.NewRandom()
+    if err != nil {
+        return nil, fmt.Errorf("generate document_id: %w", err)
+    }
+
+    attributes := map[string]string{
+        "document_namespace": g.AppID,
+        "document_type":      "{{ ping }}",
+        "document_version":   "1",
+        "document_id":        documentID.String(),
+    }
+    // Skip empty optional attributes; the decoder treats missing and empty
+    // the same, and Pub/Sub charges for attribute bytes. The publisher does
+    // not set submission_timestamp; the decoder stamps it from publishTime.
+    if requestInfo.UserAgent != "" {
+        attributes["user_agent"] = requestInfo.UserAgent
+    }
+    if requestInfo.IpAddress != "" {
+        attributes["x_forwarded_for"] = requestInfo.IpAddress
+    }
+
+    return &pubsub.Message{
+        Data:       compressed,
+        Attributes: attributes,
+    }, nil
+}
+
+// Build{{ ping|ping_type_name }}MessageWithoutUserInfo constructs a Pub/Sub
+// message carrying the given `{{ ping }}` ping with no request-derived
+// attributes.
+func (g GleanEventsBuilder) Build{{ ping|ping_type_name }}MessageWithoutUserInfo(
+    params {{ ping|ping_type_name }},
+) (*pubsub.Message, error) {
+    return g.Build{{ ping|ping_type_name }}Message(defaultRequestInfo, params)
+}
+{% else %}
 // Record and submit `{{ ping }}` ping
 func (g GleanEventsLogger) Record{{ ping|ping_type_name }}(
     requestInfo RequestInfo,
@@ -309,4 +471,5 @@ func (g GleanEventsLogger) Record{{ ping|ping_type_name}}WithoutUserInfo(
 ) error {
     return g.Record{{ ping|ping_type_name }}(defaultRequestInfo, params)
 }
+{% endif %}
 {% endfor %}

{glean_parser-19.1.0 → glean_parser-19.2.0}/glean_parser/translate.py

@@ -57,7 +57,8 @@ class Outputter:
 
 
 OUTPUTTERS = {
-    "go_server": Outputter(go_server.output_go, []),
+    "go_server": Outputter(go_server.output_go_logger, []),
+    "go_server_pubsub": Outputter(go_server.output_go_pubsub, []),
     "javascript": Outputter(javascript.output_javascript, []),
     "typescript": Outputter(javascript.output_typescript, []),
     "javascript_server": Outputter(javascript_server.output_javascript, []),

{glean_parser-19.1.0 → glean_parser-19.2.0}/glean_parser/util.py

@@ -298,14 +298,14 @@ def fetch_remote_url(url: str, cache: bool = True):
 
     if cache:
         cache_dir = platformdirs.user_cache_dir("glean_parser", "mozilla")
-        with diskcache.Cache(cache_dir) as dc:
+        with diskcache.Cache(cache_dir, disk=diskcache.JSONDisk) as dc:
             if key in dc:
                 return dc[key]
 
-    contents: str = urllib.request.urlopen(url).read()
+    contents = urllib.request.urlopen(url).read().decode("utf-8")
 
     if cache:
-        with diskcache.Cache(cache_dir) as dc:
+        with diskcache.Cache(cache_dir, disk=diskcache.JSONDisk) as dc:
             dc[key] = contents
 
     return contents

glean_parser-19.2.0/tests/test-go/test_publisher.go.tmpl

@@ -0,0 +1,103 @@
+package main
+
+import (
+    "context"
+    "encoding/base64"
+    "encoding/json"
+    "fmt"
+    "glean/glean"
+    "os"
+    "time"
+    /* IMPORTS */
+
+    pubsub "cloud.google.com/go/pubsub/v2"
+    "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
+    "cloud.google.com/go/pubsub/v2/pstest"
+    "google.golang.org/api/option"
+    "google.golang.org/grpc"
+    "google.golang.org/grpc/credentials/insecure"
+)
+
+func main() {
+    ctx := context.Background()
+
+    // In-process Pub/Sub fake. The Go pubsub client picks up PUBSUB_EMULATOR_HOST
+    // automatically; pstest exposes its bound address via srv.Addr.
+    srv := pstest.NewServer()
+    defer srv.Close()
+    os.Setenv("PUBSUB_EMULATOR_HOST", srv.Addr)
+
+    // Pre-create the topic on the fake server. Use a transient client; the
+    // test code below opens its own client to mirror normal app usage.
+    setupConn, err := grpc.Dial(srv.Addr,
+        grpc.WithTransportCredentials(insecure.NewCredentials()))
+    if err != nil {
+        fmt.Fprintln(os.Stderr, "dial:", err)
+        os.Exit(1)
+    }
+    setupClient, err := pubsub.NewClient(ctx, "test-project", option.WithGRPCConn(setupConn))
+    if err != nil {
+        fmt.Fprintln(os.Stderr, "setup-client:", err)
+        os.Exit(1)
+    }
+    if _, err := setupClient.TopicAdminClient.CreateTopic(ctx, &pubsubpb.Topic{
+        Name: "projects/test-project/topics/test-topic",
+    }); err != nil {
+        fmt.Fprintln(os.Stderr, "create-topic:", err)
+        os.Exit(1)
+    }
+    setupClient.Close()
+    setupConn.Close()
+
+    // Client + publisher used by the test snippet to publish.
+    client, err := pubsub.NewClient(ctx, "test-project")
+    if err != nil {
+        fmt.Fprintln(os.Stderr, "client:", err)
+        os.Exit(1)
+    }
+    topic := client.Publisher("test-topic")
+
+    builder := glean.GleanEventsBuilder{
+        AppID:             "glean.test",
+        AppDisplayVersion: "0.0.1",
+        AppChannel:        "nightly",
+    }
+    _ = builder // suppress unused warning if the snippet does not reference it
+
+    // The injected snippet must assign `msg, err` by calling a builder
+    // Build<Ping>Message method; the harness owns publish + result handling.
+    /* CODE */
+
+    if err != nil {
+        fmt.Fprintln(os.Stderr, "build:", err)
+        os.Exit(1)
+    }
+    result := topic.Publish(ctx, msg)
+    if _, err := result.Get(ctx); err != nil {
+        fmt.Fprintln(os.Stderr, "publish:", err)
+        os.Exit(1)
+    }
+
+    // Drain whatever was queued, then release the client. topic.Stop
+    // blocks until the batcher flushes; client.Close releases the gRPC conn.
+    // result.Get above already confirmed the server acked the message, so it
+    // is durably in pstest by the time we read srv.Messages below.
+    topic.Stop()
+    if err := client.Close(); err != nil {
+        fmt.Fprintln(os.Stderr, "client close:", err)
+        os.Exit(1)
+    }
+
+    msgs := srv.Messages()
+    out := make([]map[string]interface{}, 0, len(msgs))
+    for _, m := range msgs {
+        out = append(out, map[string]interface{}{
+            "data":       base64.StdEncoding.EncodeToString(m.Data),
+            "attributes": m.Attributes,
+        })
+    }
+    if err := json.NewEncoder(os.Stdout).Encode(out); err != nil {
+        fmt.Fprintln(os.Stderr, "encode:", err)
+        os.Exit(1)
+    }
+}