generator.views.glean_ping_view

  1"""Class to describe a Glean Ping View."""
  2
  3import logging
  4import re
  5from collections import Counter
  6from textwrap import dedent
  7from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
  8
  9import click
 10from mozilla_schema_generator.glean_ping import GleanPing
 11from mozilla_schema_generator.probes import GleanProbe
 12
 13from . import lookml_utils
 14from .lookml_utils import slug_to_title
 15from .ping_view import PingView
 16
 17DISTRIBUTION_TYPES = {
 18    "timing_distribution",
 19    "memory_distribution",
 20    "custom_distribution",
 21}
 22
 23
 24ALLOWED_TYPES = DISTRIBUTION_TYPES | {
 25    "boolean",
 26    "labeled_boolean",
 27    "counter",
 28    "labeled_counter",
 29    "datetime",
 30    "jwe",
 31    "quantity",
 32    "string",
 33    "labeled_string",
 34    "rate",
 35    "timespan",
 36    "uuid",
 37    "url",
 38    "text",
 39}
 40
 41# Bug 1737656 - some metric types are exposed under different names
 42# We need to map to the new name when building dimensions.
 43RENAMED_METRIC_TYPES = {
 44    "jwe": "jwe2",
 45    "text": "text2",
 46    "url": "url2",
 47}
 48
 49
 50DISALLOWED_PINGS = {"events"}
 51
 52# List of labeled counter names for which a suggest explore should be generated.
 53# Generating suggest explores for all labeled counters slows down Looker.
 54SUGGESTS_FOR_LABELED_COUNTERS = {"metrics__labeled_counter__glean_error_invalid_label"}
 55
 56
 57class GleanPingView(PingView):
 58    """A view on a ping table for an application using the Glean SDK."""
 59
 60    type: str = "glean_ping_view"
 61    allow_glean: bool = True
 62
 63    @classmethod
 64    def from_db_views(klass, *args, **kwargs):
 65        """Generate GleanPingViews from db views."""
 66        for view in super().from_db_views(*args, **kwargs):
 67            if view.name not in DISALLOWED_PINGS:
 68                yield view
 69
 70    def to_lookml(self, v1_name: Optional[str], dryrun) -> Dict[str, Any]:
 71        """Generate LookML for this view.
 72
 73        The Glean views include a labeled metrics, which need to be joined
 74        against the view in the explore.
 75        """
 76        lookml = super().to_lookml(v1_name, dryrun=dryrun)
 77        # ignore nested join views
 78        lookml["views"] = [lookml["views"][0]]
 79
 80        # iterate over all of the glean metrics and generate views for unnested
 81        # fields as necessary. Append them to the list of existing view
 82        # definitions.
 83        table = next(
 84            (table for table in self.tables if table.get("channel") == "release"),
 85            self.tables[0],
 86        )["table"]
 87        dimensions = self.get_dimensions(table, v1_name, dryrun=dryrun)
 88        dimension_names = {dimension["name"] for dimension in dimensions}
 89
 90        client_id_field = self.get_client_id(dimensions, table)
 91
 92        view_definitions = []
 93        metrics = self._get_glean_metrics(v1_name)
 94        for metric in metrics:
 95            looker_name = self._to_looker_name(metric)
 96            if looker_name not in dimension_names:
 97                continue  # skip metrics with no matching dimension
 98            if metric.type == "labeled_counter":
 99                view_name = f"{self.name}__{looker_name}"
100                suggest_name = f"suggest__{view_name}"
101
102                category, name = [
103                    slug_to_title(v) for v in self._get_category_and_name(metric)
104                ]
105                view_label = f"{category} - {name}"
106                metric_hidden = "no" if metric.is_in_source() else "yes"
107
108                measures = [
109                    {
110                        "name": "count",
111                        "type": "sum",
112                        "sql": "${value}",
113                        "hidden": metric_hidden,
114                    }
115                ]
116
117                if client_id_field is not None:
118                    # client_id field is missing for pings with minimal Glean schema
119                    measures.append(
120                        {
121                            "name": "client_count",
122                            "type": "count_distinct",
123                            "sql": f"case when ${{value}} > 0 then ${{{self.name}.{client_id_field}}} end",
124                            "hidden": metric_hidden,
125                        }
126                    )
127
128                join_view: Dict[str, Any] = {
129                    "name": view_name,
130                    "label": view_label,
131                    "dimensions": [
132                        {
133                            "name": "document_id",
134                            "type": "string",
135                            "sql": f"${{{self.name}.document_id}}",
136                            "hidden": "yes",
137                        },
138                        # labeled counters need a primary key that incorporates
139                        # their labels, otherwise we get jumbled results:
140                        # https://github.com/mozilla/lookml-generator/issues/171
141                        {
142                            "name": "document_label_id",
143                            "type": "string",
144                            "sql": f"${{{self.name}.document_id}}-${{label}}",
145                            "primary_key": "yes",
146                            "hidden": "yes",
147                        },
148                        {
149                            "name": "value",
150                            "type": "number",
151                            "sql": "${TABLE}.value",
152                            "hidden": "yes",
153                        },
154                    ],
155                    "measures": measures,
156                }
157
158                if looker_name in SUGGESTS_FOR_LABELED_COUNTERS:
159                    join_view["dimensions"].append(
160                        {
161                            "name": "label",
162                            "type": "string",
163                            "sql": "${TABLE}.key",
164                            "suggest_explore": suggest_name,
165                            "suggest_dimension": f"{suggest_name}.key",
166                            "hidden": metric_hidden,
167                        },
168                    )
169
170                    suggest_view = {
171                        "name": suggest_name,
172                        "derived_table": {
173                            "sql": dedent(
174                                f"""
175                                select
176                                    m.key,
177                                    count(*) as n
178                                from {table} as t,
179                                unnest(metrics.{metric.type}.{metric.id.replace(".", "_")}) as m
180                                where date(submission_timestamp) > date_sub(current_date, interval 30 day)
181                                    and sample_id = 0
182                                group by key
183                                order by n desc
184                                """
185                            )
186                        },
187                        "dimensions": [
188                            {"name": "key", "type": "string", "sql": "${TABLE}.key"}
189                        ],
190                    }
191                    view_definitions += [join_view, suggest_view]
192                else:
193                    join_view["dimensions"].append(
194                        {
195                            "name": "label",
196                            "type": "string",
197                            "sql": "${TABLE}.key",
198                            "hidden": metric_hidden,
199                        },
200                    )
201                    view_definitions += [join_view]
202
203        # deduplicate view definitions, because somehow a few entries make it in
204        # twice e.g. metrics__metrics__labeled_counter__media_audio_init_failure
205        view_definitions = sorted(
206            {v["name"]: v for v in view_definitions}.values(), key=lambda x: x["name"]  # type: ignore
207        )
208
209        [project, dataset, table] = table.split(".")
210        table_schema = dryrun.create(
211            project=project,
212            dataset=dataset,
213            table=table,
214        ).get_table_schema()
215        nested_views = lookml_utils._generate_nested_dimension_views(
216            table_schema, self.name
217        )
218
219        lookml["views"] += view_definitions + nested_views
220
221        return lookml
222
223    def _get_links(self, dimension: dict) -> List[Dict[str, str]]:
224        """Get a link annotation given a metric name."""
225        name = self._get_name(dimension)
226        title = slug_to_title(name)
227        return [
228            {
229                "label": (f"Glean Dictionary reference for {title}"),
230                "url": (
231                    f"https://dictionary.telemetry.mozilla.org"
232                    f"/apps/{self.namespace}/metrics/{name}"
233                ),
234                "icon_url": "https://dictionary.telemetry.mozilla.org/favicon.png",
235            }
236        ]
237
238    def _get_name(self, dimension: dict) -> str:
239        return dimension["name"].split("__")[-1]
240
241    def _get_metric_type(self, dimension: dict) -> str:
242        return dimension["name"].split("__")[1]
243
244    def _is_metric(self, dimension) -> bool:
245        return dimension["name"].startswith("metrics__")
246
247    def _get_glean_metrics(self, v1_name: Optional[str]) -> List[GleanProbe]:
248        if v1_name is None:
249            logging.error(
250                f"Error: Missing v1 name for ping {self.name} in namespace {self.namespace}"
251            )
252            return []
253
254        repo = next((r for r in GleanPing.get_repos() if r["name"] == v1_name))
255        glean_app = GleanPing(repo)
256
257        ping_probes = []
258        probe_ids = set()
259        for probe in glean_app.get_probes():
260            send_in_pings_snakecase = [
261                ping.replace("-", "_") for ping in probe.definition["send_in_pings"]
262            ]
263            if self.name not in send_in_pings_snakecase:
264                continue
265            if probe.id in probe_ids:
266                # Some ids are duplicated, ignore them
267                continue
268
269            ping_probes.append(probe)
270            probe_ids.add(probe.id)
271
272        return ping_probes
273
274    def _get_category_and_name(self, metric: GleanProbe) -> Tuple[str, str]:
275        *category, name = metric.id.split(".")
276        category = "_".join(category)
277
278        return category, name
279
280    def _to_looker_name(self, metric: GleanProbe, suffix: str = "") -> str:
281        """Convert a glean probe into a looker name."""
282        category, name = self._get_category_and_name(metric)
283
284        sep = "" if not category else "_"
285        label = name
286        looker_name = f"metrics__{metric.type}__{category}{sep}{label}"
287        if suffix:
288            looker_name = f"{looker_name}__{suffix}"
289        return looker_name
290
291    def _make_dimension(
292        self, metric: GleanProbe, suffix: str, sql_map: Dict[str, Dict[str, str]]
293    ) -> Optional[Dict[str, Union[str, List[Dict[str, str]]]]]:
294        *category, name = metric.id.split(".")
295        category = "_".join(category)
296
297        sep = "" if not category else "_"
298        label = name
299        type = RENAMED_METRIC_TYPES.get(metric.type, metric.type)
300        looker_name = f"metrics__{type}__{category}{sep}{name}"
301        if suffix:
302            label = f"{name}_{suffix}"
303            looker_name = f"{looker_name}__{suffix}"
304
305        if looker_name not in sql_map:
306            return None
307
308        group_label = slug_to_title(category)
309        group_item_label = slug_to_title(label)
310
311        if not group_label:
312            group_label = "Glean"
313
314        friendly_name = f"{group_label} {group_item_label}"
315
316        lookml = {
317            "name": looker_name,
318            "label": friendly_name,
319            # metrics that are no longer in the source are hidden by default
320            "hidden": "no" if metric.is_in_source() else "yes",
321            "sql": sql_map[looker_name]["sql"],
322            "type": sql_map[looker_name]["type"],
323            "group_label": group_label,
324            "group_item_label": group_item_label,
325            "links": [
326                {
327                    "label": (f"Glean Dictionary reference for {friendly_name}"),
328                    "url": (
329                        f"https://dictionary.telemetry.mozilla.org"
330                        f"/apps/{self.namespace}/metrics/{category}{sep}{name}"
331                    ),
332                    "icon_url": "https://dictionary.telemetry.mozilla.org/favicon.png",
333                },
334            ],
335        }
336
337        if lookml["type"] == "time":
338            # Remove any _{type} suffix from the dimension group name because each timeframe
339            # will add a _{type} suffix to its individual dimension name.
340            lookml["name"] = re.sub("_(date|time(stamp)?)$", "", looker_name)
341            lookml["timeframes"] = [
342                "raw",
343                "time",
344                "date",
345                "week",
346                "month",
347                "quarter",
348                "year",
349            ]
350            # Dimension groups should not be nested (see issue #82).
351            del lookml["group_label"]
352            del lookml["group_item_label"]
353            # Links are not supported for dimension groups.
354            del lookml["links"]
355
356        # remove some elements from the definition if we're handling a labeled
357        # counter, as an initial join dimension
358        if metric.type == "labeled_counter":
359            # this field is not used since labeled counters are maps
360            del lookml["type"]
361            lookml["hidden"] = "yes"
362
363        if metric.description:
364            lookml["description"] = metric.description
365
366        return lookml
367
368    def _get_metric_dimensions(
369        self, metric: GleanProbe, sql_map: Dict[str, Dict[str, str]]
370    ) -> Iterable[Optional[Dict[str, Union[str, List[Dict[str, str]]]]]]:
371        if metric.type == "rate":
372            for suffix in ("numerator", "denominator"):
373                yield self._make_dimension(metric, suffix, sql_map)
374        elif metric.type in DISTRIBUTION_TYPES:
375            yield self._make_dimension(metric, "sum", sql_map)
376        elif metric.type == "timespan":
377            yield self._make_dimension(metric, "value", sql_map)
378        elif metric.type in ALLOWED_TYPES:
379            yield self._make_dimension(metric, "", sql_map)
380
381    def _get_glean_metric_dimensions(
382        self, all_fields: List[dict], v1_name: Optional[str]
383    ):
384        sql_map = {
385            f["name"]: {"sql": f["sql"], "type": f.get("type", "string")}
386            for f in all_fields
387        }
388        metrics = self._get_glean_metrics(v1_name)
389        return [
390            dimension
391            for metric in metrics
392            for dimension in self._get_metric_dimensions(metric, sql_map)
393            if dimension is not None
394        ]
395
396    def _add_link(self, dimension):
397        annotations = {}
398        if self._is_metric(dimension) and not self._get_metric_type(
399            dimension
400        ).startswith("labeled"):
401            annotations["links"] = self._get_links(dimension)
402
403        return dict(dimension, **annotations)
404
405    def get_dimensions(
406        self, table, v1_name: Optional[str], dryrun
407    ) -> List[Dict[str, Any]]:
408        """Get the set of dimensions for this view."""
409        all_fields = super().get_dimensions(table, v1_name, dryrun=dryrun)
410        fields = self._get_glean_metric_dimensions(all_fields, v1_name) + [
411            self._add_link(d)
412            for d in all_fields
413            if not d["name"].startswith("metrics__")
414        ]
415        # later entries will override earlier entries, if there are duplicates
416        field_dict = {f["name"]: f for f in fields}
417        return list(field_dict.values())
418
419    def get_measures(
420        self, dimensions: List[dict], table: str, v1_name: Optional[str]
421    ) -> List[Dict[str, Union[str, List[Dict[str, str]]]]]:
422        """Generate measures from a list of dimensions.
423
424        When no dimension-specific measures are found, return a single "count" measure.
425
426        Raise ClickException if dimensions result in duplicate measures.
427        """
428        measures = super().get_measures(dimensions, table, v1_name)
429        client_id_field = self.get_client_id(dimensions, table)
430
431        for dimension in dimensions:
432            if (
433                self._is_metric(dimension)
434                and self._get_metric_type(dimension) == "counter"
435            ):
436                # handle the counters in the metric ping
437                name = self._get_name(dimension)
438                dimension_name = dimension["name"]
439                measures += [
440                    {
441                        "name": name,
442                        "type": "sum",
443                        "sql": f"${{{dimension_name}}}",
444                        "links": self._get_links(dimension),
445                    },
446                ]
447
448                if client_id_field is not None:
449                    measures += [
450                        {
451                            "name": f"{name}_client_count",
452                            "type": "count_distinct",
453                            "filters": [{dimension_name: ">0"}],
454                            "sql": f"${{{client_id_field}}}",
455                            "links": self._get_links(dimension),
456                        },
457                    ]
458
459        # check if there are any duplicate values
460        names = [measure["name"] for measure in measures]
461        duplicates = [k for k, v in Counter(names).items() if v > 1]
462        if duplicates:
463            raise click.ClickException(
464                f"duplicate measures {duplicates!r} for table {table!r}"
465            )
466
467        return measures