mozilla_schema_generator.glean_ping

  1# -*- coding: utf-8 -*-
  2
  3# This Source Code Form is subject to the terms of the Mozilla Public
  4# License, v. 2.0. If a copy of the MPL was not distributed with this
  5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
  6
  7import logging
  8from pathlib import Path
  9from typing import Dict, List, Set
 10
 11from requests import HTTPError
 12
 13from .config import Config
 14from .generic_ping import GenericPing
 15from .probes import GleanProbe
 16from .schema import Schema
 17
 18ROOT_DIR = Path(__file__).parent
 19BUG_1737656_TXT = ROOT_DIR / "configs" / "bug_1737656_affected.txt"
 20
 21logger = logging.getLogger(__name__)
 22
 23
 24class GleanPing(GenericPing):
 25    schema_url = (
 26        "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas"
 27        "/{branch}/schemas/glean/glean/glean.1.schema.json"
 28    )
 29    probes_url_template = GenericPing.probe_info_base_url + "/glean/{}/metrics"
 30    ping_url_template = GenericPing.probe_info_base_url + "/glean/{}/pings"
 31    repos_url = GenericPing.probe_info_base_url + "/glean/repositories"
 32    dependencies_url_template = (
 33        GenericPing.probe_info_base_url + "/glean/{}/dependencies"
 34    )
 35
 36    default_dependencies = ["glean-core"]
 37
 38    with open(BUG_1737656_TXT, "r") as f:
 39        bug_1737656_affected_tables = [
 40            line.strip() for line in f.readlines() if line.strip()
 41        ]
 42
 43    def __init__(self, repo, **kwargs):  # TODO: Make env-url optional
 44        self.repo = repo
 45        self.repo_name = repo["name"]
 46        self.app_id = repo["app_id"]
 47        super().__init__(
 48            self.schema_url,
 49            self.schema_url,
 50            self.probes_url_template.format(self.repo_name),
 51            **kwargs,
 52        )
 53
 54    def get_schema(self, generic_schema=False) -> Schema:
 55        """
 56        Fetch schema via URL.
 57
 58        Unless *generic_schema* is set to true, this function makes some modifications
 59        to allow some workarounds for proper injection of metrics.
 60        """
 61        schema = super().get_schema()
 62        if generic_schema:
 63            return schema
 64
 65        # We need to inject placeholders for the url2, text2, etc. types as part
 66        # of mitigation for https://bugzilla.mozilla.org/show_bug.cgi?id=1737656
 67        for metric_name in ["labeled_rate", "jwe", "url", "text"]:
 68            metric1 = schema.get(
 69                ("properties", "metrics", "properties", metric_name)
 70            ).copy()
 71            metric1 = schema.set_schema_elem(
 72                ("properties", "metrics", "properties", metric_name + "2"),
 73                metric1,
 74            )
 75
 76        return schema
 77
 78    def get_dependencies(self):
 79        # Get all of the library dependencies for the application that
 80        # are also known about in the repositories file.
 81
 82        # The dependencies are specified using library names, but we need to
 83        # map those back to the name of the repository in the repository file.
 84        try:
 85            dependencies = self._get_json(
 86                self.dependencies_url_template.format(self.repo_name)
 87            )
 88        except HTTPError:
 89            logging.info(f"For {self.repo_name}, using default Glean dependencies")
 90            return self.default_dependencies
 91
 92        dependency_library_names = list(dependencies.keys())
 93
 94        repos = GleanPing._get_json(GleanPing.repos_url)
 95        repos_by_dependency_name = {}
 96        for repo in repos:
 97            for library_name in repo.get("library_names", []):
 98                repos_by_dependency_name[library_name] = repo["name"]
 99
100        dependencies = []
101        for name in dependency_library_names:
102            if name in repos_by_dependency_name:
103                dependencies.append(repos_by_dependency_name[name])
104
105        if len(dependencies) == 0:
106            logging.info(f"For {self.repo_name}, using default Glean dependencies")
107            return self.default_dependencies
108
109        logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}")
110        return dependencies
111
112    def get_probes(self) -> List[GleanProbe]:
113        data = self._get_json(self.probes_url)
114        probes = list(data.items())
115
116        for dependency in self.get_dependencies():
117            dependency_probes = self._get_json(
118                self.probes_url_template.format(dependency)
119            )
120            probes += list(dependency_probes.items())
121
122        pings = self.get_pings()
123
124        processed = []
125        for _id, defn in probes:
126            probe = GleanProbe(_id, defn, pings=pings)
127            processed.append(probe)
128
129            # Manual handling of incompatible schema changes
130            issue_118_affected = {
131                "fenix",
132                "fenix-nightly",
133                "firefox-android-nightly",
134                "firefox-android-beta",
135                "firefox-android-release",
136            }
137            if (
138                self.repo_name in issue_118_affected
139                and probe.get_name() == "installation.timestamp"
140            ):
141                logging.info(f"Writing column {probe.get_name()} for compatibility.")
142                # See: https://github.com/mozilla/mozilla-schema-generator/issues/118
143                # Search through history for the "string" type and add a copy of
144                # the probe at that time in history. The changepoint signifies
145                # this event.
146                changepoint_index = 0
147                for definition in probe.definition_history:
148                    if definition["type"] != probe.get_type():
149                        break
150                    changepoint_index += 1
151                # Modify the definition with the truncated history.
152                hist_defn = defn.copy()
153                hist_defn[probe.history_key] = probe.definition_history[
154                    changepoint_index:
155                ]
156                hist_defn["type"] = hist_defn[probe.history_key][0]["type"]
157                incompatible_probe_type = GleanProbe(_id, hist_defn, pings=pings)
158                processed.append(incompatible_probe_type)
159
160            # Handling probe type changes (Bug 1870317)
161            probe_types = {hist["type"] for hist in defn[probe.history_key]}
162            if len(probe_types) > 1:
163                # The probe type changed at some point in history.
164                # Create schema entry for each type.
165                hist_defn = defn.copy()
166
167                # No new entry needs to be created for the current probe type
168                probe_types.remove(defn["type"])
169
170                for hist in hist_defn[probe.history_key]:
171                    # Create a new entry for a historic type
172                    if hist["type"] in probe_types:
173                        hist_defn["type"] = hist["type"]
174                        probe = GleanProbe(_id, hist_defn, pings=pings)
175                        processed.append(probe)
176
177                        # Keep track of the types entries were already created for
178                        probe_types.remove(hist["type"])
179
180        return processed
181
182    def _get_ping_data(self) -> Dict[str, Dict]:
183        url = self.ping_url_template.format(self.repo_name)
184        ping_data = GleanPing._get_json(url)
185        for dependency in self.get_dependencies():
186            dependency_pings = self._get_json(self.ping_url_template.format(dependency))
187            ping_data.update(dependency_pings)
188        return ping_data
189
190    def _get_ping_data_without_dependencies(self) -> Dict[str, Dict]:
191        url = self.ping_url_template.format(self.repo_name)
192        ping_data = GleanPing._get_json(url)
193        return ping_data
194
195    def _get_dependency_pings(self, dependency):
196        return self._get_json(self.ping_url_template.format(dependency))
197
198    def get_pings(self) -> Set[str]:
199        return self._get_ping_data().keys()
200
201    @staticmethod
202    def apply_default_metadata(ping_metadata, default_metadata):
203        """apply_default_metadata recurses down into dicts nested
204        to an arbitrary depth, updating keys. The ``default_metadata`` is merged into
205        ``ping_metadata``.
206        :param ping_metadata: dict onto which the merge is executed
207        :param default_metadata: dct merged into ping_metadata
208        :return: None
209        """
210        for k, v in default_metadata.items():
211            if (
212                k in ping_metadata
213                and isinstance(ping_metadata[k], dict)
214                and isinstance(default_metadata[k], dict)
215            ):
216                GleanPing.apply_default_metadata(ping_metadata[k], default_metadata[k])
217            else:
218                ping_metadata[k] = default_metadata[k]
219
220    def _get_ping_data_and_dependencies_with_default_metadata(self) -> Dict[str, Dict]:
221        # Get the ping data with the pipeline metadata
222        ping_data = self._get_ping_data_without_dependencies()
223
224        # The ping endpoint for the dependency pings does not include any repo defined
225        # moz_pipeline_metadata_defaults so they need to be applied here.
226
227        # 1.  Get repo and pipeline default metadata.
228        repos = GleanPing.get_repos()
229        current_repo = next((x for x in repos if x.get("app_id") == self.app_id), {})
230        default_metadata = current_repo.get("moz_pipeline_metadata_defaults", {})
231
232        # 2.  Apply the default metadata to each dependency defined ping.
233        for dependency in self.get_dependencies():
234            dependency_pings = self._get_dependency_pings(dependency)
235            for dependency_ping in dependency_pings.values():
236                # Although it is counter intuitive to apply the default metadata on top of the
237                # existing dependency ping metadata it does set the repo specific value for
238                # bq_dataset_family instead of using the dependency id for the bq_dataset_family
239                # value.
240                GleanPing.apply_default_metadata(
241                    dependency_ping.get("moz_pipeline_metadata"), default_metadata
242                )
243            ping_data.update(dependency_pings)
244        return ping_data
245
246    @staticmethod
247    def reorder_metadata(metadata):
248        desired_order_list = [
249            "bq_dataset_family",
250            "bq_table",
251            "bq_metadata_format",
252            "submission_timestamp_granularity",
253            "expiration_policy",
254            "override_attributes",
255            "jwe_mappings",
256        ]
257        reordered_metadata = {
258            k: metadata[k] for k in desired_order_list if k in metadata
259        }
260
261        # re-order jwe-mappings
262        desired_order_list = ["source_field_path", "decrypted_field_path"]
263        jwe_mapping_metadata = reordered_metadata.get("jwe_mappings")
264        if jwe_mapping_metadata:
265            reordered_jwe_mapping_metadata = []
266            for mapping in jwe_mapping_metadata:
267                reordered_jwe_mapping_metadata.append(
268                    {k: mapping[k] for k in desired_order_list if k in mapping}
269                )
270            reordered_metadata["jwe_mappings"] = reordered_jwe_mapping_metadata
271
272        # future proofing, in case there are other fields added at the ping top level
273        # add them to the end.
274        leftovers = {k: metadata[k] for k in set(metadata) - set(reordered_metadata)}
275        reordered_metadata = {**reordered_metadata, **leftovers}
276        return reordered_metadata
277
278    def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]:
279        pings = self._get_ping_data_and_dependencies_with_default_metadata()
280        for ping_name, ping_data in pings.items():
281            metadata = ping_data.get("moz_pipeline_metadata")
282
283            # While technically unnecessary, the dictionary elements are re-ordered to match the
284            # currently deployed order and used to verify no difference in output.
285            pings[ping_name] = GleanPing.reorder_metadata(metadata)
286        return pings
287
288    def get_ping_descriptions(self) -> Dict[str, str]:
289        return {
290            k: v["history"][-1]["description"] for k, v in self._get_ping_data().items()
291        }
292
293    def generate_schema(self, config, generic_schema=False) -> Dict[str, Schema]:
294        pings = self.get_pings_and_pipeline_metadata()
295        schemas = {}
296
297        for ping, pipeline_meta in pings.items():
298            matchers = {
299                loc: m.clone(new_table_group=ping) for loc, m in config.matchers.items()
300            }
301
302            # Four newly introduced metric types were incorrectly deployed
303            # as repeated key/value structs in all Glean ping tables existing prior
304            # to November 2021. We maintain the incorrect fields for existing tables
305            # by disabling the associated matchers.
306            # Note that each of these types now has a "2" matcher ("text2", "url2", etc.)
307            # defined that will allow metrics of these types to be injected into proper
308            # structs. The gcp-ingestion repository includes logic to rewrite these
309            # metrics under the "2" names.
310            # See https://bugzilla.mozilla.org/show_bug.cgi?id=1737656
311            bq_identifier = "{bq_dataset_family}.{bq_table}".format(**pipeline_meta)
312            if bq_identifier in self.bug_1737656_affected_tables:
313                matchers = {
314                    loc: m
315                    for loc, m in matchers.items()
316                    if not m.matcher.get("bug_1737656_affected")
317                }
318
319            for matcher in matchers.values():
320                matcher.matcher["send_in_pings"]["contains"] = ping
321            new_config = Config(ping, matchers=matchers)
322
323            defaults = {"mozPipelineMetadata": pipeline_meta}
324
325            if generic_schema:  # Use the generic glean ping schema
326                schema = self.get_schema(generic_schema=True)
327                schema.schema.update(defaults)
328                schemas[new_config.name] = schema
329            else:
330                generated = super().generate_schema(new_config)
331                for schema in generated.values():
332                    # We want to override each individual key with assembled defaults,
333                    # but keep values _inside_ them if they have been set in the schemas.
334                    for key, value in defaults.items():
335                        if key not in schema.schema:
336                            schema.schema[key] = {}
337                        schema.schema[key].update(value)
338                schemas.update(generated)
339
340        return schemas
341
342    @staticmethod
343    def get_repos():
344        """
345        Retrieve metadata for all non-library Glean repositories
346        """
347        repos = GleanPing._get_json(GleanPing.repos_url)
348        return [repo for repo in repos if "library_names" not in repo]
ROOT_DIR = PosixPath('/home/circleci/project/mozilla_schema_generator')
BUG_1737656_TXT = PosixPath('/home/circleci/project/mozilla_schema_generator/configs/bug_1737656_affected.txt')
logger = <Logger mozilla_schema_generator.glean_ping (WARNING)>
 25class GleanPing(GenericPing):
 26    schema_url = (
 27        "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas"
 28        "/{branch}/schemas/glean/glean/glean.1.schema.json"
 29    )
 30    probes_url_template = GenericPing.probe_info_base_url + "/glean/{}/metrics"
 31    ping_url_template = GenericPing.probe_info_base_url + "/glean/{}/pings"
 32    repos_url = GenericPing.probe_info_base_url + "/glean/repositories"
 33    dependencies_url_template = (
 34        GenericPing.probe_info_base_url + "/glean/{}/dependencies"
 35    )
 36
 37    default_dependencies = ["glean-core"]
 38
 39    with open(BUG_1737656_TXT, "r") as f:
 40        bug_1737656_affected_tables = [
 41            line.strip() for line in f.readlines() if line.strip()
 42        ]
 43
 44    def __init__(self, repo, **kwargs):  # TODO: Make env-url optional
 45        self.repo = repo
 46        self.repo_name = repo["name"]
 47        self.app_id = repo["app_id"]
 48        super().__init__(
 49            self.schema_url,
 50            self.schema_url,
 51            self.probes_url_template.format(self.repo_name),
 52            **kwargs,
 53        )
 54
 55    def get_schema(self, generic_schema=False) -> Schema:
 56        """
 57        Fetch schema via URL.
 58
 59        Unless *generic_schema* is set to true, this function makes some modifications
 60        to allow some workarounds for proper injection of metrics.
 61        """
 62        schema = super().get_schema()
 63        if generic_schema:
 64            return schema
 65
 66        # We need to inject placeholders for the url2, text2, etc. types as part
 67        # of mitigation for https://bugzilla.mozilla.org/show_bug.cgi?id=1737656
 68        for metric_name in ["labeled_rate", "jwe", "url", "text"]:
 69            metric1 = schema.get(
 70                ("properties", "metrics", "properties", metric_name)
 71            ).copy()
 72            metric1 = schema.set_schema_elem(
 73                ("properties", "metrics", "properties", metric_name + "2"),
 74                metric1,
 75            )
 76
 77        return schema
 78
 79    def get_dependencies(self):
 80        # Get all of the library dependencies for the application that
 81        # are also known about in the repositories file.
 82
 83        # The dependencies are specified using library names, but we need to
 84        # map those back to the name of the repository in the repository file.
 85        try:
 86            dependencies = self._get_json(
 87                self.dependencies_url_template.format(self.repo_name)
 88            )
 89        except HTTPError:
 90            logging.info(f"For {self.repo_name}, using default Glean dependencies")
 91            return self.default_dependencies
 92
 93        dependency_library_names = list(dependencies.keys())
 94
 95        repos = GleanPing._get_json(GleanPing.repos_url)
 96        repos_by_dependency_name = {}
 97        for repo in repos:
 98            for library_name in repo.get("library_names", []):
 99                repos_by_dependency_name[library_name] = repo["name"]
100
101        dependencies = []
102        for name in dependency_library_names:
103            if name in repos_by_dependency_name:
104                dependencies.append(repos_by_dependency_name[name])
105
106        if len(dependencies) == 0:
107            logging.info(f"For {self.repo_name}, using default Glean dependencies")
108            return self.default_dependencies
109
110        logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}")
111        return dependencies
112
113    def get_probes(self) -> List[GleanProbe]:
114        data = self._get_json(self.probes_url)
115        probes = list(data.items())
116
117        for dependency in self.get_dependencies():
118            dependency_probes = self._get_json(
119                self.probes_url_template.format(dependency)
120            )
121            probes += list(dependency_probes.items())
122
123        pings = self.get_pings()
124
125        processed = []
126        for _id, defn in probes:
127            probe = GleanProbe(_id, defn, pings=pings)
128            processed.append(probe)
129
130            # Manual handling of incompatible schema changes
131            issue_118_affected = {
132                "fenix",
133                "fenix-nightly",
134                "firefox-android-nightly",
135                "firefox-android-beta",
136                "firefox-android-release",
137            }
138            if (
139                self.repo_name in issue_118_affected
140                and probe.get_name() == "installation.timestamp"
141            ):
142                logging.info(f"Writing column {probe.get_name()} for compatibility.")
143                # See: https://github.com/mozilla/mozilla-schema-generator/issues/118
144                # Search through history for the "string" type and add a copy of
145                # the probe at that time in history. The changepoint signifies
146                # this event.
147                changepoint_index = 0
148                for definition in probe.definition_history:
149                    if definition["type"] != probe.get_type():
150                        break
151                    changepoint_index += 1
152                # Modify the definition with the truncated history.
153                hist_defn = defn.copy()
154                hist_defn[probe.history_key] = probe.definition_history[
155                    changepoint_index:
156                ]
157                hist_defn["type"] = hist_defn[probe.history_key][0]["type"]
158                incompatible_probe_type = GleanProbe(_id, hist_defn, pings=pings)
159                processed.append(incompatible_probe_type)
160
161            # Handling probe type changes (Bug 1870317)
162            probe_types = {hist["type"] for hist in defn[probe.history_key]}
163            if len(probe_types) > 1:
164                # The probe type changed at some point in history.
165                # Create schema entry for each type.
166                hist_defn = defn.copy()
167
168                # No new entry needs to be created for the current probe type
169                probe_types.remove(defn["type"])
170
171                for hist in hist_defn[probe.history_key]:
172                    # Create a new entry for a historic type
173                    if hist["type"] in probe_types:
174                        hist_defn["type"] = hist["type"]
175                        probe = GleanProbe(_id, hist_defn, pings=pings)
176                        processed.append(probe)
177
178                        # Keep track of the types entries were already created for
179                        probe_types.remove(hist["type"])
180
181        return processed
182
183    def _get_ping_data(self) -> Dict[str, Dict]:
184        url = self.ping_url_template.format(self.repo_name)
185        ping_data = GleanPing._get_json(url)
186        for dependency in self.get_dependencies():
187            dependency_pings = self._get_json(self.ping_url_template.format(dependency))
188            ping_data.update(dependency_pings)
189        return ping_data
190
191    def _get_ping_data_without_dependencies(self) -> Dict[str, Dict]:
192        url = self.ping_url_template.format(self.repo_name)
193        ping_data = GleanPing._get_json(url)
194        return ping_data
195
196    def _get_dependency_pings(self, dependency):
197        return self._get_json(self.ping_url_template.format(dependency))
198
199    def get_pings(self) -> Set[str]:
200        return self._get_ping_data().keys()
201
202    @staticmethod
203    def apply_default_metadata(ping_metadata, default_metadata):
204        """apply_default_metadata recurses down into dicts nested
205        to an arbitrary depth, updating keys. The ``default_metadata`` is merged into
206        ``ping_metadata``.
207        :param ping_metadata: dict onto which the merge is executed
208        :param default_metadata: dct merged into ping_metadata
209        :return: None
210        """
211        for k, v in default_metadata.items():
212            if (
213                k in ping_metadata
214                and isinstance(ping_metadata[k], dict)
215                and isinstance(default_metadata[k], dict)
216            ):
217                GleanPing.apply_default_metadata(ping_metadata[k], default_metadata[k])
218            else:
219                ping_metadata[k] = default_metadata[k]
220
221    def _get_ping_data_and_dependencies_with_default_metadata(self) -> Dict[str, Dict]:
222        # Get the ping data with the pipeline metadata
223        ping_data = self._get_ping_data_without_dependencies()
224
225        # The ping endpoint for the dependency pings does not include any repo defined
226        # moz_pipeline_metadata_defaults so they need to be applied here.
227
228        # 1.  Get repo and pipeline default metadata.
229        repos = GleanPing.get_repos()
230        current_repo = next((x for x in repos if x.get("app_id") == self.app_id), {})
231        default_metadata = current_repo.get("moz_pipeline_metadata_defaults", {})
232
233        # 2.  Apply the default metadata to each dependency defined ping.
234        for dependency in self.get_dependencies():
235            dependency_pings = self._get_dependency_pings(dependency)
236            for dependency_ping in dependency_pings.values():
237                # Although it is counter intuitive to apply the default metadata on top of the
238                # existing dependency ping metadata it does set the repo specific value for
239                # bq_dataset_family instead of using the dependency id for the bq_dataset_family
240                # value.
241                GleanPing.apply_default_metadata(
242                    dependency_ping.get("moz_pipeline_metadata"), default_metadata
243                )
244            ping_data.update(dependency_pings)
245        return ping_data
246
247    @staticmethod
248    def reorder_metadata(metadata):
249        desired_order_list = [
250            "bq_dataset_family",
251            "bq_table",
252            "bq_metadata_format",
253            "submission_timestamp_granularity",
254            "expiration_policy",
255            "override_attributes",
256            "jwe_mappings",
257        ]
258        reordered_metadata = {
259            k: metadata[k] for k in desired_order_list if k in metadata
260        }
261
262        # re-order jwe-mappings
263        desired_order_list = ["source_field_path", "decrypted_field_path"]
264        jwe_mapping_metadata = reordered_metadata.get("jwe_mappings")
265        if jwe_mapping_metadata:
266            reordered_jwe_mapping_metadata = []
267            for mapping in jwe_mapping_metadata:
268                reordered_jwe_mapping_metadata.append(
269                    {k: mapping[k] for k in desired_order_list if k in mapping}
270                )
271            reordered_metadata["jwe_mappings"] = reordered_jwe_mapping_metadata
272
273        # future proofing, in case there are other fields added at the ping top level
274        # add them to the end.
275        leftovers = {k: metadata[k] for k in set(metadata) - set(reordered_metadata)}
276        reordered_metadata = {**reordered_metadata, **leftovers}
277        return reordered_metadata
278
279    def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]:
280        pings = self._get_ping_data_and_dependencies_with_default_metadata()
281        for ping_name, ping_data in pings.items():
282            metadata = ping_data.get("moz_pipeline_metadata")
283
284            # While technically unnecessary, the dictionary elements are re-ordered to match the
285            # currently deployed order and used to verify no difference in output.
286            pings[ping_name] = GleanPing.reorder_metadata(metadata)
287        return pings
288
289    def get_ping_descriptions(self) -> Dict[str, str]:
290        return {
291            k: v["history"][-1]["description"] for k, v in self._get_ping_data().items()
292        }
293
294    def generate_schema(self, config, generic_schema=False) -> Dict[str, Schema]:
295        pings = self.get_pings_and_pipeline_metadata()
296        schemas = {}
297
298        for ping, pipeline_meta in pings.items():
299            matchers = {
300                loc: m.clone(new_table_group=ping) for loc, m in config.matchers.items()
301            }
302
303            # Four newly introduced metric types were incorrectly deployed
304            # as repeated key/value structs in all Glean ping tables existing prior
305            # to November 2021. We maintain the incorrect fields for existing tables
306            # by disabling the associated matchers.
307            # Note that each of these types now has a "2" matcher ("text2", "url2", etc.)
308            # defined that will allow metrics of these types to be injected into proper
309            # structs. The gcp-ingestion repository includes logic to rewrite these
310            # metrics under the "2" names.
311            # See https://bugzilla.mozilla.org/show_bug.cgi?id=1737656
312            bq_identifier = "{bq_dataset_family}.{bq_table}".format(**pipeline_meta)
313            if bq_identifier in self.bug_1737656_affected_tables:
314                matchers = {
315                    loc: m
316                    for loc, m in matchers.items()
317                    if not m.matcher.get("bug_1737656_affected")
318                }
319
320            for matcher in matchers.values():
321                matcher.matcher["send_in_pings"]["contains"] = ping
322            new_config = Config(ping, matchers=matchers)
323
324            defaults = {"mozPipelineMetadata": pipeline_meta}
325
326            if generic_schema:  # Use the generic glean ping schema
327                schema = self.get_schema(generic_schema=True)
328                schema.schema.update(defaults)
329                schemas[new_config.name] = schema
330            else:
331                generated = super().generate_schema(new_config)
332                for schema in generated.values():
333                    # We want to override each individual key with assembled defaults,
334                    # but keep values _inside_ them if they have been set in the schemas.
335                    for key, value in defaults.items():
336                        if key not in schema.schema:
337                            schema.schema[key] = {}
338                        schema.schema[key].update(value)
339                schemas.update(generated)
340
341        return schemas
342
343    @staticmethod
344    def get_repos():
345        """
346        Retrieve metadata for all non-library Glean repositories
347        """
348        repos = GleanPing._get_json(GleanPing.repos_url)
349        return [repo for repo in repos if "library_names" not in repo]
GleanPing(repo, **kwargs)
44    def __init__(self, repo, **kwargs):  # TODO: Make env-url optional
45        self.repo = repo
46        self.repo_name = repo["name"]
47        self.app_id = repo["app_id"]
48        super().__init__(
49            self.schema_url,
50            self.schema_url,
51            self.probes_url_template.format(self.repo_name),
52            **kwargs,
53        )
schema_url = 'https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas/{branch}/schemas/glean/glean/glean.1.schema.json'
probes_url_template = 'https://probeinfo.telemetry.mozilla.org/glean/{}/metrics'
ping_url_template = 'https://probeinfo.telemetry.mozilla.org/glean/{}/pings'
repos_url = 'https://probeinfo.telemetry.mozilla.org/glean/repositories'
dependencies_url_template = 'https://probeinfo.telemetry.mozilla.org/glean/{}/dependencies'
default_dependencies = ['glean-core']
repo
repo_name
app_id
def get_schema(self, generic_schema=False) -> mozilla_schema_generator.schema.Schema:
55    def get_schema(self, generic_schema=False) -> Schema:
56        """
57        Fetch schema via URL.
58
59        Unless *generic_schema* is set to true, this function makes some modifications
60        to allow some workarounds for proper injection of metrics.
61        """
62        schema = super().get_schema()
63        if generic_schema:
64            return schema
65
66        # We need to inject placeholders for the url2, text2, etc. types as part
67        # of mitigation for https://bugzilla.mozilla.org/show_bug.cgi?id=1737656
68        for metric_name in ["labeled_rate", "jwe", "url", "text"]:
69            metric1 = schema.get(
70                ("properties", "metrics", "properties", metric_name)
71            ).copy()
72            metric1 = schema.set_schema_elem(
73                ("properties", "metrics", "properties", metric_name + "2"),
74                metric1,
75            )
76
77        return schema

Fetch schema via URL.

Unless generic_schema is set to true, this function makes some modifications to allow some workarounds for proper injection of metrics.

def get_dependencies(self):
 79    def get_dependencies(self):
 80        # Get all of the library dependencies for the application that
 81        # are also known about in the repositories file.
 82
 83        # The dependencies are specified using library names, but we need to
 84        # map those back to the name of the repository in the repository file.
 85        try:
 86            dependencies = self._get_json(
 87                self.dependencies_url_template.format(self.repo_name)
 88            )
 89        except HTTPError:
 90            logging.info(f"For {self.repo_name}, using default Glean dependencies")
 91            return self.default_dependencies
 92
 93        dependency_library_names = list(dependencies.keys())
 94
 95        repos = GleanPing._get_json(GleanPing.repos_url)
 96        repos_by_dependency_name = {}
 97        for repo in repos:
 98            for library_name in repo.get("library_names", []):
 99                repos_by_dependency_name[library_name] = repo["name"]
100
101        dependencies = []
102        for name in dependency_library_names:
103            if name in repos_by_dependency_name:
104                dependencies.append(repos_by_dependency_name[name])
105
106        if len(dependencies) == 0:
107            logging.info(f"For {self.repo_name}, using default Glean dependencies")
108            return self.default_dependencies
109
110        logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}")
111        return dependencies
def get_probes(self) -> List[mozilla_schema_generator.probes.GleanProbe]:
113    def get_probes(self) -> List[GleanProbe]:
114        data = self._get_json(self.probes_url)
115        probes = list(data.items())
116
117        for dependency in self.get_dependencies():
118            dependency_probes = self._get_json(
119                self.probes_url_template.format(dependency)
120            )
121            probes += list(dependency_probes.items())
122
123        pings = self.get_pings()
124
125        processed = []
126        for _id, defn in probes:
127            probe = GleanProbe(_id, defn, pings=pings)
128            processed.append(probe)
129
130            # Manual handling of incompatible schema changes
131            issue_118_affected = {
132                "fenix",
133                "fenix-nightly",
134                "firefox-android-nightly",
135                "firefox-android-beta",
136                "firefox-android-release",
137            }
138            if (
139                self.repo_name in issue_118_affected
140                and probe.get_name() == "installation.timestamp"
141            ):
142                logging.info(f"Writing column {probe.get_name()} for compatibility.")
143                # See: https://github.com/mozilla/mozilla-schema-generator/issues/118
144                # Search through history for the "string" type and add a copy of
145                # the probe at that time in history. The changepoint signifies
146                # this event.
147                changepoint_index = 0
148                for definition in probe.definition_history:
149                    if definition["type"] != probe.get_type():
150                        break
151                    changepoint_index += 1
152                # Modify the definition with the truncated history.
153                hist_defn = defn.copy()
154                hist_defn[probe.history_key] = probe.definition_history[
155                    changepoint_index:
156                ]
157                hist_defn["type"] = hist_defn[probe.history_key][0]["type"]
158                incompatible_probe_type = GleanProbe(_id, hist_defn, pings=pings)
159                processed.append(incompatible_probe_type)
160
161            # Handling probe type changes (Bug 1870317)
162            probe_types = {hist["type"] for hist in defn[probe.history_key]}
163            if len(probe_types) > 1:
164                # The probe type changed at some point in history.
165                # Create schema entry for each type.
166                hist_defn = defn.copy()
167
168                # No new entry needs to be created for the current probe type
169                probe_types.remove(defn["type"])
170
171                for hist in hist_defn[probe.history_key]:
172                    # Create a new entry for a historic type
173                    if hist["type"] in probe_types:
174                        hist_defn["type"] = hist["type"]
175                        probe = GleanProbe(_id, hist_defn, pings=pings)
176                        processed.append(probe)
177
178                        # Keep track of the types entries were already created for
179                        probe_types.remove(hist["type"])
180
181        return processed
def get_pings(self) -> Set[str]:
199    def get_pings(self) -> Set[str]:
200        return self._get_ping_data().keys()
@staticmethod
def apply_default_metadata(ping_metadata, default_metadata):
202    @staticmethod
203    def apply_default_metadata(ping_metadata, default_metadata):
204        """apply_default_metadata recurses down into dicts nested
205        to an arbitrary depth, updating keys. The ``default_metadata`` is merged into
206        ``ping_metadata``.
207        :param ping_metadata: dict onto which the merge is executed
208        :param default_metadata: dct merged into ping_metadata
209        :return: None
210        """
211        for k, v in default_metadata.items():
212            if (
213                k in ping_metadata
214                and isinstance(ping_metadata[k], dict)
215                and isinstance(default_metadata[k], dict)
216            ):
217                GleanPing.apply_default_metadata(ping_metadata[k], default_metadata[k])
218            else:
219                ping_metadata[k] = default_metadata[k]

apply_default_metadata recurses down into dicts nested to an arbitrary depth, updating keys. The default_metadata is merged into ping_metadata.

Parameters
  • ping_metadata: dict onto which the merge is executed
  • default_metadata: dct merged into ping_metadata
Returns

None

@staticmethod
def reorder_metadata(metadata):
247    @staticmethod
248    def reorder_metadata(metadata):
249        desired_order_list = [
250            "bq_dataset_family",
251            "bq_table",
252            "bq_metadata_format",
253            "submission_timestamp_granularity",
254            "expiration_policy",
255            "override_attributes",
256            "jwe_mappings",
257        ]
258        reordered_metadata = {
259            k: metadata[k] for k in desired_order_list if k in metadata
260        }
261
262        # re-order jwe-mappings
263        desired_order_list = ["source_field_path", "decrypted_field_path"]
264        jwe_mapping_metadata = reordered_metadata.get("jwe_mappings")
265        if jwe_mapping_metadata:
266            reordered_jwe_mapping_metadata = []
267            for mapping in jwe_mapping_metadata:
268                reordered_jwe_mapping_metadata.append(
269                    {k: mapping[k] for k in desired_order_list if k in mapping}
270                )
271            reordered_metadata["jwe_mappings"] = reordered_jwe_mapping_metadata
272
273        # future proofing, in case there are other fields added at the ping top level
274        # add them to the end.
275        leftovers = {k: metadata[k] for k in set(metadata) - set(reordered_metadata)}
276        reordered_metadata = {**reordered_metadata, **leftovers}
277        return reordered_metadata
def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]:
279    def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]:
280        pings = self._get_ping_data_and_dependencies_with_default_metadata()
281        for ping_name, ping_data in pings.items():
282            metadata = ping_data.get("moz_pipeline_metadata")
283
284            # While technically unnecessary, the dictionary elements are re-ordered to match the
285            # currently deployed order and used to verify no difference in output.
286            pings[ping_name] = GleanPing.reorder_metadata(metadata)
287        return pings
def get_ping_descriptions(self) -> Dict[str, str]:
289    def get_ping_descriptions(self) -> Dict[str, str]:
290        return {
291            k: v["history"][-1]["description"] for k, v in self._get_ping_data().items()
292        }
def generate_schema( self, config, generic_schema=False) -> Dict[str, mozilla_schema_generator.schema.Schema]:
294    def generate_schema(self, config, generic_schema=False) -> Dict[str, Schema]:
295        pings = self.get_pings_and_pipeline_metadata()
296        schemas = {}
297
298        for ping, pipeline_meta in pings.items():
299            matchers = {
300                loc: m.clone(new_table_group=ping) for loc, m in config.matchers.items()
301            }
302
303            # Four newly introduced metric types were incorrectly deployed
304            # as repeated key/value structs in all Glean ping tables existing prior
305            # to November 2021. We maintain the incorrect fields for existing tables
306            # by disabling the associated matchers.
307            # Note that each of these types now has a "2" matcher ("text2", "url2", etc.)
308            # defined that will allow metrics of these types to be injected into proper
309            # structs. The gcp-ingestion repository includes logic to rewrite these
310            # metrics under the "2" names.
311            # See https://bugzilla.mozilla.org/show_bug.cgi?id=1737656
312            bq_identifier = "{bq_dataset_family}.{bq_table}".format(**pipeline_meta)
313            if bq_identifier in self.bug_1737656_affected_tables:
314                matchers = {
315                    loc: m
316                    for loc, m in matchers.items()
317                    if not m.matcher.get("bug_1737656_affected")
318                }
319
320            for matcher in matchers.values():
321                matcher.matcher["send_in_pings"]["contains"] = ping
322            new_config = Config(ping, matchers=matchers)
323
324            defaults = {"mozPipelineMetadata": pipeline_meta}
325
326            if generic_schema:  # Use the generic glean ping schema
327                schema = self.get_schema(generic_schema=True)
328                schema.schema.update(defaults)
329                schemas[new_config.name] = schema
330            else:
331                generated = super().generate_schema(new_config)
332                for schema in generated.values():
333                    # We want to override each individual key with assembled defaults,
334                    # but keep values _inside_ them if they have been set in the schemas.
335                    for key, value in defaults.items():
336                        if key not in schema.schema:
337                            schema.schema[key] = {}
338                        schema.schema[key].update(value)
339                schemas.update(generated)
340
341        return schemas
@staticmethod
def get_repos():
343    @staticmethod
344    def get_repos():
345        """
346        Retrieve metadata for all non-library Glean repositories
347        """
348        repos = GleanPing._get_json(GleanPing.repos_url)
349        return [repo for repo in repos if "library_names" not in repo]

Retrieve metadata for all non-library Glean repositories

f = <_io.TextIOWrapper name='/home/circleci/project/mozilla_schema_generator/configs/bug_1737656_affected.txt' mode='r' encoding='UTF-8'>
bug_1737656_affected_tables = ['burnham.baseline_v1', 'burnham.deletion_request_v1', 'burnham.discovery_v1', 'burnham.events_v1', 'burnham.metrics_v1', 'burnham.space_ship_ready_v1', 'burnham.starbase46_v1', 'firefox_desktop_background_update.background_update_v1', 'firefox_desktop_background_update.baseline_v1', 'firefox_desktop_background_update.deletion_request_v1', 'firefox_desktop_background_update.events_v1', 'firefox_desktop_background_update.metrics_v1', 'firefox_desktop.baseline_v1', 'firefox_desktop.deletion_request_v1', 'firefox_desktop.events_v1', 'firefox_desktop.fog_validation_v1', 'firefox_desktop.metrics_v1', 'firefox_installer.install_v1', 'firefox_launcher_process.launcher_process_failure_v1', 'messaging_system.cfr_v1', 'messaging_system.infobar_v1', 'messaging_system.moments_v1', 'messaging_system.onboarding_v1', 'messaging_system.personalization_experiment_v1', 'messaging_system.snippets_v1', 'messaging_system.spotlight_v1', 'messaging_system.undesired_events_v1', 'messaging_system.whats_new_panel_v1', 'mlhackweek_search.action_v1', 'mlhackweek_search.baseline_v1', 'mlhackweek_search.custom_v1', 'mlhackweek_search.deletion_request_v1', 'mlhackweek_search.events_v1', 'mlhackweek_search.metrics_v1', 'mozilla_lockbox.addresses_sync_v1', 'mozilla_lockbox.baseline_v1', 'mozilla_lockbox.bookmarks_sync_v1', 'mozilla_lockbox.creditcards_sync_v1', 'mozilla_lockbox.deletion_request_v1', 'mozilla_lockbox.events_v1', 'mozilla_lockbox.history_sync_v1', 'mozilla_lockbox.logins_sync_v1', 'mozilla_lockbox.metrics_v1', 'mozilla_lockbox.sync_v1', 'mozilla_lockbox.tabs_sync_v1', 'mozilla_mach.baseline_v1', 'mozilla_mach.deletion_request_v1', 'mozilla_mach.events_v1', 'mozilla_mach.metrics_v1', 'mozilla_mach.usage_v1', 'mozillavpn.deletion_request_v1', 'mozillavpn.main_v1', 'mozphab.baseline_v1', 'mozphab.deletion_request_v1', 'mozphab.events_v1', 'mozphab.metrics_v1', 'mozphab.usage_v1', 'org_mozilla_bergamot.custom_v1', 'org_mozilla_bergamot.deletion_request_v1', 'org_mozilla_connect_firefox.baseline_v1', 'org_mozilla_connect_firefox.deletion_request_v1', 'org_mozilla_connect_firefox.events_v1', 'org_mozilla_connect_firefox.metrics_v1', 'org_mozilla_fenix.activation_v1', 'org_mozilla_fenix.addresses_sync_v1', 'org_mozilla_fenix.baseline_v1', 'org_mozilla_fenix.bookmarks_sync_v1', 'org_mozilla_fenix.creditcards_sync_v1', 'org_mozilla_fenix.deletion_request_v1', 'org_mozilla_fenix.events_v1', 'org_mozilla_fenix.first_session_v1', 'org_mozilla_fenix.fog_validation_v1', 'org_mozilla_fenix.history_sync_v1', 'org_mozilla_fenix.installation_v1', 'org_mozilla_fenix.logins_sync_v1', 'org_mozilla_fenix.metrics_v1', 'org_mozilla_fenix.migration_v1', 'org_mozilla_fenix.startup_timeline_v1', 'org_mozilla_fenix.sync_v1', 'org_mozilla_fenix.tabs_sync_v1', 'org_mozilla_fenix_nightly.activation_v1', 'org_mozilla_fenix_nightly.addresses_sync_v1', 'org_mozilla_fenix_nightly.baseline_v1', 'org_mozilla_fenix_nightly.bookmarks_sync_v1', 'org_mozilla_fenix_nightly.creditcards_sync_v1', 'org_mozilla_fenix_nightly.deletion_request_v1', 'org_mozilla_fenix_nightly.events_v1', 'org_mozilla_fenix_nightly.first_session_v1', 'org_mozilla_fenix_nightly.fog_validation_v1', 'org_mozilla_fenix_nightly.history_sync_v1', 'org_mozilla_fenix_nightly.installation_v1', 'org_mozilla_fenix_nightly.logins_sync_v1', 'org_mozilla_fenix_nightly.metrics_v1', 'org_mozilla_fenix_nightly.migration_v1', 'org_mozilla_fenix_nightly.startup_timeline_v1', 'org_mozilla_fenix_nightly.sync_v1', 'org_mozilla_fenix_nightly.tabs_sync_v1', 'org_mozilla_fennec_aurora.activation_v1', 'org_mozilla_fennec_aurora.addresses_sync_v1', 'org_mozilla_fennec_aurora.baseline_v1', 'org_mozilla_fennec_aurora.bookmarks_sync_v1', 'org_mozilla_fennec_aurora.creditcards_sync_v1', 'org_mozilla_fennec_aurora.deletion_request_v1', 'org_mozilla_fennec_aurora.events_v1', 'org_mozilla_fennec_aurora.first_session_v1', 'org_mozilla_fennec_aurora.fog_validation_v1', 'org_mozilla_fennec_aurora.history_sync_v1', 'org_mozilla_fennec_aurora.installation_v1', 'org_mozilla_fennec_aurora.logins_sync_v1', 'org_mozilla_fennec_aurora.metrics_v1', 'org_mozilla_fennec_aurora.migration_v1', 'org_mozilla_fennec_aurora.startup_timeline_v1', 'org_mozilla_fennec_aurora.sync_v1', 'org_mozilla_fennec_aurora.tabs_sync_v1', 'org_mozilla_firefox_beta.activation_v1', 'org_mozilla_firefox_beta.addresses_sync_v1', 'org_mozilla_firefox_beta.baseline_v1', 'org_mozilla_firefox_beta.bookmarks_sync_v1', 'org_mozilla_firefox_beta.creditcards_sync_v1', 'org_mozilla_firefox_beta.deletion_request_v1', 'org_mozilla_firefox_beta.events_v1', 'org_mozilla_firefox_beta.first_session_v1', 'org_mozilla_firefox_beta.fog_validation_v1', 'org_mozilla_firefox_beta.history_sync_v1', 'org_mozilla_firefox_beta.installation_v1', 'org_mozilla_firefox_beta.logins_sync_v1', 'org_mozilla_firefox_beta.metrics_v1', 'org_mozilla_firefox_beta.migration_v1', 'org_mozilla_firefox_beta.startup_timeline_v1', 'org_mozilla_firefox_beta.sync_v1', 'org_mozilla_firefox_beta.tabs_sync_v1', 'org_mozilla_firefox.activation_v1', 'org_mozilla_firefox.addresses_sync_v1', 'org_mozilla_firefox.baseline_v1', 'org_mozilla_firefox.bookmarks_sync_v1', 'org_mozilla_firefox.creditcards_sync_v1', 'org_mozilla_firefox.deletion_request_v1', 'org_mozilla_firefox.events_v1', 'org_mozilla_firefox.first_session_v1', 'org_mozilla_firefox.fog_validation_v1', 'org_mozilla_firefox.history_sync_v1', 'org_mozilla_firefox.installation_v1', 'org_mozilla_firefox.logins_sync_v1', 'org_mozilla_firefox.metrics_v1', 'org_mozilla_firefox.migration_v1', 'org_mozilla_firefox.startup_timeline_v1', 'org_mozilla_firefox.sync_v1', 'org_mozilla_firefox.tabs_sync_v1', 'org_mozilla_firefoxreality.baseline_v1', 'org_mozilla_firefoxreality.deletion_request_v1', 'org_mozilla_firefoxreality.events_v1', 'org_mozilla_firefoxreality.launch_v1', 'org_mozilla_firefoxreality.metrics_v1', 'org_mozilla_focus_beta.activation_v1', 'org_mozilla_focus_beta.baseline_v1', 'org_mozilla_focus_beta.deletion_request_v1', 'org_mozilla_focus_beta.events_v1', 'org_mozilla_focus_beta.metrics_v1', 'org_mozilla_focus.activation_v1', 'org_mozilla_focus.baseline_v1', 'org_mozilla_focus.deletion_request_v1', 'org_mozilla_focus.events_v1', 'org_mozilla_focus.metrics_v1', 'org_mozilla_focus_nightly.activation_v1', 'org_mozilla_focus_nightly.baseline_v1', 'org_mozilla_focus_nightly.deletion_request_v1', 'org_mozilla_focus_nightly.events_v1', 'org_mozilla_focus_nightly.metrics_v1', 'org_mozilla_ios_fennec.baseline_v1', 'org_mozilla_ios_fennec.deletion_request_v1', 'org_mozilla_ios_fennec.events_v1', 'org_mozilla_ios_fennec.metrics_v1', 'org_mozilla_ios_firefox.baseline_v1', 'org_mozilla_ios_firefox.deletion_request_v1', 'org_mozilla_ios_firefox.events_v1', 'org_mozilla_ios_firefox.metrics_v1', 'org_mozilla_ios_firefoxbeta.baseline_v1', 'org_mozilla_ios_firefoxbeta.deletion_request_v1', 'org_mozilla_ios_firefoxbeta.events_v1', 'org_mozilla_ios_firefoxbeta.metrics_v1', 'org_mozilla_ios_focus.baseline_v1', 'org_mozilla_ios_focus.deletion_request_v1', 'org_mozilla_ios_focus.events_v1', 'org_mozilla_ios_focus.metrics_v1', 'org_mozilla_ios_klar.baseline_v1', 'org_mozilla_ios_klar.deletion_request_v1', 'org_mozilla_ios_klar.events_v1', 'org_mozilla_ios_klar.metrics_v1', 'org_mozilla_ios_lockbox.baseline_v1', 'org_mozilla_ios_lockbox.deletion_request_v1', 'org_mozilla_ios_lockbox.events_v1', 'org_mozilla_ios_lockbox.metrics_v1', 'org_mozilla_klar.activation_v1', 'org_mozilla_klar.baseline_v1', 'org_mozilla_klar.deletion_request_v1', 'org_mozilla_klar.events_v1', 'org_mozilla_klar.metrics_v1', 'org_mozilla_mozregression.baseline_v1', 'org_mozilla_mozregression.deletion_request_v1', 'org_mozilla_mozregression.events_v1', 'org_mozilla_mozregression.metrics_v1', 'org_mozilla_mozregression.usage_v1', 'org_mozilla_reference_browser.baseline_v1', 'org_mozilla_reference_browser.deletion_request_v1', 'org_mozilla_reference_browser.events_v1', 'org_mozilla_reference_browser.metrics_v1', 'org_mozilla_tv_firefox.baseline_v1', 'org_mozilla_tv_firefox.deletion_request_v1', 'org_mozilla_tv_firefox.events_v1', 'org_mozilla_tv_firefox.metrics_v1', 'org_mozilla_vrbrowser.addresses_sync_v1', 'org_mozilla_vrbrowser.baseline_v1', 'org_mozilla_vrbrowser.bookmarks_sync_v1', 'org_mozilla_vrbrowser.creditcards_sync_v1', 'org_mozilla_vrbrowser.deletion_request_v1', 'org_mozilla_vrbrowser.events_v1', 'org_mozilla_vrbrowser.history_sync_v1', 'org_mozilla_vrbrowser.logins_sync_v1', 'org_mozilla_vrbrowser.metrics_v1', 'org_mozilla_vrbrowser.session_end_v1', 'org_mozilla_vrbrowser.sync_v1', 'org_mozilla_vrbrowser.tabs_sync_v1', 'rally_core.deletion_request_v1', 'rally_core.demographics_v1', 'rally_core.enrollment_v1', 'rally_core.study_enrollment_v1', 'rally_core.study_unenrollment_v1', 'rally_core.uninstall_deletion_v1', 'rally_debug.deletion_request_v1', 'rally_debug.demographics_v1', 'rally_debug.enrollment_v1', 'rally_debug.study_enrollment_v1', 'rally_debug.study_unenrollment_v1', 'rally_debug.uninstall_deletion_v1', 'rally_study_zero_one.deletion_request_v1', 'rally_study_zero_one.rs01_event_v1', 'rally_study_zero_one.study_enrollment_v1', 'rally_zero_one.deletion_request_v1', 'rally_zero_one.measurements_v1', 'rally_zero_one.pioneer_enrollment_v1']