mozilla_schema_generator.glean_ping
1# -*- coding: utf-8 -*- 2 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7import logging 8from pathlib import Path 9from typing import Dict, List, Set 10 11from requests import HTTPError 12 13from .config import Config 14from .generic_ping import GenericPing 15from .probes import GleanProbe 16from .schema import Schema 17 18ROOT_DIR = Path(__file__).parent 19BUG_1737656_TXT = ROOT_DIR / "configs" / "bug_1737656_affected.txt" 20 21logger = logging.getLogger(__name__) 22 23DEFAULT_SCHEMA_URL = ( 24 "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas" 25 "/{branch}/schemas/glean/glean/glean.1.schema.json" 26) 27 28MINIMUM_SCHEMA_URL = ( 29 "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas" 30 "/{branch}/schemas/glean/glean/glean-min.1.schema.json" 31) 32 33 34class GleanPing(GenericPing): 35 probes_url_template = GenericPing.probe_info_base_url + "/glean/{}/metrics" 36 ping_url_template = GenericPing.probe_info_base_url + "/glean/{}/pings" 37 repos_url = GenericPing.probe_info_base_url + "/glean/repositories" 38 dependencies_url_template = ( 39 GenericPing.probe_info_base_url + "/glean/{}/dependencies" 40 ) 41 42 default_dependencies = ["glean-core"] 43 44 with open(BUG_1737656_TXT, "r") as f: 45 bug_1737656_affected_tables = [ 46 line.strip() for line in f.readlines() if line.strip() 47 ] 48 49 def __init__(self, repo, **kwargs): # TODO: Make env-url optional 50 self.repo = repo 51 self.repo_name = repo["name"] 52 self.app_id = repo["app_id"] 53 super().__init__( 54 DEFAULT_SCHEMA_URL, 55 DEFAULT_SCHEMA_URL, 56 self.probes_url_template.format(self.repo_name), 57 **kwargs, 58 ) 59 60 def get_schema(self, generic_schema=False) -> Schema: 61 """ 62 Fetch schema via URL. 63 64 Unless *generic_schema* is set to true, this function makes some modifications 65 to allow some workarounds for proper injection of metrics. 66 """ 67 schema = super().get_schema() 68 if generic_schema: 69 return schema 70 71 # We need to inject placeholders for the url2, text2, etc. types as part 72 # of mitigation for https://bugzilla.mozilla.org/show_bug.cgi?id=1737656 73 for metric_name in ["labeled_rate", "jwe", "url", "text"]: 74 metric1 = schema.get( 75 ("properties", "metrics", "properties", metric_name) 76 ).copy() 77 metric1 = schema.set_schema_elem( 78 ("properties", "metrics", "properties", metric_name + "2"), 79 metric1, 80 ) 81 82 return schema 83 84 def get_dependencies(self): 85 # Get all of the library dependencies for the application that 86 # are also known about in the repositories file. 87 88 # The dependencies are specified using library names, but we need to 89 # map those back to the name of the repository in the repository file. 90 try: 91 dependencies = self._get_json( 92 self.dependencies_url_template.format(self.repo_name) 93 ) 94 except HTTPError: 95 logging.info(f"For {self.repo_name}, using default Glean dependencies") 96 return self.default_dependencies 97 98 dependency_library_names = list(dependencies.keys()) 99 100 repos = GleanPing._get_json(GleanPing.repos_url) 101 repos_by_dependency_name = {} 102 for repo in repos: 103 for library_name in repo.get("library_names", []): 104 repos_by_dependency_name[library_name] = repo["name"] 105 106 dependencies = [] 107 for name in dependency_library_names: 108 if name in repos_by_dependency_name: 109 dependencies.append(repos_by_dependency_name[name]) 110 111 if len(dependencies) == 0: 112 logging.info(f"For {self.repo_name}, using default Glean dependencies") 113 return self.default_dependencies 114 115 logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}") 116 return dependencies 117 118 def get_probes(self) -> List[GleanProbe]: 119 data = self._get_json(self.probes_url) 120 probes = list(data.items()) 121 122 for dependency in self.get_dependencies(): 123 dependency_probes = self._get_json( 124 self.probes_url_template.format(dependency) 125 ) 126 probes += list(dependency_probes.items()) 127 128 pings = self.get_pings() 129 130 processed = [] 131 for _id, defn in probes: 132 probe = GleanProbe(_id, defn, pings=pings) 133 processed.append(probe) 134 135 # Manual handling of incompatible schema changes 136 issue_118_affected = { 137 "fenix", 138 "fenix-nightly", 139 "firefox-android-nightly", 140 "firefox-android-beta", 141 "firefox-android-release", 142 } 143 if ( 144 self.repo_name in issue_118_affected 145 and probe.get_name() == "installation.timestamp" 146 ): 147 logging.info(f"Writing column {probe.get_name()} for compatibility.") 148 # See: https://github.com/mozilla/mozilla-schema-generator/issues/118 149 # Search through history for the "string" type and add a copy of 150 # the probe at that time in history. The changepoint signifies 151 # this event. 152 changepoint_index = 0 153 for definition in probe.definition_history: 154 if definition["type"] != probe.get_type(): 155 break 156 changepoint_index += 1 157 # Modify the definition with the truncated history. 158 hist_defn = defn.copy() 159 hist_defn[probe.history_key] = probe.definition_history[ 160 changepoint_index: 161 ] 162 hist_defn["type"] = hist_defn[probe.history_key][0]["type"] 163 incompatible_probe_type = GleanProbe(_id, hist_defn, pings=pings) 164 processed.append(incompatible_probe_type) 165 166 # Handling probe type changes (Bug 1870317) 167 probe_types = {hist["type"] for hist in defn[probe.history_key]} 168 if len(probe_types) > 1: 169 # The probe type changed at some point in history. 170 # Create schema entry for each type. 171 hist_defn = defn.copy() 172 173 # No new entry needs to be created for the current probe type 174 probe_types.remove(defn["type"]) 175 176 for hist in hist_defn[probe.history_key]: 177 # Create a new entry for a historic type 178 if hist["type"] in probe_types: 179 hist_defn["type"] = hist["type"] 180 probe = GleanProbe(_id, hist_defn, pings=pings) 181 processed.append(probe) 182 183 # Keep track of the types entries were already created for 184 probe_types.remove(hist["type"]) 185 186 return processed 187 188 def _get_ping_data(self) -> Dict[str, Dict]: 189 url = self.ping_url_template.format(self.repo_name) 190 ping_data = GleanPing._get_json(url) 191 for dependency in self.get_dependencies(): 192 dependency_pings = self._get_json(self.ping_url_template.format(dependency)) 193 ping_data.update(dependency_pings) 194 return ping_data 195 196 def _get_ping_data_without_dependencies(self) -> Dict[str, Dict]: 197 url = self.ping_url_template.format(self.repo_name) 198 ping_data = GleanPing._get_json(url) 199 return ping_data 200 201 def _get_dependency_pings(self, dependency): 202 return self._get_json(self.ping_url_template.format(dependency)) 203 204 def get_pings(self) -> Set[str]: 205 return self._get_ping_data().keys() 206 207 @staticmethod 208 def apply_default_metadata(ping_metadata, default_metadata): 209 """apply_default_metadata recurses down into dicts nested 210 to an arbitrary depth, updating keys. The ``default_metadata`` is merged into 211 ``ping_metadata``. 212 :param ping_metadata: dict onto which the merge is executed 213 :param default_metadata: dct merged into ping_metadata 214 :return: None 215 """ 216 for k, v in default_metadata.items(): 217 if ( 218 k in ping_metadata 219 and isinstance(ping_metadata[k], dict) 220 and isinstance(default_metadata[k], dict) 221 ): 222 GleanPing.apply_default_metadata(ping_metadata[k], default_metadata[k]) 223 else: 224 ping_metadata[k] = default_metadata[k] 225 226 def _get_ping_data_and_dependencies_with_default_metadata(self) -> Dict[str, Dict]: 227 # Get the ping data with the pipeline metadata 228 ping_data = self._get_ping_data_without_dependencies() 229 230 # The ping endpoint for the dependency pings does not include any repo defined 231 # moz_pipeline_metadata_defaults so they need to be applied here. 232 233 # 1. Get repo and pipeline default metadata. 234 repos = GleanPing.get_repos() 235 current_repo = next((x for x in repos if x.get("app_id") == self.app_id), {}) 236 default_metadata = current_repo.get("moz_pipeline_metadata_defaults", {}) 237 238 # 2. Apply the default metadata to each dependency defined ping. 239 for dependency in self.get_dependencies(): 240 dependency_pings = self._get_dependency_pings(dependency) 241 for dependency_ping in dependency_pings.values(): 242 # Although it is counter intuitive to apply the default metadata on top of the 243 # existing dependency ping metadata it does set the repo specific value for 244 # bq_dataset_family instead of using the dependency id for the bq_dataset_family 245 # value. 246 GleanPing.apply_default_metadata( 247 dependency_ping.get("moz_pipeline_metadata"), default_metadata 248 ) 249 ping_data.update(dependency_pings) 250 return ping_data 251 252 @staticmethod 253 def reorder_metadata(metadata): 254 desired_order_list = [ 255 "bq_dataset_family", 256 "bq_table", 257 "bq_metadata_format", 258 "include_info_sections", 259 "submission_timestamp_granularity", 260 "expiration_policy", 261 "override_attributes", 262 "jwe_mappings", 263 ] 264 reordered_metadata = { 265 k: metadata[k] for k in desired_order_list if k in metadata 266 } 267 268 # re-order jwe-mappings 269 desired_order_list = ["source_field_path", "decrypted_field_path"] 270 jwe_mapping_metadata = reordered_metadata.get("jwe_mappings") 271 if jwe_mapping_metadata: 272 reordered_jwe_mapping_metadata = [] 273 for mapping in jwe_mapping_metadata: 274 reordered_jwe_mapping_metadata.append( 275 {k: mapping[k] for k in desired_order_list if k in mapping} 276 ) 277 reordered_metadata["jwe_mappings"] = reordered_jwe_mapping_metadata 278 279 # future proofing, in case there are other fields added at the ping top level 280 # add them to the end. 281 leftovers = {k: metadata[k] for k in set(metadata) - set(reordered_metadata)} 282 reordered_metadata = {**reordered_metadata, **leftovers} 283 return reordered_metadata 284 285 def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]: 286 pings = self._get_ping_data_and_dependencies_with_default_metadata() 287 for ping_name, ping_data in pings.items(): 288 metadata = ping_data.get("moz_pipeline_metadata") 289 metadata["include_info_sections"] = self._include_info_sections(ping_data) 290 291 # While technically unnecessary, the dictionary elements are re-ordered to match the 292 # currently deployed order and used to verify no difference in output. 293 pings[ping_name] = GleanPing.reorder_metadata(metadata) 294 return pings 295 296 def get_ping_descriptions(self) -> Dict[str, str]: 297 return { 298 k: v["history"][-1]["description"] for k, v in self._get_ping_data().items() 299 } 300 301 def _include_info_sections(self, ping_data) -> bool: 302 # Default to true if not specified. 303 if "history" not in ping_data or len(ping_data["history"]) == 0: 304 return True 305 latest_ping_data = ping_data["history"][-1] 306 return ( 307 "include_info_sections" not in latest_ping_data 308 or latest_ping_data["include_info_sections"] 309 ) 310 311 def set_schema_url(self, metadata): 312 """ 313 Switch between the glean-min and glean schemas if the ping does not require 314 info sections as specified in the parsed ping info in probe scraper. 315 """ 316 if not metadata["include_info_sections"]: 317 self.schema_url = MINIMUM_SCHEMA_URL.format(branch=self.branch_name) 318 else: 319 self.schema_url = DEFAULT_SCHEMA_URL.format(branch=self.branch_name) 320 321 def generate_schema(self, config, generic_schema=False) -> Dict[str, Schema]: 322 pings = self.get_pings_and_pipeline_metadata() 323 schemas = {} 324 325 for ping, pipeline_meta in pings.items(): 326 matchers = { 327 loc: m.clone(new_table_group=ping) for loc, m in config.matchers.items() 328 } 329 330 # Four newly introduced metric types were incorrectly deployed 331 # as repeated key/value structs in all Glean ping tables existing prior 332 # to November 2021. We maintain the incorrect fields for existing tables 333 # by disabling the associated matchers. 334 # Note that each of these types now has a "2" matcher ("text2", "url2", etc.) 335 # defined that will allow metrics of these types to be injected into proper 336 # structs. The gcp-ingestion repository includes logic to rewrite these 337 # metrics under the "2" names. 338 # See https://bugzilla.mozilla.org/show_bug.cgi?id=1737656 339 bq_identifier = "{bq_dataset_family}.{bq_table}".format(**pipeline_meta) 340 if bq_identifier in self.bug_1737656_affected_tables: 341 matchers = { 342 loc: m 343 for loc, m in matchers.items() 344 if not m.matcher.get("bug_1737656_affected") 345 } 346 347 for matcher in matchers.values(): 348 matcher.matcher["send_in_pings"]["contains"] = ping 349 new_config = Config(ping, matchers=matchers) 350 351 defaults = {"mozPipelineMetadata": pipeline_meta} 352 353 # Adjust the schema path if the ping does not require info sections 354 self.set_schema_url(pipeline_meta) 355 if generic_schema: # Use the generic glean ping schema 356 schema = self.get_schema(generic_schema=True) 357 schema.schema.update(defaults) 358 schemas[new_config.name] = schema 359 else: 360 generated = super().generate_schema(new_config) 361 for schema in generated.values(): 362 # We want to override each individual key with assembled defaults, 363 # but keep values _inside_ them if they have been set in the schemas. 364 for key, value in defaults.items(): 365 if key not in schema.schema: 366 schema.schema[key] = {} 367 schema.schema[key].update(value) 368 schemas.update(generated) 369 370 return schemas 371 372 @staticmethod 373 def get_repos(): 374 """ 375 Retrieve metadata for all non-library Glean repositories 376 """ 377 repos = GleanPing._get_json(GleanPing.repos_url) 378 return [repo for repo in repos if "library_names" not in repo]
ROOT_DIR =
PosixPath('/home/circleci/project/mozilla_schema_generator')
BUG_1737656_TXT =
PosixPath('/home/circleci/project/mozilla_schema_generator/configs/bug_1737656_affected.txt')
logger =
<Logger mozilla_schema_generator.glean_ping (WARNING)>
DEFAULT_SCHEMA_URL =
'https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas/{branch}/schemas/glean/glean/glean.1.schema.json'
MINIMUM_SCHEMA_URL =
'https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas/{branch}/schemas/glean/glean/glean-min.1.schema.json'
35class GleanPing(GenericPing): 36 probes_url_template = GenericPing.probe_info_base_url + "/glean/{}/metrics" 37 ping_url_template = GenericPing.probe_info_base_url + "/glean/{}/pings" 38 repos_url = GenericPing.probe_info_base_url + "/glean/repositories" 39 dependencies_url_template = ( 40 GenericPing.probe_info_base_url + "/glean/{}/dependencies" 41 ) 42 43 default_dependencies = ["glean-core"] 44 45 with open(BUG_1737656_TXT, "r") as f: 46 bug_1737656_affected_tables = [ 47 line.strip() for line in f.readlines() if line.strip() 48 ] 49 50 def __init__(self, repo, **kwargs): # TODO: Make env-url optional 51 self.repo = repo 52 self.repo_name = repo["name"] 53 self.app_id = repo["app_id"] 54 super().__init__( 55 DEFAULT_SCHEMA_URL, 56 DEFAULT_SCHEMA_URL, 57 self.probes_url_template.format(self.repo_name), 58 **kwargs, 59 ) 60 61 def get_schema(self, generic_schema=False) -> Schema: 62 """ 63 Fetch schema via URL. 64 65 Unless *generic_schema* is set to true, this function makes some modifications 66 to allow some workarounds for proper injection of metrics. 67 """ 68 schema = super().get_schema() 69 if generic_schema: 70 return schema 71 72 # We need to inject placeholders for the url2, text2, etc. types as part 73 # of mitigation for https://bugzilla.mozilla.org/show_bug.cgi?id=1737656 74 for metric_name in ["labeled_rate", "jwe", "url", "text"]: 75 metric1 = schema.get( 76 ("properties", "metrics", "properties", metric_name) 77 ).copy() 78 metric1 = schema.set_schema_elem( 79 ("properties", "metrics", "properties", metric_name + "2"), 80 metric1, 81 ) 82 83 return schema 84 85 def get_dependencies(self): 86 # Get all of the library dependencies for the application that 87 # are also known about in the repositories file. 88 89 # The dependencies are specified using library names, but we need to 90 # map those back to the name of the repository in the repository file. 91 try: 92 dependencies = self._get_json( 93 self.dependencies_url_template.format(self.repo_name) 94 ) 95 except HTTPError: 96 logging.info(f"For {self.repo_name}, using default Glean dependencies") 97 return self.default_dependencies 98 99 dependency_library_names = list(dependencies.keys()) 100 101 repos = GleanPing._get_json(GleanPing.repos_url) 102 repos_by_dependency_name = {} 103 for repo in repos: 104 for library_name in repo.get("library_names", []): 105 repos_by_dependency_name[library_name] = repo["name"] 106 107 dependencies = [] 108 for name in dependency_library_names: 109 if name in repos_by_dependency_name: 110 dependencies.append(repos_by_dependency_name[name]) 111 112 if len(dependencies) == 0: 113 logging.info(f"For {self.repo_name}, using default Glean dependencies") 114 return self.default_dependencies 115 116 logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}") 117 return dependencies 118 119 def get_probes(self) -> List[GleanProbe]: 120 data = self._get_json(self.probes_url) 121 probes = list(data.items()) 122 123 for dependency in self.get_dependencies(): 124 dependency_probes = self._get_json( 125 self.probes_url_template.format(dependency) 126 ) 127 probes += list(dependency_probes.items()) 128 129 pings = self.get_pings() 130 131 processed = [] 132 for _id, defn in probes: 133 probe = GleanProbe(_id, defn, pings=pings) 134 processed.append(probe) 135 136 # Manual handling of incompatible schema changes 137 issue_118_affected = { 138 "fenix", 139 "fenix-nightly", 140 "firefox-android-nightly", 141 "firefox-android-beta", 142 "firefox-android-release", 143 } 144 if ( 145 self.repo_name in issue_118_affected 146 and probe.get_name() == "installation.timestamp" 147 ): 148 logging.info(f"Writing column {probe.get_name()} for compatibility.") 149 # See: https://github.com/mozilla/mozilla-schema-generator/issues/118 150 # Search through history for the "string" type and add a copy of 151 # the probe at that time in history. The changepoint signifies 152 # this event. 153 changepoint_index = 0 154 for definition in probe.definition_history: 155 if definition["type"] != probe.get_type(): 156 break 157 changepoint_index += 1 158 # Modify the definition with the truncated history. 159 hist_defn = defn.copy() 160 hist_defn[probe.history_key] = probe.definition_history[ 161 changepoint_index: 162 ] 163 hist_defn["type"] = hist_defn[probe.history_key][0]["type"] 164 incompatible_probe_type = GleanProbe(_id, hist_defn, pings=pings) 165 processed.append(incompatible_probe_type) 166 167 # Handling probe type changes (Bug 1870317) 168 probe_types = {hist["type"] for hist in defn[probe.history_key]} 169 if len(probe_types) > 1: 170 # The probe type changed at some point in history. 171 # Create schema entry for each type. 172 hist_defn = defn.copy() 173 174 # No new entry needs to be created for the current probe type 175 probe_types.remove(defn["type"]) 176 177 for hist in hist_defn[probe.history_key]: 178 # Create a new entry for a historic type 179 if hist["type"] in probe_types: 180 hist_defn["type"] = hist["type"] 181 probe = GleanProbe(_id, hist_defn, pings=pings) 182 processed.append(probe) 183 184 # Keep track of the types entries were already created for 185 probe_types.remove(hist["type"]) 186 187 return processed 188 189 def _get_ping_data(self) -> Dict[str, Dict]: 190 url = self.ping_url_template.format(self.repo_name) 191 ping_data = GleanPing._get_json(url) 192 for dependency in self.get_dependencies(): 193 dependency_pings = self._get_json(self.ping_url_template.format(dependency)) 194 ping_data.update(dependency_pings) 195 return ping_data 196 197 def _get_ping_data_without_dependencies(self) -> Dict[str, Dict]: 198 url = self.ping_url_template.format(self.repo_name) 199 ping_data = GleanPing._get_json(url) 200 return ping_data 201 202 def _get_dependency_pings(self, dependency): 203 return self._get_json(self.ping_url_template.format(dependency)) 204 205 def get_pings(self) -> Set[str]: 206 return self._get_ping_data().keys() 207 208 @staticmethod 209 def apply_default_metadata(ping_metadata, default_metadata): 210 """apply_default_metadata recurses down into dicts nested 211 to an arbitrary depth, updating keys. The ``default_metadata`` is merged into 212 ``ping_metadata``. 213 :param ping_metadata: dict onto which the merge is executed 214 :param default_metadata: dct merged into ping_metadata 215 :return: None 216 """ 217 for k, v in default_metadata.items(): 218 if ( 219 k in ping_metadata 220 and isinstance(ping_metadata[k], dict) 221 and isinstance(default_metadata[k], dict) 222 ): 223 GleanPing.apply_default_metadata(ping_metadata[k], default_metadata[k]) 224 else: 225 ping_metadata[k] = default_metadata[k] 226 227 def _get_ping_data_and_dependencies_with_default_metadata(self) -> Dict[str, Dict]: 228 # Get the ping data with the pipeline metadata 229 ping_data = self._get_ping_data_without_dependencies() 230 231 # The ping endpoint for the dependency pings does not include any repo defined 232 # moz_pipeline_metadata_defaults so they need to be applied here. 233 234 # 1. Get repo and pipeline default metadata. 235 repos = GleanPing.get_repos() 236 current_repo = next((x for x in repos if x.get("app_id") == self.app_id), {}) 237 default_metadata = current_repo.get("moz_pipeline_metadata_defaults", {}) 238 239 # 2. Apply the default metadata to each dependency defined ping. 240 for dependency in self.get_dependencies(): 241 dependency_pings = self._get_dependency_pings(dependency) 242 for dependency_ping in dependency_pings.values(): 243 # Although it is counter intuitive to apply the default metadata on top of the 244 # existing dependency ping metadata it does set the repo specific value for 245 # bq_dataset_family instead of using the dependency id for the bq_dataset_family 246 # value. 247 GleanPing.apply_default_metadata( 248 dependency_ping.get("moz_pipeline_metadata"), default_metadata 249 ) 250 ping_data.update(dependency_pings) 251 return ping_data 252 253 @staticmethod 254 def reorder_metadata(metadata): 255 desired_order_list = [ 256 "bq_dataset_family", 257 "bq_table", 258 "bq_metadata_format", 259 "include_info_sections", 260 "submission_timestamp_granularity", 261 "expiration_policy", 262 "override_attributes", 263 "jwe_mappings", 264 ] 265 reordered_metadata = { 266 k: metadata[k] for k in desired_order_list if k in metadata 267 } 268 269 # re-order jwe-mappings 270 desired_order_list = ["source_field_path", "decrypted_field_path"] 271 jwe_mapping_metadata = reordered_metadata.get("jwe_mappings") 272 if jwe_mapping_metadata: 273 reordered_jwe_mapping_metadata = [] 274 for mapping in jwe_mapping_metadata: 275 reordered_jwe_mapping_metadata.append( 276 {k: mapping[k] for k in desired_order_list if k in mapping} 277 ) 278 reordered_metadata["jwe_mappings"] = reordered_jwe_mapping_metadata 279 280 # future proofing, in case there are other fields added at the ping top level 281 # add them to the end. 282 leftovers = {k: metadata[k] for k in set(metadata) - set(reordered_metadata)} 283 reordered_metadata = {**reordered_metadata, **leftovers} 284 return reordered_metadata 285 286 def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]: 287 pings = self._get_ping_data_and_dependencies_with_default_metadata() 288 for ping_name, ping_data in pings.items(): 289 metadata = ping_data.get("moz_pipeline_metadata") 290 metadata["include_info_sections"] = self._include_info_sections(ping_data) 291 292 # While technically unnecessary, the dictionary elements are re-ordered to match the 293 # currently deployed order and used to verify no difference in output. 294 pings[ping_name] = GleanPing.reorder_metadata(metadata) 295 return pings 296 297 def get_ping_descriptions(self) -> Dict[str, str]: 298 return { 299 k: v["history"][-1]["description"] for k, v in self._get_ping_data().items() 300 } 301 302 def _include_info_sections(self, ping_data) -> bool: 303 # Default to true if not specified. 304 if "history" not in ping_data or len(ping_data["history"]) == 0: 305 return True 306 latest_ping_data = ping_data["history"][-1] 307 return ( 308 "include_info_sections" not in latest_ping_data 309 or latest_ping_data["include_info_sections"] 310 ) 311 312 def set_schema_url(self, metadata): 313 """ 314 Switch between the glean-min and glean schemas if the ping does not require 315 info sections as specified in the parsed ping info in probe scraper. 316 """ 317 if not metadata["include_info_sections"]: 318 self.schema_url = MINIMUM_SCHEMA_URL.format(branch=self.branch_name) 319 else: 320 self.schema_url = DEFAULT_SCHEMA_URL.format(branch=self.branch_name) 321 322 def generate_schema(self, config, generic_schema=False) -> Dict[str, Schema]: 323 pings = self.get_pings_and_pipeline_metadata() 324 schemas = {} 325 326 for ping, pipeline_meta in pings.items(): 327 matchers = { 328 loc: m.clone(new_table_group=ping) for loc, m in config.matchers.items() 329 } 330 331 # Four newly introduced metric types were incorrectly deployed 332 # as repeated key/value structs in all Glean ping tables existing prior 333 # to November 2021. We maintain the incorrect fields for existing tables 334 # by disabling the associated matchers. 335 # Note that each of these types now has a "2" matcher ("text2", "url2", etc.) 336 # defined that will allow metrics of these types to be injected into proper 337 # structs. The gcp-ingestion repository includes logic to rewrite these 338 # metrics under the "2" names. 339 # See https://bugzilla.mozilla.org/show_bug.cgi?id=1737656 340 bq_identifier = "{bq_dataset_family}.{bq_table}".format(**pipeline_meta) 341 if bq_identifier in self.bug_1737656_affected_tables: 342 matchers = { 343 loc: m 344 for loc, m in matchers.items() 345 if not m.matcher.get("bug_1737656_affected") 346 } 347 348 for matcher in matchers.values(): 349 matcher.matcher["send_in_pings"]["contains"] = ping 350 new_config = Config(ping, matchers=matchers) 351 352 defaults = {"mozPipelineMetadata": pipeline_meta} 353 354 # Adjust the schema path if the ping does not require info sections 355 self.set_schema_url(pipeline_meta) 356 if generic_schema: # Use the generic glean ping schema 357 schema = self.get_schema(generic_schema=True) 358 schema.schema.update(defaults) 359 schemas[new_config.name] = schema 360 else: 361 generated = super().generate_schema(new_config) 362 for schema in generated.values(): 363 # We want to override each individual key with assembled defaults, 364 # but keep values _inside_ them if they have been set in the schemas. 365 for key, value in defaults.items(): 366 if key not in schema.schema: 367 schema.schema[key] = {} 368 schema.schema[key].update(value) 369 schemas.update(generated) 370 371 return schemas 372 373 @staticmethod 374 def get_repos(): 375 """ 376 Retrieve metadata for all non-library Glean repositories 377 """ 378 repos = GleanPing._get_json(GleanPing.repos_url) 379 return [repo for repo in repos if "library_names" not in repo]
61 def get_schema(self, generic_schema=False) -> Schema: 62 """ 63 Fetch schema via URL. 64 65 Unless *generic_schema* is set to true, this function makes some modifications 66 to allow some workarounds for proper injection of metrics. 67 """ 68 schema = super().get_schema() 69 if generic_schema: 70 return schema 71 72 # We need to inject placeholders for the url2, text2, etc. types as part 73 # of mitigation for https://bugzilla.mozilla.org/show_bug.cgi?id=1737656 74 for metric_name in ["labeled_rate", "jwe", "url", "text"]: 75 metric1 = schema.get( 76 ("properties", "metrics", "properties", metric_name) 77 ).copy() 78 metric1 = schema.set_schema_elem( 79 ("properties", "metrics", "properties", metric_name + "2"), 80 metric1, 81 ) 82 83 return schema
Fetch schema via URL.
Unless generic_schema is set to true, this function makes some modifications to allow some workarounds for proper injection of metrics.
def
get_dependencies(self):
85 def get_dependencies(self): 86 # Get all of the library dependencies for the application that 87 # are also known about in the repositories file. 88 89 # The dependencies are specified using library names, but we need to 90 # map those back to the name of the repository in the repository file. 91 try: 92 dependencies = self._get_json( 93 self.dependencies_url_template.format(self.repo_name) 94 ) 95 except HTTPError: 96 logging.info(f"For {self.repo_name}, using default Glean dependencies") 97 return self.default_dependencies 98 99 dependency_library_names = list(dependencies.keys()) 100 101 repos = GleanPing._get_json(GleanPing.repos_url) 102 repos_by_dependency_name = {} 103 for repo in repos: 104 for library_name in repo.get("library_names", []): 105 repos_by_dependency_name[library_name] = repo["name"] 106 107 dependencies = [] 108 for name in dependency_library_names: 109 if name in repos_by_dependency_name: 110 dependencies.append(repos_by_dependency_name[name]) 111 112 if len(dependencies) == 0: 113 logging.info(f"For {self.repo_name}, using default Glean dependencies") 114 return self.default_dependencies 115 116 logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}") 117 return dependencies
119 def get_probes(self) -> List[GleanProbe]: 120 data = self._get_json(self.probes_url) 121 probes = list(data.items()) 122 123 for dependency in self.get_dependencies(): 124 dependency_probes = self._get_json( 125 self.probes_url_template.format(dependency) 126 ) 127 probes += list(dependency_probes.items()) 128 129 pings = self.get_pings() 130 131 processed = [] 132 for _id, defn in probes: 133 probe = GleanProbe(_id, defn, pings=pings) 134 processed.append(probe) 135 136 # Manual handling of incompatible schema changes 137 issue_118_affected = { 138 "fenix", 139 "fenix-nightly", 140 "firefox-android-nightly", 141 "firefox-android-beta", 142 "firefox-android-release", 143 } 144 if ( 145 self.repo_name in issue_118_affected 146 and probe.get_name() == "installation.timestamp" 147 ): 148 logging.info(f"Writing column {probe.get_name()} for compatibility.") 149 # See: https://github.com/mozilla/mozilla-schema-generator/issues/118 150 # Search through history for the "string" type and add a copy of 151 # the probe at that time in history. The changepoint signifies 152 # this event. 153 changepoint_index = 0 154 for definition in probe.definition_history: 155 if definition["type"] != probe.get_type(): 156 break 157 changepoint_index += 1 158 # Modify the definition with the truncated history. 159 hist_defn = defn.copy() 160 hist_defn[probe.history_key] = probe.definition_history[ 161 changepoint_index: 162 ] 163 hist_defn["type"] = hist_defn[probe.history_key][0]["type"] 164 incompatible_probe_type = GleanProbe(_id, hist_defn, pings=pings) 165 processed.append(incompatible_probe_type) 166 167 # Handling probe type changes (Bug 1870317) 168 probe_types = {hist["type"] for hist in defn[probe.history_key]} 169 if len(probe_types) > 1: 170 # The probe type changed at some point in history. 171 # Create schema entry for each type. 172 hist_defn = defn.copy() 173 174 # No new entry needs to be created for the current probe type 175 probe_types.remove(defn["type"]) 176 177 for hist in hist_defn[probe.history_key]: 178 # Create a new entry for a historic type 179 if hist["type"] in probe_types: 180 hist_defn["type"] = hist["type"] 181 probe = GleanProbe(_id, hist_defn, pings=pings) 182 processed.append(probe) 183 184 # Keep track of the types entries were already created for 185 probe_types.remove(hist["type"]) 186 187 return processed
@staticmethod
def
apply_default_metadata(ping_metadata, default_metadata):
208 @staticmethod 209 def apply_default_metadata(ping_metadata, default_metadata): 210 """apply_default_metadata recurses down into dicts nested 211 to an arbitrary depth, updating keys. The ``default_metadata`` is merged into 212 ``ping_metadata``. 213 :param ping_metadata: dict onto which the merge is executed 214 :param default_metadata: dct merged into ping_metadata 215 :return: None 216 """ 217 for k, v in default_metadata.items(): 218 if ( 219 k in ping_metadata 220 and isinstance(ping_metadata[k], dict) 221 and isinstance(default_metadata[k], dict) 222 ): 223 GleanPing.apply_default_metadata(ping_metadata[k], default_metadata[k]) 224 else: 225 ping_metadata[k] = default_metadata[k]
apply_default_metadata recurses down into dicts nested
to an arbitrary depth, updating keys. The default_metadata is merged into
ping_metadata.
Parameters
- ping_metadata: dict onto which the merge is executed
- default_metadata: dct merged into ping_metadata
Returns
None
@staticmethod
def
reorder_metadata(metadata):
253 @staticmethod 254 def reorder_metadata(metadata): 255 desired_order_list = [ 256 "bq_dataset_family", 257 "bq_table", 258 "bq_metadata_format", 259 "include_info_sections", 260 "submission_timestamp_granularity", 261 "expiration_policy", 262 "override_attributes", 263 "jwe_mappings", 264 ] 265 reordered_metadata = { 266 k: metadata[k] for k in desired_order_list if k in metadata 267 } 268 269 # re-order jwe-mappings 270 desired_order_list = ["source_field_path", "decrypted_field_path"] 271 jwe_mapping_metadata = reordered_metadata.get("jwe_mappings") 272 if jwe_mapping_metadata: 273 reordered_jwe_mapping_metadata = [] 274 for mapping in jwe_mapping_metadata: 275 reordered_jwe_mapping_metadata.append( 276 {k: mapping[k] for k in desired_order_list if k in mapping} 277 ) 278 reordered_metadata["jwe_mappings"] = reordered_jwe_mapping_metadata 279 280 # future proofing, in case there are other fields added at the ping top level 281 # add them to the end. 282 leftovers = {k: metadata[k] for k in set(metadata) - set(reordered_metadata)} 283 reordered_metadata = {**reordered_metadata, **leftovers} 284 return reordered_metadata
def
get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]:
286 def get_pings_and_pipeline_metadata(self) -> Dict[str, Dict]: 287 pings = self._get_ping_data_and_dependencies_with_default_metadata() 288 for ping_name, ping_data in pings.items(): 289 metadata = ping_data.get("moz_pipeline_metadata") 290 metadata["include_info_sections"] = self._include_info_sections(ping_data) 291 292 # While technically unnecessary, the dictionary elements are re-ordered to match the 293 # currently deployed order and used to verify no difference in output. 294 pings[ping_name] = GleanPing.reorder_metadata(metadata) 295 return pings
def
set_schema_url(self, metadata):
312 def set_schema_url(self, metadata): 313 """ 314 Switch between the glean-min and glean schemas if the ping does not require 315 info sections as specified in the parsed ping info in probe scraper. 316 """ 317 if not metadata["include_info_sections"]: 318 self.schema_url = MINIMUM_SCHEMA_URL.format(branch=self.branch_name) 319 else: 320 self.schema_url = DEFAULT_SCHEMA_URL.format(branch=self.branch_name)
Switch between the glean-min and glean schemas if the ping does not require info sections as specified in the parsed ping info in probe scraper.
def
generate_schema( self, config, generic_schema=False) -> Dict[str, mozilla_schema_generator.schema.Schema]:
322 def generate_schema(self, config, generic_schema=False) -> Dict[str, Schema]: 323 pings = self.get_pings_and_pipeline_metadata() 324 schemas = {} 325 326 for ping, pipeline_meta in pings.items(): 327 matchers = { 328 loc: m.clone(new_table_group=ping) for loc, m in config.matchers.items() 329 } 330 331 # Four newly introduced metric types were incorrectly deployed 332 # as repeated key/value structs in all Glean ping tables existing prior 333 # to November 2021. We maintain the incorrect fields for existing tables 334 # by disabling the associated matchers. 335 # Note that each of these types now has a "2" matcher ("text2", "url2", etc.) 336 # defined that will allow metrics of these types to be injected into proper 337 # structs. The gcp-ingestion repository includes logic to rewrite these 338 # metrics under the "2" names. 339 # See https://bugzilla.mozilla.org/show_bug.cgi?id=1737656 340 bq_identifier = "{bq_dataset_family}.{bq_table}".format(**pipeline_meta) 341 if bq_identifier in self.bug_1737656_affected_tables: 342 matchers = { 343 loc: m 344 for loc, m in matchers.items() 345 if not m.matcher.get("bug_1737656_affected") 346 } 347 348 for matcher in matchers.values(): 349 matcher.matcher["send_in_pings"]["contains"] = ping 350 new_config = Config(ping, matchers=matchers) 351 352 defaults = {"mozPipelineMetadata": pipeline_meta} 353 354 # Adjust the schema path if the ping does not require info sections 355 self.set_schema_url(pipeline_meta) 356 if generic_schema: # Use the generic glean ping schema 357 schema = self.get_schema(generic_schema=True) 358 schema.schema.update(defaults) 359 schemas[new_config.name] = schema 360 else: 361 generated = super().generate_schema(new_config) 362 for schema in generated.values(): 363 # We want to override each individual key with assembled defaults, 364 # but keep values _inside_ them if they have been set in the schemas. 365 for key, value in defaults.items(): 366 if key not in schema.schema: 367 schema.schema[key] = {} 368 schema.schema[key].update(value) 369 schemas.update(generated) 370 371 return schemas
@staticmethod
def
get_repos():
373 @staticmethod 374 def get_repos(): 375 """ 376 Retrieve metadata for all non-library Glean repositories 377 """ 378 repos = GleanPing._get_json(GleanPing.repos_url) 379 return [repo for repo in repos if "library_names" not in repo]
Retrieve metadata for all non-library Glean repositories
f =
<_io.TextIOWrapper name='/home/circleci/project/mozilla_schema_generator/configs/bug_1737656_affected.txt' mode='r' encoding='UTF-8'>
bug_1737656_affected_tables =
['burnham.baseline_v1', 'burnham.deletion_request_v1', 'burnham.discovery_v1', 'burnham.events_v1', 'burnham.metrics_v1', 'burnham.space_ship_ready_v1', 'burnham.starbase46_v1', 'firefox_desktop_background_update.background_update_v1', 'firefox_desktop_background_update.baseline_v1', 'firefox_desktop_background_update.deletion_request_v1', 'firefox_desktop_background_update.events_v1', 'firefox_desktop_background_update.metrics_v1', 'firefox_desktop.baseline_v1', 'firefox_desktop.deletion_request_v1', 'firefox_desktop.events_v1', 'firefox_desktop.fog_validation_v1', 'firefox_desktop.metrics_v1', 'firefox_installer.install_v1', 'firefox_launcher_process.launcher_process_failure_v1', 'messaging_system.cfr_v1', 'messaging_system.infobar_v1', 'messaging_system.moments_v1', 'messaging_system.onboarding_v1', 'messaging_system.personalization_experiment_v1', 'messaging_system.snippets_v1', 'messaging_system.spotlight_v1', 'messaging_system.undesired_events_v1', 'messaging_system.whats_new_panel_v1', 'mlhackweek_search.action_v1', 'mlhackweek_search.baseline_v1', 'mlhackweek_search.custom_v1', 'mlhackweek_search.deletion_request_v1', 'mlhackweek_search.events_v1', 'mlhackweek_search.metrics_v1', 'mozilla_lockbox.addresses_sync_v1', 'mozilla_lockbox.baseline_v1', 'mozilla_lockbox.bookmarks_sync_v1', 'mozilla_lockbox.creditcards_sync_v1', 'mozilla_lockbox.deletion_request_v1', 'mozilla_lockbox.events_v1', 'mozilla_lockbox.history_sync_v1', 'mozilla_lockbox.logins_sync_v1', 'mozilla_lockbox.metrics_v1', 'mozilla_lockbox.sync_v1', 'mozilla_lockbox.tabs_sync_v1', 'mozilla_mach.baseline_v1', 'mozilla_mach.deletion_request_v1', 'mozilla_mach.events_v1', 'mozilla_mach.metrics_v1', 'mozilla_mach.usage_v1', 'mozillavpn.deletion_request_v1', 'mozillavpn.main_v1', 'mozphab.baseline_v1', 'mozphab.deletion_request_v1', 'mozphab.events_v1', 'mozphab.metrics_v1', 'mozphab.usage_v1', 'org_mozilla_bergamot.custom_v1', 'org_mozilla_bergamot.deletion_request_v1', 'org_mozilla_connect_firefox.baseline_v1', 'org_mozilla_connect_firefox.deletion_request_v1', 'org_mozilla_connect_firefox.events_v1', 'org_mozilla_connect_firefox.metrics_v1', 'org_mozilla_fenix.activation_v1', 'org_mozilla_fenix.addresses_sync_v1', 'org_mozilla_fenix.baseline_v1', 'org_mozilla_fenix.bookmarks_sync_v1', 'org_mozilla_fenix.creditcards_sync_v1', 'org_mozilla_fenix.deletion_request_v1', 'org_mozilla_fenix.events_v1', 'org_mozilla_fenix.first_session_v1', 'org_mozilla_fenix.fog_validation_v1', 'org_mozilla_fenix.history_sync_v1', 'org_mozilla_fenix.installation_v1', 'org_mozilla_fenix.logins_sync_v1', 'org_mozilla_fenix.metrics_v1', 'org_mozilla_fenix.migration_v1', 'org_mozilla_fenix.startup_timeline_v1', 'org_mozilla_fenix.sync_v1', 'org_mozilla_fenix.tabs_sync_v1', 'org_mozilla_fenix_nightly.activation_v1', 'org_mozilla_fenix_nightly.addresses_sync_v1', 'org_mozilla_fenix_nightly.baseline_v1', 'org_mozilla_fenix_nightly.bookmarks_sync_v1', 'org_mozilla_fenix_nightly.creditcards_sync_v1', 'org_mozilla_fenix_nightly.deletion_request_v1', 'org_mozilla_fenix_nightly.events_v1', 'org_mozilla_fenix_nightly.first_session_v1', 'org_mozilla_fenix_nightly.fog_validation_v1', 'org_mozilla_fenix_nightly.history_sync_v1', 'org_mozilla_fenix_nightly.installation_v1', 'org_mozilla_fenix_nightly.logins_sync_v1', 'org_mozilla_fenix_nightly.metrics_v1', 'org_mozilla_fenix_nightly.migration_v1', 'org_mozilla_fenix_nightly.startup_timeline_v1', 'org_mozilla_fenix_nightly.sync_v1', 'org_mozilla_fenix_nightly.tabs_sync_v1', 'org_mozilla_fennec_aurora.activation_v1', 'org_mozilla_fennec_aurora.addresses_sync_v1', 'org_mozilla_fennec_aurora.baseline_v1', 'org_mozilla_fennec_aurora.bookmarks_sync_v1', 'org_mozilla_fennec_aurora.creditcards_sync_v1', 'org_mozilla_fennec_aurora.deletion_request_v1', 'org_mozilla_fennec_aurora.events_v1', 'org_mozilla_fennec_aurora.first_session_v1', 'org_mozilla_fennec_aurora.fog_validation_v1', 'org_mozilla_fennec_aurora.history_sync_v1', 'org_mozilla_fennec_aurora.installation_v1', 'org_mozilla_fennec_aurora.logins_sync_v1', 'org_mozilla_fennec_aurora.metrics_v1', 'org_mozilla_fennec_aurora.migration_v1', 'org_mozilla_fennec_aurora.startup_timeline_v1', 'org_mozilla_fennec_aurora.sync_v1', 'org_mozilla_fennec_aurora.tabs_sync_v1', 'org_mozilla_firefox_beta.activation_v1', 'org_mozilla_firefox_beta.addresses_sync_v1', 'org_mozilla_firefox_beta.baseline_v1', 'org_mozilla_firefox_beta.bookmarks_sync_v1', 'org_mozilla_firefox_beta.creditcards_sync_v1', 'org_mozilla_firefox_beta.deletion_request_v1', 'org_mozilla_firefox_beta.events_v1', 'org_mozilla_firefox_beta.first_session_v1', 'org_mozilla_firefox_beta.fog_validation_v1', 'org_mozilla_firefox_beta.history_sync_v1', 'org_mozilla_firefox_beta.installation_v1', 'org_mozilla_firefox_beta.logins_sync_v1', 'org_mozilla_firefox_beta.metrics_v1', 'org_mozilla_firefox_beta.migration_v1', 'org_mozilla_firefox_beta.startup_timeline_v1', 'org_mozilla_firefox_beta.sync_v1', 'org_mozilla_firefox_beta.tabs_sync_v1', 'org_mozilla_firefox.activation_v1', 'org_mozilla_firefox.addresses_sync_v1', 'org_mozilla_firefox.baseline_v1', 'org_mozilla_firefox.bookmarks_sync_v1', 'org_mozilla_firefox.creditcards_sync_v1', 'org_mozilla_firefox.deletion_request_v1', 'org_mozilla_firefox.events_v1', 'org_mozilla_firefox.first_session_v1', 'org_mozilla_firefox.fog_validation_v1', 'org_mozilla_firefox.history_sync_v1', 'org_mozilla_firefox.installation_v1', 'org_mozilla_firefox.logins_sync_v1', 'org_mozilla_firefox.metrics_v1', 'org_mozilla_firefox.migration_v1', 'org_mozilla_firefox.startup_timeline_v1', 'org_mozilla_firefox.sync_v1', 'org_mozilla_firefox.tabs_sync_v1', 'org_mozilla_firefoxreality.baseline_v1', 'org_mozilla_firefoxreality.deletion_request_v1', 'org_mozilla_firefoxreality.events_v1', 'org_mozilla_firefoxreality.launch_v1', 'org_mozilla_firefoxreality.metrics_v1', 'org_mozilla_focus_beta.activation_v1', 'org_mozilla_focus_beta.baseline_v1', 'org_mozilla_focus_beta.deletion_request_v1', 'org_mozilla_focus_beta.events_v1', 'org_mozilla_focus_beta.metrics_v1', 'org_mozilla_focus.activation_v1', 'org_mozilla_focus.baseline_v1', 'org_mozilla_focus.deletion_request_v1', 'org_mozilla_focus.events_v1', 'org_mozilla_focus.metrics_v1', 'org_mozilla_focus_nightly.activation_v1', 'org_mozilla_focus_nightly.baseline_v1', 'org_mozilla_focus_nightly.deletion_request_v1', 'org_mozilla_focus_nightly.events_v1', 'org_mozilla_focus_nightly.metrics_v1', 'org_mozilla_ios_fennec.baseline_v1', 'org_mozilla_ios_fennec.deletion_request_v1', 'org_mozilla_ios_fennec.events_v1', 'org_mozilla_ios_fennec.metrics_v1', 'org_mozilla_ios_firefox.baseline_v1', 'org_mozilla_ios_firefox.deletion_request_v1', 'org_mozilla_ios_firefox.events_v1', 'org_mozilla_ios_firefox.metrics_v1', 'org_mozilla_ios_firefoxbeta.baseline_v1', 'org_mozilla_ios_firefoxbeta.deletion_request_v1', 'org_mozilla_ios_firefoxbeta.events_v1', 'org_mozilla_ios_firefoxbeta.metrics_v1', 'org_mozilla_ios_focus.baseline_v1', 'org_mozilla_ios_focus.deletion_request_v1', 'org_mozilla_ios_focus.events_v1', 'org_mozilla_ios_focus.metrics_v1', 'org_mozilla_ios_klar.baseline_v1', 'org_mozilla_ios_klar.deletion_request_v1', 'org_mozilla_ios_klar.events_v1', 'org_mozilla_ios_klar.metrics_v1', 'org_mozilla_ios_lockbox.baseline_v1', 'org_mozilla_ios_lockbox.deletion_request_v1', 'org_mozilla_ios_lockbox.events_v1', 'org_mozilla_ios_lockbox.metrics_v1', 'org_mozilla_klar.activation_v1', 'org_mozilla_klar.baseline_v1', 'org_mozilla_klar.deletion_request_v1', 'org_mozilla_klar.events_v1', 'org_mozilla_klar.metrics_v1', 'org_mozilla_mozregression.baseline_v1', 'org_mozilla_mozregression.deletion_request_v1', 'org_mozilla_mozregression.events_v1', 'org_mozilla_mozregression.metrics_v1', 'org_mozilla_mozregression.usage_v1', 'org_mozilla_reference_browser.baseline_v1', 'org_mozilla_reference_browser.deletion_request_v1', 'org_mozilla_reference_browser.events_v1', 'org_mozilla_reference_browser.metrics_v1', 'org_mozilla_tv_firefox.baseline_v1', 'org_mozilla_tv_firefox.deletion_request_v1', 'org_mozilla_tv_firefox.events_v1', 'org_mozilla_tv_firefox.metrics_v1', 'org_mozilla_vrbrowser.addresses_sync_v1', 'org_mozilla_vrbrowser.baseline_v1', 'org_mozilla_vrbrowser.bookmarks_sync_v1', 'org_mozilla_vrbrowser.creditcards_sync_v1', 'org_mozilla_vrbrowser.deletion_request_v1', 'org_mozilla_vrbrowser.events_v1', 'org_mozilla_vrbrowser.history_sync_v1', 'org_mozilla_vrbrowser.logins_sync_v1', 'org_mozilla_vrbrowser.metrics_v1', 'org_mozilla_vrbrowser.session_end_v1', 'org_mozilla_vrbrowser.sync_v1', 'org_mozilla_vrbrowser.tabs_sync_v1', 'rally_core.deletion_request_v1', 'rally_core.demographics_v1', 'rally_core.enrollment_v1', 'rally_core.study_enrollment_v1', 'rally_core.study_unenrollment_v1', 'rally_core.uninstall_deletion_v1', 'rally_debug.deletion_request_v1', 'rally_debug.demographics_v1', 'rally_debug.enrollment_v1', 'rally_debug.study_enrollment_v1', 'rally_debug.study_unenrollment_v1', 'rally_debug.uninstall_deletion_v1', 'rally_study_zero_one.deletion_request_v1', 'rally_study_zero_one.rs01_event_v1', 'rally_study_zero_one.study_enrollment_v1', 'rally_zero_one.deletion_request_v1', 'rally_zero_one.measurements_v1', 'rally_zero_one.pioneer_enrollment_v1']