generator.lookml

  1"""Generate lookml from namespaces."""
  2
  3import logging
  4from pathlib import Path
  5from typing import Dict, Iterable, Optional
  6
  7import click
  8import lkml
  9import yaml
 10from google.cloud import bigquery
 11
 12from .dashboards import DASHBOARD_TYPES
 13from .explores import EXPLORE_TYPES
 14from .metrics_utils import LOOKER_METRIC_HUB_REPO, METRIC_HUB_REPO, MetricsConfigLoader
 15from .namespaces import _get_glean_apps
 16from .views import VIEW_TYPES, View, ViewDict
 17from .views.datagroups import generate_datagroups
 18
 19FILE_HEADER = """
 20# *Do not manually modify this file*
 21#
 22# This file has been generated via https://github.com/mozilla/lookml-generator
 23# You can extend this view in the looker-spoke-default project (https://github.com/mozilla/looker-spoke-default)
 24
 25"""
 26
 27
 28def _generate_views(
 29    client, out_dir: Path, views: Iterable[View], v1_name: Optional[str]
 30) -> Iterable[Path]:
 31    for view in views:
 32        logging.info(
 33            f"Generating lookml for view {view.name} in {view.namespace} of type {view.view_type}"
 34        )
 35        path = out_dir / f"{view.name}.view.lkml"
 36        lookml = view.to_lookml(client, v1_name)
 37        if lookml == {}:
 38            continue
 39
 40        # lkml.dump may return None, in which case write an empty file
 41        path.write_text(FILE_HEADER + (lkml.dump(lookml) or ""))
 42        yield path
 43
 44
 45def _generate_explores(
 46    client,
 47    out_dir: Path,
 48    namespace: str,
 49    explores: dict,
 50    views_dir: Path,
 51    v1_name: Optional[
 52        str
 53    ],  # v1_name for Glean explores: see: https://mozilla.github.io/probe-scraper/#tag/library
 54) -> Iterable[Path]:
 55    for explore_name, defn in explores.items():
 56        logging.info(f"Generating lookml for explore {explore_name} in {namespace}")
 57        explore = EXPLORE_TYPES[defn["type"]].from_dict(explore_name, defn, views_dir)
 58        file_lookml = {
 59            # Looker validates all included files,
 60            # so if we're not explicit about files here, validation takes
 61            # forever as looker re-validates all views for every explore (if we used *).
 62            "includes": [
 63                f"/looker-hub/{namespace}/views/{view}.view.lkml"
 64                for view in explore.get_dependent_views()
 65            ],
 66            "explores": explore.to_lookml(client, v1_name),
 67        }
 68        path = out_dir / (explore_name + ".explore.lkml")
 69        # lkml.dump may return None, in which case write an empty file
 70        path.write_text(FILE_HEADER + (lkml.dump(file_lookml) or ""))
 71        yield path
 72
 73
 74def _generate_dashboards(
 75    client,
 76    dash_dir: Path,
 77    namespace: str,
 78    dashboards: dict,
 79):
 80    for dashboard_name, dashboard_info in dashboards.items():
 81        logging.info(f"Generating lookml for dashboard {dashboard_name} in {namespace}")
 82        dashboard = DASHBOARD_TYPES[dashboard_info["type"]].from_dict(
 83            namespace, dashboard_name, dashboard_info
 84        )
 85
 86        dashboard_lookml = dashboard.to_lookml(client)
 87        dash_path = dash_dir / f"{dashboard_name}.dashboard.lookml"
 88        dash_path.write_text(FILE_HEADER + dashboard_lookml)
 89        yield dash_path
 90
 91
 92def _get_views_from_dict(views: Dict[str, ViewDict], namespace: str) -> Iterable[View]:
 93    for view_name, view_info in views.items():
 94        yield VIEW_TYPES[view_info["type"]].from_dict(  # type: ignore
 95            namespace, view_name, view_info
 96        )
 97
 98
 99def _glean_apps_to_v1_map(glean_apps):
100    return {d["name"]: d["v1_name"] for d in glean_apps}
101
102
103def _lookml(namespaces, glean_apps, target_dir, namespace_filter=[]):
104    client = bigquery.Client()
105
106    namespaces_content = namespaces.read()
107    _namespaces = yaml.safe_load(namespaces_content)
108    target = Path(target_dir)
109    target.mkdir(parents=True, exist_ok=True)
110
111    # Write namespaces file to target directory, for use
112    # by the Glean Dictionary and other tools
113    with open(target / "namespaces.yaml", "w") as target_namespaces_file:
114        target_namespaces_file.write(namespaces_content)
115
116    v1_mapping = _glean_apps_to_v1_map(glean_apps)
117    for namespace, lookml_objects in _namespaces.items():
118        if len(namespace_filter) == 0 or namespace in namespace_filter:
119            logging.info(f"\nGenerating namespace {namespace}")
120
121            view_dir = target / namespace / "views"
122            view_dir.mkdir(parents=True, exist_ok=True)
123            views = list(
124                _get_views_from_dict(lookml_objects.get("views", {}), namespace)
125            )
126
127            logging.info("  Generating views")
128            v1_name: Optional[str] = v1_mapping.get(namespace)
129            for view_path in _generate_views(client, view_dir, views, v1_name):
130                logging.info(f"    ...Generating {view_path}")
131
132            logging.info("  Generating datagroups")
133            generate_datagroups(views, target, namespace, client)
134
135            explore_dir = target / namespace / "explores"
136            explore_dir.mkdir(parents=True, exist_ok=True)
137            explores = lookml_objects.get("explores", {})
138            logging.info("  Generating explores")
139            for explore_path in _generate_explores(
140                client, explore_dir, namespace, explores, view_dir, v1_name
141            ):
142                logging.info(f"    ...Generating {explore_path}")
143
144            logging.info("  Generating dashboards")
145            dashboard_dir = target / namespace / "dashboards"
146            dashboard_dir.mkdir(parents=True, exist_ok=True)
147            dashboards = lookml_objects.get("dashboards", {})
148            for dashboard_path in _generate_dashboards(
149                client, dashboard_dir, namespace, dashboards
150            ):
151                logging.info(f"    ...Generating {dashboard_path}")
152
153
154@click.command(help=__doc__)
155@click.option(
156    "--namespaces",
157    default="namespaces.yaml",
158    type=click.File(),
159    help="Path to a yaml namespaces file",
160)
161@click.option(
162    "--app-listings-uri",
163    default="https://probeinfo.telemetry.mozilla.org/v2/glean/app-listings",
164    help="URI for probeinfo service v2 glean app listings",
165)
166@click.option(
167    "--target-dir",
168    default="looker-hub/",
169    type=click.Path(),
170    help="Path to a directory where lookml will be written",
171)
172@click.option(
173    "--metric-hub-repos",
174    "--metric-hub-repos",
175    multiple=True,
176    default=[METRIC_HUB_REPO, LOOKER_METRIC_HUB_REPO],
177    help="Repos to load metric configs from.",
178)
179@click.option(
180    "--only",
181    multiple=True,
182    default=[],
183    help="List of namespace names to generate lookml for.",
184)
185def lookml(namespaces, app_listings_uri, target_dir, metric_hub_repos, only):
186    """Generate lookml from namespaces."""
187    if metric_hub_repos:
188        MetricsConfigLoader.update_repos(metric_hub_repos)
189
190    glean_apps = _get_glean_apps(app_listings_uri)
191    return _lookml(namespaces, glean_apps, target_dir, only)