generator.views.datagroups

  1"""Generate datagroup lkml files for each namespace."""
  2
  3import logging
  4from dataclasses import dataclass
  5from pathlib import Path
  6from typing import Any, Dict, Optional
  7
  8import lkml
  9
 10from generator.dryrun import DryRunError, Errors
 11from generator.namespaces import DEFAULT_GENERATED_SQL_URI
 12from generator.utils import get_file_from_looker_hub
 13from generator.views import TableView, View, lookml_utils
 14from generator.views.lookml_utils import BQViewReferenceMap
 15
 16DEFAULT_MAX_CACHE_AGE = "24 hours"
 17
 18SQL_TRIGGER_TEMPLATE = """
 19    SELECT MAX(storage_last_modified_time)
 20    FROM `{project_id}`.`region-us`.INFORMATION_SCHEMA.TABLE_STORAGE
 21    WHERE table_schema = '{dataset_id}'
 22    AND table_name = '{table_id}'
 23"""
 24
 25# To map views to their underlying tables:
 26DATASET_VIEW_MAP = lookml_utils.get_bigquery_view_reference_map(
 27    DEFAULT_GENERATED_SQL_URI
 28)
 29
 30FILE_HEADER = """# *Do not manually modify this file*
 31
 32# This file has been generated via https://github.com/mozilla/lookml-generator
 33
 34# Using a datagroup in an Explore: https://cloud.google.com/looker/docs/reference/param-explore-persist-with
 35# Using a datagroup in a derived table: https://cloud.google.com/looker/docs/reference/param-view-datagroup-trigger
 36
 37"""
 38
 39
 40@dataclass(frozen=True, eq=True)
 41class Datagroup:
 42    """Represents a Datagroup."""
 43
 44    name: str
 45    label: str
 46    sql_trigger: str
 47    description: str
 48    max_cache_age: str = DEFAULT_MAX_CACHE_AGE
 49
 50    def __str__(self) -> str:
 51        """Return the LookML string representation of a Datagroup."""
 52        return lkml.dump({"datagroups": [self.__dict__]})  # type: ignore
 53
 54    def __lt__(self, other) -> bool:
 55        """Make datagroups sortable."""
 56        return self.name < other.name
 57
 58
 59def _get_datagroup_from_bigquery_table(
 60    project_id, dataset_id, table_id, table_metadata: Dict[str, Any]
 61) -> Datagroup:
 62    """Use template and default values to create a Datagroup from a BQ Table."""
 63    full_table_id = f"{project_id}.{dataset_id}.{table_id}"
 64    return Datagroup(
 65        name=f"{table_id}_last_updated",
 66        label=f"{table_metadata.get('friendlyName', table_id) or table_id} Last Updated",
 67        description=f"Updates when {full_table_id} is modified.",
 68        sql_trigger=SQL_TRIGGER_TEMPLATE.format(
 69            project_id=project_id,
 70            dataset_id=dataset_id,
 71            table_id=table_id,
 72        ),
 73        # Note: Ideally we'd use the table's ETL schedule as the `maximum_cache_age` but we don't have schedule
 74        # metadata here.
 75    )
 76
 77
 78def _get_datagroup_from_bigquery_view(
 79    project_id,
 80    dataset_id,
 81    table_id,
 82    table_metadata: Dict[str, Any],
 83    dataset_view_map: BQViewReferenceMap,
 84    dryrun,
 85) -> Optional[Datagroup]:
 86    # Dataset view map only contains references for shared-prod views.
 87    full_table_id = f"{project_id}.{dataset_id}.{table_id}"
 88    if project_id not in ("moz-fx-data-shared-prod", "mozdata"):
 89        logging.debug(
 90            f"Unable to get sources for non shared-prod/mozdata view: {full_table_id} in generated-sql."
 91        )
 92        return None
 93
 94    dataset_view_references = dataset_view_map.get(dataset_id)
 95    if dataset_view_references is None:
 96        logging.debug(f"Unable to find dataset {dataset_id} in generated-sql.")
 97        return None
 98
 99    view_references = dataset_view_references.get(table_id)
100    if not view_references or len(view_references) > 1:
101        # For views with multiple sources it's unclear which table to check for updates.
102        logging.debug(
103            f"Unable to find a single source for {full_table_id} in generated-sql."
104        )
105        return None
106
107    source_table_id = ".".join(view_references[0])
108    try:
109        table_metadata = dryrun.create(
110            project=view_references[0][0],
111            dataset=view_references[0][1],
112            table=view_references[0][2],
113        ).get_table_metadata()
114        if "TABLE" == table_metadata.get("tableType"):
115            return _get_datagroup_from_bigquery_table(
116                view_references[0][0],
117                view_references[0][1],
118                view_references[0][2],
119                table_metadata,
120            )
121        elif "VIEW" == table_metadata.get("tableType"):
122            return _get_datagroup_from_bigquery_view(
123                view_references[0][0],
124                view_references[0][1],
125                view_references[0][2],
126                table_metadata,
127                dataset_view_map,
128                dryrun=dryrun,
129            )
130    except DryRunError as e:
131        raise ValueError(
132            f"Unable to find {source_table_id} referenced in {full_table_id}"
133        ) from e
134
135    return None
136
137
138def _generate_view_datagroup(
139    view: View,
140    dataset_view_map: BQViewReferenceMap,
141    dryrun,
142) -> Optional[Datagroup]:
143    """Generate the Datagroup LookML for a Looker View."""
144    # Only generate datagroup for views that can be linked to a BigQuery table:
145    if view.view_type != TableView.type:
146        return None
147
148    # Use the release channel table or the first available table (usually the only one):
149    view_table = next(
150        (table for table in view.tables if table.get("channel") == "release"),
151        view.tables[0],
152    )["table"]
153
154    [project, dataset, table] = view_table.split(".")
155    table_metadata = dryrun.create(
156        project=project,
157        dataset=dataset,
158        table=table,
159    ).get_table_metadata()
160
161    if "TABLE" == table_metadata.get("tableType"):
162        datagroup: Optional[Datagroup] = _get_datagroup_from_bigquery_table(
163            project, dataset, table, table_metadata
164        )
165        return datagroup
166    elif "VIEW" == table_metadata.get("tableType"):
167        datagroup = _get_datagroup_from_bigquery_view(
168            project,
169            dataset,
170            table,
171            table_metadata,
172            dataset_view_map,
173            dryrun,
174        )
175        return datagroup
176
177    return None
178
179
180def generate_datagroup(
181    view: View,
182    target_dir: Path,
183    namespace: str,
184    dryrun,
185) -> Optional[Path]:
186    """Generate and write a datagroups.lkml file to the namespace folder."""
187    datagroups_folder_path = target_dir / namespace / "datagroups"
188
189    datagroup = None
190    try:
191        datagroup = _generate_view_datagroup(view, DATASET_VIEW_MAP, dryrun)
192    except DryRunError as e:
193        if e.error == Errors.PERMISSION_DENIED and e.use_cloud_function:
194            path = datagroups_folder_path / f"{e.table_id}_last_updated.datagroup.lkml"
195            print(
196                f"Permission error dry running: {path}. Copy existing file from looker-hub."
197            )
198            try:
199                get_file_from_looker_hub(path)
200            except Exception as ex:
201                print(f"Skip generating datagroup for {path}: {ex}")
202        else:
203            raise
204
205    if datagroup:
206        datagroups_folder_path.mkdir(exist_ok=True)
207        datagroup_lkml_path = (
208            datagroups_folder_path / f"{datagroup.name}.datagroup.lkml"
209        )
210        datagroup_lkml_path.write_text(FILE_HEADER + str(datagroup))
211        return datagroup_lkml_path
212
213    return None