generator.views.datagroups

  1"""Generate datagroup lkml files for each namespace."""
  2
  3import logging
  4from dataclasses import dataclass
  5from pathlib import Path
  6from typing import Any, List, Optional
  7
  8import lkml
  9
 10from generator.dryrun import DryRunError, Errors
 11from generator.namespaces import DEFAULT_GENERATED_SQL_URI
 12from generator.utils import get_file_from_looker_hub
 13from generator.views import View, lookml_utils
 14from generator.views.lookml_utils import BQViewReferenceMap
 15
 16DEFAULT_MAX_CACHE_AGE = "24 hours"
 17
 18SQL_TRIGGER_TEMPLATE_SINGLE_TABLE = """
 19    SELECT MAX(storage_last_modified_time) AS storage_last_modified_time
 20    FROM `{project_id}`.`region-us`.INFORMATION_SCHEMA.TABLE_STORAGE
 21    WHERE {table}
 22"""
 23
 24SQL_TRIGGER_TEMPLATE_ALL_TABLES = """
 25    SELECT MAX(storage_last_modified_time)
 26    FROM (
 27        {tables}
 28    )
 29"""
 30
 31# To map views to their underlying tables:
 32DATASET_VIEW_MAP = lookml_utils.get_bigquery_view_reference_map(
 33    DEFAULT_GENERATED_SQL_URI
 34)
 35
 36FILE_HEADER = """# *Do not manually modify this file*
 37
 38# This file has been generated via https://github.com/mozilla/lookml-generator
 39
 40# Using a datagroup in an Explore: https://cloud.google.com/looker/docs/reference/param-explore-persist-with
 41# Using a datagroup in a derived table: https://cloud.google.com/looker/docs/reference/param-view-datagroup-trigger
 42
 43"""
 44
 45
 46@dataclass(frozen=True, eq=True)
 47class Datagroup:
 48    """Represents a Datagroup."""
 49
 50    name: str
 51    label: str
 52    sql_trigger: str
 53    description: str
 54    max_cache_age: str = DEFAULT_MAX_CACHE_AGE
 55
 56    def __str__(self) -> str:
 57        """Return the LookML string representation of a Datagroup."""
 58        return lkml.dump({"datagroups": [self.__dict__]})  # type: ignore
 59
 60    def __lt__(self, other) -> bool:
 61        """Make datagroups sortable."""
 62        return self.name < other.name
 63
 64
 65def _get_datagroup_from_bigquery_tables(
 66    project_id, tables, view: View
 67) -> Optional[Datagroup]:
 68    """Use template and default values to create a Datagroup from a BQ Table."""
 69    if len(tables) == 0:
 70        return None
 71
 72    datagroup_tables = []
 73    for table in tables:
 74        dataset_id = table[1]
 75        table_id = table[2]
 76
 77        datagroup_tables.append(
 78            SQL_TRIGGER_TEMPLATE_SINGLE_TABLE.format(
 79                project_id=table[0],
 80                table=f"(table_schema = '{dataset_id}' AND table_name = '{table_id}')",
 81            )
 82        )
 83
 84    # create a datagroup associated to a view which will be used for caching
 85    return Datagroup(
 86        name=f"{view.name}_last_updated",
 87        label=f"{view.name} Last Updated",
 88        description=f"Updates for {view.name} when referenced tables are modified.",
 89        sql_trigger=SQL_TRIGGER_TEMPLATE_ALL_TABLES.format(
 90            project_id=project_id, tables=" UNION ALL ".join(datagroup_tables)
 91        ),
 92    )
 93
 94
 95def _get_datagroup_from_bigquery_view(
 96    project_id,
 97    dataset_id,
 98    table_id,
 99    dataset_view_map: BQViewReferenceMap,
100    view: View,
101) -> Optional[Datagroup]:
102    # Dataset view map only contains references for shared-prod views.
103    full_table_id = f"{project_id}.{dataset_id}.{table_id}"
104
105    view_references = _get_referenced_tables(
106        project_id, dataset_id, table_id, dataset_view_map, []
107    )
108
109    if not view_references or len(view_references) == 0:
110        # Some views might not reference a source table
111        logging.debug(f"Unable to find a source for {full_table_id} in generated-sql.")
112        return None
113
114    return _get_datagroup_from_bigquery_tables(project_id, view_references, view)
115
116
117def _get_referenced_tables(
118    project_id,
119    dataset_id,
120    table_id,
121    dataset_view_map: BQViewReferenceMap,
122    seen: List[List[str]],
123) -> List[List[str]]:
124    """
125    Return a list of all tables referenced by the provided view.
126
127    Recursively, resolve references of referenced views to only get table dependencies.
128    """
129    if [project_id, dataset_id, table_id] in seen:
130        return [[project_id, dataset_id, table_id]]
131
132    seen += [[project_id, dataset_id, table_id]]
133
134    dataset_view_references = dataset_view_map.get(dataset_id)
135
136    if dataset_view_references is None:
137        return [[project_id, dataset_id, table_id]]
138
139    view_references = dataset_view_references.get(table_id)
140    if view_references is None:
141        return [[project_id, dataset_id, table_id]]
142
143    return [
144        ref
145        for view_reference in view_references
146        for ref in _get_referenced_tables(
147            view_reference[0],
148            view_reference[1],
149            view_reference[2],
150            dataset_view_map,
151            seen.copy(),
152        )
153        if view_reference not in seen
154    ]
155
156
157def _generate_view_datagroup(
158    view: View,
159    dataset_view_map: BQViewReferenceMap,
160    dryrun,
161) -> Optional[Datagroup]:
162    """Generate the Datagroup LookML for a Looker View."""
163    if len(view.tables) == 0:
164        return None
165
166    # Use the release channel table or the first available table (usually the only one):
167    view_tables = next(
168        (table for table in view.tables if table.get("channel") == "release"),
169        view.tables[0],
170    )
171
172    if "table" not in view_tables:
173        return None
174
175    view_table = view_tables["table"]
176
177    [project, dataset, table] = view_table.split(".")
178    table_metadata = dryrun.create(
179        project=project,
180        dataset=dataset,
181        table=table,
182    ).get_table_metadata()
183
184    if "TABLE" == table_metadata.get("tableType"):
185        datagroups = _get_datagroup_from_bigquery_tables(
186            project, [[project, dataset, table]], view
187        )
188        return datagroups
189    elif "VIEW" == table_metadata.get("tableType"):
190        datagroups = _get_datagroup_from_bigquery_view(
191            project, dataset, table, dataset_view_map, view
192        )
193        return datagroups
194
195    return None
196
197
198def generate_datagroup(
199    view: View,
200    target_dir: Path,
201    namespace: str,
202    dryrun,
203) -> Any:
204    """Generate and write a datagroups.lkml file to the namespace folder."""
205    datagroups_folder_path = target_dir / namespace / "datagroups"
206
207    datagroup = None
208    try:
209        datagroup = _generate_view_datagroup(view, DATASET_VIEW_MAP, dryrun)
210    except DryRunError as e:
211        if e.error == Errors.PERMISSION_DENIED and e.use_cloud_function:
212            path = datagroups_folder_path / f"{e.table_id}_last_updated.datagroup.lkml"
213            print(
214                f"Permission error dry running: {path}. Copy existing file from looker-hub."
215            )
216            try:
217                get_file_from_looker_hub(path)
218            except Exception as ex:
219                print(f"Skip generating datagroup for {path}: {ex}")
220        else:
221            raise
222
223    datagroup_paths = []
224    if datagroup:
225        datagroups_folder_path.mkdir(exist_ok=True)
226        datagroup_lkml_path = (
227            datagroups_folder_path / f"{datagroup.name}.datagroup.lkml"
228        )
229        datagroup_lkml_path.write_text(FILE_HEADER + str(datagroup))
230        datagroup_paths.append(datagroup_lkml_path)
231
232    return datagroup_paths