places/storage/
mod.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5// A "storage" module - this module is intended to be the layer between the
6// API and the database.
7
8pub mod bookmarks;
9pub mod history;
10pub mod history_metadata;
11pub mod tags;
12
13use crate::db::PlacesDb;
14use crate::error::{warn, Error, InvalidPlaceInfo, Result};
15use crate::ffi::HistoryVisitInfo;
16use crate::ffi::TopFrecentSiteInfo;
17use crate::frecency::{calculate_frecency, DEFAULT_FRECENCY_SETTINGS};
18use crate::types::{SyncStatus, UnknownFields, VisitType};
19use interrupt_support::SqlInterruptScope;
20use rusqlite::types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef};
21use rusqlite::Result as RusqliteResult;
22use rusqlite::{Connection, Row};
23use serde_derive::*;
24use sql_support::{self, ConnExt};
25use std::fmt;
26use sync_guid::Guid as SyncGuid;
27use types::Timestamp;
28use url::Url;
29
30/// From https://searchfox.org/mozilla-central/rev/93905b660f/toolkit/components/places/PlacesUtils.jsm#189
31pub const URL_LENGTH_MAX: usize = 65536;
32pub const TITLE_LENGTH_MAX: usize = 4096;
33pub const TAG_LENGTH_MAX: usize = 100;
34// pub const DESCRIPTION_LENGTH_MAX: usize = 256;
35
36// Typesafe way to manage RowIds. Does it make sense? A better way?
37#[derive(
38    Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Deserialize, Serialize, Default, Hash,
39)]
40pub struct RowId(pub i64);
41
42impl From<RowId> for i64 {
43    // XXX - ToSql!
44    #[inline]
45    fn from(id: RowId) -> Self {
46        id.0
47    }
48}
49
50impl fmt::Display for RowId {
51    #[inline]
52    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53        write!(f, "{}", self.0)
54    }
55}
56
57impl ToSql for RowId {
58    fn to_sql(&self) -> RusqliteResult<ToSqlOutput<'_>> {
59        Ok(ToSqlOutput::from(self.0))
60    }
61}
62
63impl FromSql for RowId {
64    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
65        value.as_i64().map(RowId)
66    }
67}
68
69#[derive(Debug)]
70pub struct PageInfo {
71    pub url: Url,
72    pub guid: SyncGuid,
73    pub row_id: RowId,
74    pub title: String,
75    pub hidden: bool,
76    pub preview_image_url: Option<Url>,
77    pub typed: u32,
78    pub frecency: i32,
79    pub visit_count_local: i32,
80    pub visit_count_remote: i32,
81    pub last_visit_date_local: Timestamp,
82    pub last_visit_date_remote: Timestamp,
83    pub sync_status: SyncStatus,
84    pub sync_change_counter: u32,
85    pub unknown_fields: UnknownFields,
86}
87
88impl PageInfo {
89    pub fn from_row(row: &Row<'_>) -> Result<Self> {
90        Ok(Self {
91            url: Url::parse(&row.get::<_, String>("url")?)?,
92            guid: row.get::<_, String>("guid")?.into(),
93            row_id: row.get("id")?,
94            title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
95            hidden: row.get("hidden")?,
96            preview_image_url: match row.get::<_, Option<String>>("preview_image_url")? {
97                Some(ref preview_image_url) => Some(Url::parse(preview_image_url)?),
98                None => None,
99            },
100            typed: row.get("typed")?,
101
102            frecency: row.get("frecency")?,
103            visit_count_local: row.get("visit_count_local")?,
104            visit_count_remote: row.get("visit_count_remote")?,
105
106            last_visit_date_local: row
107                .get::<_, Option<Timestamp>>("last_visit_date_local")?
108                .unwrap_or_default(),
109            last_visit_date_remote: row
110                .get::<_, Option<Timestamp>>("last_visit_date_remote")?
111                .unwrap_or_default(),
112
113            sync_status: SyncStatus::from_u8(row.get::<_, u8>("sync_status")?),
114            sync_change_counter: row
115                .get::<_, Option<u32>>("sync_change_counter")?
116                .unwrap_or_default(),
117            unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
118                Some(v) => serde_json::from_str(&v)?,
119                None => UnknownFields::new(),
120            },
121        })
122    }
123}
124
125// fetch_page_info gives you one of these.
126#[derive(Debug)]
127pub struct FetchedPageInfo {
128    pub page: PageInfo,
129    // XXX - not clear what this is used for yet, and whether it should be local, remote or either?
130    // The sql below isn't quite sure either :)
131    pub last_visit_id: Option<RowId>,
132}
133
134impl FetchedPageInfo {
135    pub fn from_row(row: &Row<'_>) -> Result<Self> {
136        Ok(Self {
137            page: PageInfo::from_row(row)?,
138            last_visit_id: row.get::<_, Option<RowId>>("last_visit_id")?,
139        })
140    }
141}
142
143// History::FetchPageInfo
144pub fn fetch_page_info(db: &PlacesDb, url: &Url) -> Result<Option<FetchedPageInfo>> {
145    let sql = "
146      SELECT guid, url, id, title, hidden, typed, frecency,
147             visit_count_local, visit_count_remote,
148             last_visit_date_local, last_visit_date_remote,
149             sync_status, sync_change_counter, preview_image_url,
150             unknown_fields,
151             (SELECT id FROM moz_historyvisits
152              WHERE place_id = h.id
153                AND (visit_date = h.last_visit_date_local OR
154                     visit_date = h.last_visit_date_remote)) AS last_visit_id
155      FROM moz_places h
156      WHERE url_hash = hash(:page_url) AND url = :page_url";
157    db.try_query_row(
158        sql,
159        &[(":page_url", &String::from(url.clone()))],
160        FetchedPageInfo::from_row,
161        true,
162    )
163}
164
165fn new_page_info(db: &PlacesDb, url: &Url, new_guid: Option<SyncGuid>) -> Result<PageInfo> {
166    let guid = match new_guid {
167        Some(guid) => guid,
168        None => SyncGuid::random(),
169    };
170    let url_str = url.as_str();
171    if url_str.len() > URL_LENGTH_MAX {
172        // Generally callers check this first (bookmarks don't, history does).
173        return Err(Error::InvalidPlaceInfo(InvalidPlaceInfo::UrlTooLong));
174    }
175    let sql = "INSERT INTO moz_places (guid, url, url_hash)
176               VALUES (:guid, :url, hash(:url))";
177    db.execute_cached(sql, &[(":guid", &guid as &dyn ToSql), (":url", &url_str)])?;
178    Ok(PageInfo {
179        url: url.clone(),
180        guid,
181        row_id: RowId(db.conn().last_insert_rowid()),
182        title: "".into(),
183        hidden: true, // will be set to false as soon as a non-hidden visit appears.
184        preview_image_url: None,
185        typed: 0,
186        frecency: -1,
187        visit_count_local: 0,
188        visit_count_remote: 0,
189        last_visit_date_local: Timestamp(0),
190        last_visit_date_remote: Timestamp(0),
191        sync_status: SyncStatus::New,
192        sync_change_counter: 0,
193        unknown_fields: UnknownFields::new(),
194    })
195}
196
197impl HistoryVisitInfo {
198    fn from_row(row: &rusqlite::Row<'_>) -> Result<Self> {
199        let visit_type = VisitType::from_primitive(row.get::<_, u8>("visit_type")?)
200            // Do we have an existing error we use for this? For now they
201            // probably don't care too much about VisitType, so this
202            // is fine.
203            .unwrap_or(VisitType::Link);
204        let visit_date: Timestamp = row.get("visit_date")?;
205        let url: String = row.get("url")?;
206        let preview_image_url: Option<String> = row.get("preview_image_url")?;
207        Ok(Self {
208            url: Url::parse(&url)?,
209            title: row.get("title")?,
210            timestamp: visit_date,
211            visit_type,
212            is_hidden: row.get("hidden")?,
213            preview_image_url: match preview_image_url {
214                Some(s) => Some(Url::parse(&s)?),
215                None => None,
216            },
217            is_remote: !row.get("is_local")?,
218        })
219    }
220}
221
222impl TopFrecentSiteInfo {
223    pub(crate) fn from_row(row: &rusqlite::Row<'_>) -> Result<Self> {
224        let url: String = row.get("url")?;
225        Ok(Self {
226            url: Url::parse(&url)?,
227            title: row.get("title")?,
228        })
229    }
230}
231
232#[derive(Debug)]
233pub struct RunMaintenanceMetrics {
234    pub pruned_visits: bool,
235    pub db_size_before: u32,
236    pub db_size_after: u32,
237}
238
239/// Run maintenance on the places DB (prune step)
240///
241/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
242/// to clean up / shrink the database.  They're split up so that we can time each one in the
243/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
244/// it supports a stop-watch style API, not recording specific values).
245///
246/// db_size_limit is the approximate storage limit in bytes.  If the database is using more space
247/// than this, some older visits will be deleted to free up space.  Pass in a 0 to skip this.
248///
249/// prune_limit is the maximum number of visits to prune if the database is over db_size_limit
250pub fn run_maintenance_prune(
251    conn: &PlacesDb,
252    db_size_limit: u32,
253    prune_limit: u32,
254) -> Result<RunMaintenanceMetrics> {
255    let db_size_before = conn.get_db_size()?;
256    let should_prune = db_size_limit > 0 && db_size_before > db_size_limit;
257    if should_prune {
258        history::prune_older_visits(conn, prune_limit)?;
259    }
260    let db_size_after = conn.get_db_size()?;
261    Ok(RunMaintenanceMetrics {
262        pruned_visits: should_prune,
263        db_size_before,
264        db_size_after,
265    })
266}
267
268/// Run maintenance on the places DB (vacuum step)
269///
270/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
271/// to clean up / shrink the database.  They're split up so that we can time each one in the
272/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
273/// it supports a stop-watch style API, not recording specific values).
274pub fn run_maintenance_vacuum(conn: &PlacesDb) -> Result<()> {
275    let auto_vacuum_setting: u32 = conn.conn_ext_query_one("PRAGMA auto_vacuum")?;
276    if auto_vacuum_setting == 2 {
277        // Ideally, we run an incremental vacuum to delete 2 pages
278        conn.execute_one("PRAGMA incremental_vacuum(2)")?;
279    } else {
280        // If auto_vacuum=incremental isn't set, configure it and run a full vacuum.
281        warn!("run_maintenance_vacuum: Need to run a full vacuum to set auto_vacuum=incremental");
282        conn.execute_one("PRAGMA auto_vacuum=incremental")?;
283        conn.execute_one("VACUUM")?;
284    }
285    Ok(())
286}
287
288/// Run maintenance on the places DB (optimize step)
289///
290/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
291/// to clean up / shrink the database.  They're split up so that we can time each one in the
292/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
293/// it supports a stop-watch style API, not recording specific values).
294pub fn run_maintenance_optimize(conn: &PlacesDb) -> Result<()> {
295    // 0x10012: run ANALYZE on tables that might benefit (0x02), with a row limit to keep
296    // runtime bounded (0x10), including tables not queried during this connection (0x10000).
297    // The 0x10000 bit lets maintenance refresh stats for tables the writer never queries;
298    // desktop added this alongside the Bug 2017227 shutdown fix.
299    conn.execute_one("PRAGMA optimize(0x10012)")?;
300    Ok(())
301}
302
303/// Run maintenance on the places DB (checkpoint step)
304///
305/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
306/// to clean up / shrink the database.  They're split up so that we can time each one in the
307/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
308/// it supports a stop-watch style API, not recording specific values).
309pub fn run_maintenance_checkpoint(conn: &PlacesDb) -> Result<()> {
310    conn.execute_one("PRAGMA wal_checkpoint(PASSIVE)")?;
311    Ok(())
312}
313
314pub fn update_all_frecencies_at_once(db: &PlacesDb, scope: &SqlInterruptScope) -> Result<()> {
315    let tx = db.begin_transaction()?;
316
317    let need_frecency_update = tx.query_rows_and_then(
318        "SELECT place_id FROM moz_places_stale_frecencies",
319        [],
320        |r| r.get::<_, i64>(0),
321    )?;
322    scope.err_if_interrupted()?;
323    let frecencies = need_frecency_update
324        .iter()
325        .map(|places_id| {
326            scope.err_if_interrupted()?;
327            Ok((
328                *places_id,
329                calculate_frecency(db, &DEFAULT_FRECENCY_SETTINGS, *places_id, Some(false))?,
330            ))
331        })
332        .collect::<Result<Vec<(i64, i32)>>>()?;
333
334    if frecencies.is_empty() {
335        return Ok(());
336    }
337    // Update all frecencies in one fell swoop
338    tx.execute_batch(&format!(
339        "WITH frecencies(id, frecency) AS (
340            VALUES {}
341            )
342            UPDATE moz_places SET
343            frecency = (SELECT frecency FROM frecencies f
344                        WHERE f.id = id)
345            WHERE id IN (SELECT f.id FROM frecencies f)",
346        sql_support::repeat_display(frecencies.len(), ",", |index, f| {
347            let (id, frecency) = frecencies[index];
348            write!(f, "({}, {})", id, frecency)
349        })
350    ))?;
351
352    scope.err_if_interrupted()?;
353
354    // ...And remove them from the stale table.
355    tx.execute_batch(&format!(
356        "DELETE FROM moz_places_stale_frecencies
357         WHERE place_id IN ({})",
358        sql_support::repeat_display(frecencies.len(), ",", |index, f| {
359            let (id, _) = frecencies[index];
360            write!(f, "{}", id)
361        })
362    ))?;
363    tx.commit()?;
364
365    Ok(())
366}
367
368pub(crate) fn put_meta(conn: &Connection, key: &str, value: &dyn ToSql) -> Result<()> {
369    conn.execute_cached(
370        "REPLACE INTO moz_meta (key, value) VALUES (:key, :value)",
371        &[(":key", &key as &dyn ToSql), (":value", value)],
372    )?;
373    Ok(())
374}
375
376pub(crate) fn get_meta<T: FromSql>(db: &PlacesDb, key: &str) -> Result<Option<T>> {
377    let res = db.try_query_one(
378        "SELECT value FROM moz_meta WHERE key = :key",
379        &[(":key", &key)],
380        true,
381    )?;
382    Ok(res)
383}
384
385pub(crate) fn delete_meta(db: &PlacesDb, key: &str) -> Result<()> {
386    db.execute_cached("DELETE FROM moz_meta WHERE key = :key", &[(":key", &key)])?;
387    Ok(())
388}
389
390/// Delete all items in the temp tables we use for staging changes.
391pub fn delete_pending_temp_tables(conn: &PlacesDb) -> Result<()> {
392    conn.execute_batch(
393        "DELETE FROM moz_updateoriginsinsert_temp;
394         DELETE FROM moz_updateoriginsupdate_temp;
395         DELETE FROM moz_updateoriginsdelete_temp;",
396    )?;
397    Ok(())
398}
399
400#[cfg(test)]
401mod tests {
402    use super::*;
403    use crate::api::places_api::test::new_mem_connection;
404    use crate::observation::VisitObservation;
405    use bookmarks::{
406        delete_bookmark, insert_bookmark, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
407        InsertableItem,
408    };
409    use history::apply_observation;
410
411    #[test]
412    fn test_meta() {
413        let conn = new_mem_connection();
414        let value1 = "value 1".to_string();
415        let value2 = "value 2".to_string();
416        assert!(get_meta::<String>(&conn, "foo")
417            .expect("should get")
418            .is_none());
419        put_meta(&conn, "foo", &value1).expect("should put");
420        assert_eq!(
421            get_meta(&conn, "foo").expect("should get new val"),
422            Some(value1)
423        );
424        put_meta(&conn, "foo", &value2).expect("should put an existing value");
425        assert_eq!(get_meta(&conn, "foo").expect("should get"), Some(value2));
426        delete_meta(&conn, "foo").expect("should delete");
427        assert!(get_meta::<String>(&conn, "foo")
428            .expect("should get non-existing")
429            .is_none());
430        delete_meta(&conn, "foo").expect("delete non-existing should work");
431    }
432
433    // Here we try and test that we replicate desktop behaviour, which isn't that obvious.
434    // * create a bookmark
435    // * remove the bookmark - this doesn't remove the place or origin - probably because in
436    //   real browsers there will be visits for the URL existing, but this still smells like
437    //   a bug - see https://bugzilla.mozilla.org/show_bug.cgi?id=1650511#c34
438    // * Arrange for history for that item to be removed, via various means
439    // At this point the origin and place should be removed. The only code (in desktop and here) which
440    // removes places with a foreign_count of zero is that history removal!
441
442    #[test]
443    fn test_removal_delete_visits_between() {
444        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
445            history::delete_visits_between(conn, Timestamp::EARLIEST, Timestamp::now())
446        })
447    }
448
449    #[test]
450    fn test_removal_delete_visits_for() {
451        do_test_removal_places_and_origins(|conn: &PlacesDb, guid: &SyncGuid| {
452            history::delete_visits_for(conn, guid)
453        })
454    }
455
456    #[test]
457    fn test_removal_prune() {
458        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
459            history::prune_older_visits(conn, 6)
460        })
461    }
462
463    #[test]
464    fn test_removal_visit_at_time() {
465        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
466            let url = Url::parse("http://example.com/foo").unwrap();
467            let visit = Timestamp::from(727_747_200_001);
468            history::delete_place_visit_at_time(conn, &url, visit)
469        })
470    }
471
472    #[test]
473    fn test_removal_everything() {
474        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
475            history::delete_everything(conn)
476        })
477    }
478
479    // The core test - takes a function which deletes history.
480    fn do_test_removal_places_and_origins<F>(removal_fn: F)
481    where
482        F: FnOnce(&PlacesDb, &SyncGuid) -> Result<()>,
483    {
484        let conn = new_mem_connection();
485        let url = Url::parse("http://example.com/foo").unwrap();
486        let bm = InsertableItem::Bookmark {
487            b: InsertableBookmark {
488                parent_guid: BookmarkRootGuid::Unfiled.into(),
489                position: BookmarkPosition::Append,
490                date_added: None,
491                last_modified: None,
492                guid: None,
493                url: url.clone(),
494                title: Some("the title".into()),
495            },
496        };
497        assert_eq!(
498            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
499                .unwrap(),
500            5
501        ); // our 5 roots.
502        let bookmark_guid = insert_bookmark(&conn, bm).unwrap();
503        let place_guid = fetch_page_info(&conn, &url)
504            .expect("should work")
505            .expect("must exist")
506            .page
507            .guid;
508        // the place should exist with a foreign_count of 1.
509        assert_eq!(
510            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
511                .unwrap(),
512            6
513        ); // our 5 roots + new bookmark
514        assert_eq!(
515            conn.conn_ext_query_one::<i64>(
516                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
517            )
518            .unwrap(),
519            1
520        );
521        // visit the bookmark.
522        assert!(apply_observation(
523            &conn,
524            VisitObservation::new(url)
525                .with_at(Timestamp::from(727_747_200_001))
526                .with_visit_type(VisitType::Link)
527        )
528        .unwrap()
529        .is_some());
530
531        delete_bookmark(&conn, &bookmark_guid).unwrap();
532        assert_eq!(
533            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
534                .unwrap(),
535            5
536        ); // our 5 roots
537           // the place should have no foreign references, but still exists.
538        assert_eq!(
539            conn.conn_ext_query_one::<i64>(
540                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
541            )
542            .unwrap(),
543            0
544        );
545        removal_fn(&conn, &place_guid).expect("removal function should work");
546        assert_eq!(
547            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_places;")
548                .unwrap(),
549            0
550        );
551        assert_eq!(
552            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_origins;")
553                .unwrap(),
554            0
555        );
556    }
557
558    // Similar to the above, but if the bookmark has no visits the place/origin should die
559    // without requiring history removal
560    #[test]
561    fn test_visitless_removal_places_and_origins() {
562        let conn = new_mem_connection();
563        let url = Url::parse("http://example.com/foo").unwrap();
564        let bm = InsertableItem::Bookmark {
565            b: InsertableBookmark {
566                parent_guid: BookmarkRootGuid::Unfiled.into(),
567                position: BookmarkPosition::Append,
568                date_added: None,
569                last_modified: None,
570                guid: None,
571                url,
572                title: Some("the title".into()),
573            },
574        };
575        assert_eq!(
576            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
577                .unwrap(),
578            5
579        ); // our 5 roots.
580        let bookmark_guid = insert_bookmark(&conn, bm).unwrap();
581        // the place should exist with a foreign_count of 1.
582        assert_eq!(
583            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
584                .unwrap(),
585            6
586        ); // our 5 roots + new bookmark
587        assert_eq!(
588            conn.conn_ext_query_one::<i64>(
589                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
590            )
591            .unwrap(),
592            1
593        );
594        // Delete it.
595        delete_bookmark(&conn, &bookmark_guid).unwrap();
596        assert_eq!(
597            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
598                .unwrap(),
599            5
600        ); // our 5 roots
601           // should be gone from places and origins.
602        assert_eq!(
603            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_places;")
604                .unwrap(),
605            0
606        );
607        assert_eq!(
608            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_origins;")
609                .unwrap(),
610            0
611        );
612    }
613}
places/storage/mod.rs

places/storage/
mod.rs
