places/storage/
mod.rs

Help
1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5// A "storage" module - this module is intended to be the layer between the
6// API and the database.
7
8pub mod bookmarks;
9pub mod history;
10pub mod history_metadata;
11pub mod tags;
12
13use crate::db::PlacesDb;
14use crate::error::{warn, Error, InvalidPlaceInfo, Result};
15use crate::ffi::HistoryVisitInfo;
16use crate::ffi::TopFrecentSiteInfo;
17use crate::frecency::{calculate_frecency, DEFAULT_FRECENCY_SETTINGS};
18use crate::types::{SyncStatus, UnknownFields, VisitType};
19use interrupt_support::SqlInterruptScope;
20use rusqlite::types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef};
21use rusqlite::Result as RusqliteResult;
22use rusqlite::{Connection, Row};
23use serde_derive::*;
24use sql_support::{self, ConnExt};
25use std::fmt;
26use sync_guid::Guid as SyncGuid;
27use types::Timestamp;
28use url::Url;
29
30/// From https://searchfox.org/mozilla-central/rev/93905b660f/toolkit/components/places/PlacesUtils.jsm#189
31pub const URL_LENGTH_MAX: usize = 65536;
32pub const TITLE_LENGTH_MAX: usize = 4096;
33pub const TAG_LENGTH_MAX: usize = 100;
34// pub const DESCRIPTION_LENGTH_MAX: usize = 256;
35
36// Typesafe way to manage RowIds. Does it make sense? A better way?
37#[derive(
38    Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Deserialize, Serialize, Default, Hash,
39)]
40pub struct RowId(pub i64);
41
42impl From<RowId> for i64 {
43    // XXX - ToSql!
44    #[inline]
45    fn from(id: RowId) -> Self {
46        id.0
47    }
48}
49
50impl fmt::Display for RowId {
51    #[inline]
52    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53        write!(f, "{}", self.0)
54    }
55}
56
57impl ToSql for RowId {
58    fn to_sql(&self) -> RusqliteResult<ToSqlOutput<'_>> {
59        Ok(ToSqlOutput::from(self.0))
60    }
61}
62
63impl FromSql for RowId {
64    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
65        value.as_i64().map(RowId)
66    }
67}
68
69#[derive(Debug)]
70pub struct PageInfo {
71    pub url: Url,
72    pub guid: SyncGuid,
73    pub row_id: RowId,
74    pub title: String,
75    pub hidden: bool,
76    pub preview_image_url: Option<Url>,
77    pub typed: u32,
78    pub frecency: i32,
79    pub visit_count_local: i32,
80    pub visit_count_remote: i32,
81    pub last_visit_date_local: Timestamp,
82    pub last_visit_date_remote: Timestamp,
83    pub sync_status: SyncStatus,
84    pub sync_change_counter: u32,
85    pub unknown_fields: UnknownFields,
86}
87
88impl PageInfo {
89    pub fn from_row(row: &Row<'_>) -> Result<Self> {
90        Ok(Self {
91            url: Url::parse(&row.get::<_, String>("url")?)?,
92            guid: row.get::<_, String>("guid")?.into(),
93            row_id: row.get("id")?,
94            title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
95            hidden: row.get("hidden")?,
96            preview_image_url: match row.get::<_, Option<String>>("preview_image_url")? {
97                Some(ref preview_image_url) => Some(Url::parse(preview_image_url)?),
98                None => None,
99            },
100            typed: row.get("typed")?,
101
102            frecency: row.get("frecency")?,
103            visit_count_local: row.get("visit_count_local")?,
104            visit_count_remote: row.get("visit_count_remote")?,
105
106            last_visit_date_local: row
107                .get::<_, Option<Timestamp>>("last_visit_date_local")?
108                .unwrap_or_default(),
109            last_visit_date_remote: row
110                .get::<_, Option<Timestamp>>("last_visit_date_remote")?
111                .unwrap_or_default(),
112
113            sync_status: SyncStatus::from_u8(row.get::<_, u8>("sync_status")?),
114            sync_change_counter: row
115                .get::<_, Option<u32>>("sync_change_counter")?
116                .unwrap_or_default(),
117            unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
118                Some(v) => serde_json::from_str(&v)?,
119                None => UnknownFields::new(),
120            },
121        })
122    }
123}
124
125// fetch_page_info gives you one of these.
126#[derive(Debug)]
127pub struct FetchedPageInfo {
128    pub page: PageInfo,
129    // XXX - not clear what this is used for yet, and whether it should be local, remote or either?
130    // The sql below isn't quite sure either :)
131    pub last_visit_id: Option<RowId>,
132}
133
134impl FetchedPageInfo {
135    pub fn from_row(row: &Row<'_>) -> Result<Self> {
136        Ok(Self {
137            page: PageInfo::from_row(row)?,
138            last_visit_id: row.get::<_, Option<RowId>>("last_visit_id")?,
139        })
140    }
141}
142
143// History::FetchPageInfo
144pub fn fetch_page_info(db: &PlacesDb, url: &Url) -> Result<Option<FetchedPageInfo>> {
145    let sql = "
146      SELECT guid, url, id, title, hidden, typed, frecency,
147             visit_count_local, visit_count_remote,
148             last_visit_date_local, last_visit_date_remote,
149             sync_status, sync_change_counter, preview_image_url,
150             unknown_fields,
151             (SELECT id FROM moz_historyvisits
152              WHERE place_id = h.id
153                AND (visit_date = h.last_visit_date_local OR
154                     visit_date = h.last_visit_date_remote)) AS last_visit_id
155      FROM moz_places h
156      WHERE url_hash = hash(:page_url) AND url = :page_url";
157    db.try_query_row(
158        sql,
159        &[(":page_url", &String::from(url.clone()))],
160        FetchedPageInfo::from_row,
161        true,
162    )
163}
164
165fn new_page_info(db: &PlacesDb, url: &Url, new_guid: Option<SyncGuid>) -> Result<PageInfo> {
166    let guid = match new_guid {
167        Some(guid) => guid,
168        None => SyncGuid::random(),
169    };
170    let url_str = url.as_str();
171    if url_str.len() > URL_LENGTH_MAX {
172        // Generally callers check this first (bookmarks don't, history does).
173        return Err(Error::InvalidPlaceInfo(InvalidPlaceInfo::UrlTooLong));
174    }
175    let sql = "INSERT INTO moz_places (guid, url, url_hash)
176               VALUES (:guid, :url, hash(:url))";
177    db.execute_cached(sql, &[(":guid", &guid as &dyn ToSql), (":url", &url_str)])?;
178    Ok(PageInfo {
179        url: url.clone(),
180        guid,
181        row_id: RowId(db.conn().last_insert_rowid()),
182        title: "".into(),
183        hidden: true, // will be set to false as soon as a non-hidden visit appears.
184        preview_image_url: None,
185        typed: 0,
186        frecency: -1,
187        visit_count_local: 0,
188        visit_count_remote: 0,
189        last_visit_date_local: Timestamp(0),
190        last_visit_date_remote: Timestamp(0),
191        sync_status: SyncStatus::New,
192        sync_change_counter: 0,
193        unknown_fields: UnknownFields::new(),
194    })
195}
196
197impl HistoryVisitInfo {
198    fn from_row(row: &rusqlite::Row<'_>) -> Result<Self> {
199        let visit_type = VisitType::from_primitive(row.get::<_, u8>("visit_type")?)
200            // Do we have an existing error we use for this? For now they
201            // probably don't care too much about VisitType, so this
202            // is fine.
203            .unwrap_or(VisitType::Link);
204        let visit_date: Timestamp = row.get("visit_date")?;
205        let url: String = row.get("url")?;
206        let preview_image_url: Option<String> = row.get("preview_image_url")?;
207        Ok(Self {
208            url: Url::parse(&url)?,
209            title: row.get("title")?,
210            timestamp: visit_date,
211            visit_type,
212            is_hidden: row.get("hidden")?,
213            preview_image_url: match preview_image_url {
214                Some(s) => Some(Url::parse(&s)?),
215                None => None,
216            },
217            is_remote: !row.get("is_local")?,
218        })
219    }
220}
221
222impl TopFrecentSiteInfo {
223    pub(crate) fn from_row(row: &rusqlite::Row<'_>) -> Result<Self> {
224        let url: String = row.get("url")?;
225        Ok(Self {
226            url: Url::parse(&url)?,
227            title: row.get("title")?,
228        })
229    }
230}
231
232#[derive(Debug)]
233pub struct RunMaintenanceMetrics {
234    pub pruned_visits: bool,
235    pub db_size_before: u32,
236    pub db_size_after: u32,
237}
238
239/// Run maintenance on the places DB (prune step)
240///
241/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
242/// to clean up / shrink the database.  They're split up so that we can time each one in the
243/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
244/// it supports a stop-watch style API, not recording specific values).
245///
246/// db_size_limit is the approximate storage limit in bytes.  If the database is using more space
247/// than this, some older visits will be deleted to free up space.  Pass in a 0 to skip this.
248///
249/// prune_limit is the maximum number of visits to prune if the database is over db_size_limit
250pub fn run_maintenance_prune(
251    conn: &PlacesDb,
252    db_size_limit: u32,
253    prune_limit: u32,
254) -> Result<RunMaintenanceMetrics> {
255    let db_size_before = conn.get_db_size()?;
256    let should_prune = db_size_limit > 0 && db_size_before > db_size_limit;
257    if should_prune {
258        history::prune_older_visits(conn, prune_limit)?;
259    }
260    let db_size_after = conn.get_db_size()?;
261    Ok(RunMaintenanceMetrics {
262        pruned_visits: should_prune,
263        db_size_before,
264        db_size_after,
265    })
266}
267
268/// Run maintenance on the places DB (vacuum step)
269///
270/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
271/// to clean up / shrink the database.  They're split up so that we can time each one in the
272/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
273/// it supports a stop-watch style API, not recording specific values).
274pub fn run_maintenance_vacuum(conn: &PlacesDb) -> Result<()> {
275    let auto_vacuum_setting: u32 = conn.query_one("PRAGMA auto_vacuum")?;
276    if auto_vacuum_setting == 2 {
277        // Ideally, we run an incremental vacuum to delete 2 pages
278        conn.execute_one("PRAGMA incremental_vacuum(2)")?;
279    } else {
280        // If auto_vacuum=incremental isn't set, configure it and run a full vacuum.
281        warn!("run_maintenance_vacuum: Need to run a full vacuum to set auto_vacuum=incremental");
282        conn.execute_one("PRAGMA auto_vacuum=incremental")?;
283        conn.execute_one("VACUUM")?;
284    }
285    Ok(())
286}
287
288/// Run maintenance on the places DB (optimize step)
289///
290/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
291/// to clean up / shrink the database.  They're split up so that we can time each one in the
292/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
293/// it supports a stop-watch style API, not recording specific values).
294pub fn run_maintenance_optimize(conn: &PlacesDb) -> Result<()> {
295    conn.execute_one("PRAGMA optimize")?;
296    Ok(())
297}
298
299/// Run maintenance on the places DB (checkpoint step)
300///
301/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
302/// to clean up / shrink the database.  They're split up so that we can time each one in the
303/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
304/// it supports a stop-watch style API, not recording specific values).
305pub fn run_maintenance_checkpoint(conn: &PlacesDb) -> Result<()> {
306    conn.execute_one("PRAGMA wal_checkpoint(PASSIVE)")?;
307    Ok(())
308}
309
310pub fn update_all_frecencies_at_once(db: &PlacesDb, scope: &SqlInterruptScope) -> Result<()> {
311    let tx = db.begin_transaction()?;
312
313    let need_frecency_update = tx.query_rows_and_then(
314        "SELECT place_id FROM moz_places_stale_frecencies",
315        [],
316        |r| r.get::<_, i64>(0),
317    )?;
318    scope.err_if_interrupted()?;
319    let frecencies = need_frecency_update
320        .iter()
321        .map(|places_id| {
322            scope.err_if_interrupted()?;
323            Ok((
324                *places_id,
325                calculate_frecency(db, &DEFAULT_FRECENCY_SETTINGS, *places_id, Some(false))?,
326            ))
327        })
328        .collect::<Result<Vec<(i64, i32)>>>()?;
329
330    if frecencies.is_empty() {
331        return Ok(());
332    }
333    // Update all frecencies in one fell swoop
334    tx.execute_batch(&format!(
335        "WITH frecencies(id, frecency) AS (
336            VALUES {}
337            )
338            UPDATE moz_places SET
339            frecency = (SELECT frecency FROM frecencies f
340                        WHERE f.id = id)
341            WHERE id IN (SELECT f.id FROM frecencies f)",
342        sql_support::repeat_display(frecencies.len(), ",", |index, f| {
343            let (id, frecency) = frecencies[index];
344            write!(f, "({}, {})", id, frecency)
345        })
346    ))?;
347
348    scope.err_if_interrupted()?;
349
350    // ...And remove them from the stale table.
351    tx.execute_batch(&format!(
352        "DELETE FROM moz_places_stale_frecencies
353         WHERE place_id IN ({})",
354        sql_support::repeat_display(frecencies.len(), ",", |index, f| {
355            let (id, _) = frecencies[index];
356            write!(f, "{}", id)
357        })
358    ))?;
359    tx.commit()?;
360
361    Ok(())
362}
363
364pub(crate) fn put_meta(conn: &Connection, key: &str, value: &dyn ToSql) -> Result<()> {
365    conn.execute_cached(
366        "REPLACE INTO moz_meta (key, value) VALUES (:key, :value)",
367        &[(":key", &key as &dyn ToSql), (":value", value)],
368    )?;
369    Ok(())
370}
371
372pub(crate) fn get_meta<T: FromSql>(db: &PlacesDb, key: &str) -> Result<Option<T>> {
373    let res = db.try_query_one(
374        "SELECT value FROM moz_meta WHERE key = :key",
375        &[(":key", &key)],
376        true,
377    )?;
378    Ok(res)
379}
380
381pub(crate) fn delete_meta(db: &PlacesDb, key: &str) -> Result<()> {
382    db.execute_cached("DELETE FROM moz_meta WHERE key = :key", &[(":key", &key)])?;
383    Ok(())
384}
385
386/// Delete all items in the temp tables we use for staging changes.
387pub fn delete_pending_temp_tables(conn: &PlacesDb) -> Result<()> {
388    conn.execute_batch(
389        "DELETE FROM moz_updateoriginsinsert_temp;
390         DELETE FROM moz_updateoriginsupdate_temp;
391         DELETE FROM moz_updateoriginsdelete_temp;",
392    )?;
393    Ok(())
394}
395
396#[cfg(test)]
397mod tests {
398    use super::*;
399    use crate::api::places_api::test::new_mem_connection;
400    use crate::observation::VisitObservation;
401    use bookmarks::{
402        delete_bookmark, insert_bookmark, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
403        InsertableItem,
404    };
405    use history::apply_observation;
406
407    #[test]
408    fn test_meta() {
409        let conn = new_mem_connection();
410        let value1 = "value 1".to_string();
411        let value2 = "value 2".to_string();
412        assert!(get_meta::<String>(&conn, "foo")
413            .expect("should get")
414            .is_none());
415        put_meta(&conn, "foo", &value1).expect("should put");
416        assert_eq!(
417            get_meta(&conn, "foo").expect("should get new val"),
418            Some(value1)
419        );
420        put_meta(&conn, "foo", &value2).expect("should put an existing value");
421        assert_eq!(get_meta(&conn, "foo").expect("should get"), Some(value2));
422        delete_meta(&conn, "foo").expect("should delete");
423        assert!(get_meta::<String>(&conn, "foo")
424            .expect("should get non-existing")
425            .is_none());
426        delete_meta(&conn, "foo").expect("delete non-existing should work");
427    }
428
429    // Here we try and test that we replicate desktop behaviour, which isn't that obvious.
430    // * create a bookmark
431    // * remove the bookmark - this doesn't remove the place or origin - probably because in
432    //   real browsers there will be visits for the URL existing, but this still smells like
433    //   a bug - see https://bugzilla.mozilla.org/show_bug.cgi?id=1650511#c34
434    // * Arrange for history for that item to be removed, via various means
435    // At this point the origin and place should be removed. The only code (in desktop and here) which
436    // removes places with a foreign_count of zero is that history removal!
437
438    #[test]
439    fn test_removal_delete_visits_between() {
440        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
441            history::delete_visits_between(conn, Timestamp::EARLIEST, Timestamp::now())
442        })
443    }
444
445    #[test]
446    fn test_removal_delete_visits_for() {
447        do_test_removal_places_and_origins(|conn: &PlacesDb, guid: &SyncGuid| {
448            history::delete_visits_for(conn, guid)
449        })
450    }
451
452    #[test]
453    fn test_removal_prune() {
454        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
455            history::prune_older_visits(conn, 6)
456        })
457    }
458
459    #[test]
460    fn test_removal_visit_at_time() {
461        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
462            let url = Url::parse("http://example.com/foo").unwrap();
463            let visit = Timestamp::from(727_747_200_001);
464            history::delete_place_visit_at_time(conn, &url, visit)
465        })
466    }
467
468    #[test]
469    fn test_removal_everything() {
470        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
471            history::delete_everything(conn)
472        })
473    }
474
475    // The core test - takes a function which deletes history.
476    fn do_test_removal_places_and_origins<F>(removal_fn: F)
477    where
478        F: FnOnce(&PlacesDb, &SyncGuid) -> Result<()>,
479    {
480        let conn = new_mem_connection();
481        let url = Url::parse("http://example.com/foo").unwrap();
482        let bm = InsertableItem::Bookmark {
483            b: InsertableBookmark {
484                parent_guid: BookmarkRootGuid::Unfiled.into(),
485                position: BookmarkPosition::Append,
486                date_added: None,
487                last_modified: None,
488                guid: None,
489                url: url.clone(),
490                title: Some("the title".into()),
491            },
492        };
493        assert_eq!(
494            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
495                .unwrap(),
496            5
497        ); // our 5 roots.
498        let bookmark_guid = insert_bookmark(&conn, bm).unwrap();
499        let place_guid = fetch_page_info(&conn, &url)
500            .expect("should work")
501            .expect("must exist")
502            .page
503            .guid;
504        // the place should exist with a foreign_count of 1.
505        assert_eq!(
506            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
507                .unwrap(),
508            6
509        ); // our 5 roots + new bookmark
510        assert_eq!(
511            conn.query_one::<i64>(
512                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
513            )
514            .unwrap(),
515            1
516        );
517        // visit the bookmark.
518        assert!(apply_observation(
519            &conn,
520            VisitObservation::new(url)
521                .with_at(Timestamp::from(727_747_200_001))
522                .with_visit_type(VisitType::Link)
523        )
524        .unwrap()
525        .is_some());
526
527        delete_bookmark(&conn, &bookmark_guid).unwrap();
528        assert_eq!(
529            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
530                .unwrap(),
531            5
532        ); // our 5 roots
533           // the place should have no foreign references, but still exists.
534        assert_eq!(
535            conn.query_one::<i64>(
536                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
537            )
538            .unwrap(),
539            0
540        );
541        removal_fn(&conn, &place_guid).expect("removal function should work");
542        assert_eq!(
543            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places;")
544                .unwrap(),
545            0
546        );
547        assert_eq!(
548            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_origins;")
549                .unwrap(),
550            0
551        );
552    }
553
554    // Similar to the above, but if the bookmark has no visits the place/origin should die
555    // without requiring history removal
556    #[test]
557    fn test_visitless_removal_places_and_origins() {
558        let conn = new_mem_connection();
559        let url = Url::parse("http://example.com/foo").unwrap();
560        let bm = InsertableItem::Bookmark {
561            b: InsertableBookmark {
562                parent_guid: BookmarkRootGuid::Unfiled.into(),
563                position: BookmarkPosition::Append,
564                date_added: None,
565                last_modified: None,
566                guid: None,
567                url,
568                title: Some("the title".into()),
569            },
570        };
571        assert_eq!(
572            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
573                .unwrap(),
574            5
575        ); // our 5 roots.
576        let bookmark_guid = insert_bookmark(&conn, bm).unwrap();
577        // the place should exist with a foreign_count of 1.
578        assert_eq!(
579            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
580                .unwrap(),
581            6
582        ); // our 5 roots + new bookmark
583        assert_eq!(
584            conn.query_one::<i64>(
585                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
586            )
587            .unwrap(),
588            1
589        );
590        // Delete it.
591        delete_bookmark(&conn, &bookmark_guid).unwrap();
592        assert_eq!(
593            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
594                .unwrap(),
595            5
596        ); // our 5 roots
597           // should be gone from places and origins.
598        assert_eq!(
599            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places;")
600                .unwrap(),
601            0
602        );
603        assert_eq!(
604            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_origins;")
605                .unwrap(),
606            0
607        );
608    }
609}
places/storage/mod.rs

places/storage/
mod.rs
