places/storage/
mod.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5// A "storage" module - this module is intended to be the layer between the
6// API and the database.
7
8pub mod bookmarks;
9pub mod history;
10pub mod history_metadata;
11pub mod tags;
12
13use crate::db::PlacesDb;
14use crate::error::{Error, InvalidPlaceInfo, Result};
15use crate::ffi::HistoryVisitInfo;
16use crate::ffi::TopFrecentSiteInfo;
17use crate::frecency::{calculate_frecency, DEFAULT_FRECENCY_SETTINGS};
18use crate::types::{SyncStatus, UnknownFields, VisitType};
19use interrupt_support::SqlInterruptScope;
20use rusqlite::types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef};
21use rusqlite::Result as RusqliteResult;
22use rusqlite::{Connection, Row};
23use serde_derive::*;
24use sql_support::{self, ConnExt};
25use std::fmt;
26use sync_guid::Guid as SyncGuid;
27use types::Timestamp;
28use url::Url;
29
30/// From https://searchfox.org/mozilla-central/rev/93905b660f/toolkit/components/places/PlacesUtils.jsm#189
31pub const URL_LENGTH_MAX: usize = 65536;
32pub const TITLE_LENGTH_MAX: usize = 4096;
33pub const TAG_LENGTH_MAX: usize = 100;
34// pub const DESCRIPTION_LENGTH_MAX: usize = 256;
35
36// Typesafe way to manage RowIds. Does it make sense? A better way?
37#[derive(
38    Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Deserialize, Serialize, Default, Hash,
39)]
40pub struct RowId(pub i64);
41
42impl From<RowId> for i64 {
43    // XXX - ToSql!
44    #[inline]
45    fn from(id: RowId) -> Self {
46        id.0
47    }
48}
49
50impl fmt::Display for RowId {
51    #[inline]
52    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53        write!(f, "{}", self.0)
54    }
55}
56
57impl ToSql for RowId {
58    fn to_sql(&self) -> RusqliteResult<ToSqlOutput<'_>> {
59        Ok(ToSqlOutput::from(self.0))
60    }
61}
62
63impl FromSql for RowId {
64    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
65        value.as_i64().map(RowId)
66    }
67}
68
69#[derive(Debug)]
70pub struct PageInfo {
71    pub url: Url,
72    pub guid: SyncGuid,
73    pub row_id: RowId,
74    pub title: String,
75    pub hidden: bool,
76    pub preview_image_url: Option<Url>,
77    pub typed: u32,
78    pub frecency: i32,
79    pub visit_count_local: i32,
80    pub visit_count_remote: i32,
81    pub last_visit_date_local: Timestamp,
82    pub last_visit_date_remote: Timestamp,
83    pub sync_status: SyncStatus,
84    pub sync_change_counter: u32,
85    pub unknown_fields: UnknownFields,
86}
87
88impl PageInfo {
89    pub fn from_row(row: &Row<'_>) -> Result<Self> {
90        Ok(Self {
91            url: Url::parse(&row.get::<_, String>("url")?)?,
92            guid: row.get::<_, String>("guid")?.into(),
93            row_id: row.get("id")?,
94            title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
95            hidden: row.get("hidden")?,
96            preview_image_url: match row.get::<_, Option<String>>("preview_image_url")? {
97                Some(ref preview_image_url) => Some(Url::parse(preview_image_url)?),
98                None => None,
99            },
100            typed: row.get("typed")?,
101
102            frecency: row.get("frecency")?,
103            visit_count_local: row.get("visit_count_local")?,
104            visit_count_remote: row.get("visit_count_remote")?,
105
106            last_visit_date_local: row
107                .get::<_, Option<Timestamp>>("last_visit_date_local")?
108                .unwrap_or_default(),
109            last_visit_date_remote: row
110                .get::<_, Option<Timestamp>>("last_visit_date_remote")?
111                .unwrap_or_default(),
112
113            sync_status: SyncStatus::from_u8(row.get::<_, u8>("sync_status")?),
114            sync_change_counter: row
115                .get::<_, Option<u32>>("sync_change_counter")?
116                .unwrap_or_default(),
117            unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
118                Some(v) => serde_json::from_str(&v)?,
119                None => UnknownFields::new(),
120            },
121        })
122    }
123}
124
125// fetch_page_info gives you one of these.
126#[derive(Debug)]
127pub struct FetchedPageInfo {
128    pub page: PageInfo,
129    // XXX - not clear what this is used for yet, and whether it should be local, remote or either?
130    // The sql below isn't quite sure either :)
131    pub last_visit_id: Option<RowId>,
132}
133
134impl FetchedPageInfo {
135    pub fn from_row(row: &Row<'_>) -> Result<Self> {
136        Ok(Self {
137            page: PageInfo::from_row(row)?,
138            last_visit_id: row.get::<_, Option<RowId>>("last_visit_id")?,
139        })
140    }
141}
142
143// History::FetchPageInfo
144pub fn fetch_page_info(db: &PlacesDb, url: &Url) -> Result<Option<FetchedPageInfo>> {
145    let sql = "
146      SELECT guid, url, id, title, hidden, typed, frecency,
147             visit_count_local, visit_count_remote,
148             last_visit_date_local, last_visit_date_remote,
149             sync_status, sync_change_counter, preview_image_url,
150             unknown_fields,
151             (SELECT id FROM moz_historyvisits
152              WHERE place_id = h.id
153                AND (visit_date = h.last_visit_date_local OR
154                     visit_date = h.last_visit_date_remote)) AS last_visit_id
155      FROM moz_places h
156      WHERE url_hash = hash(:page_url) AND url = :page_url";
157    db.try_query_row(
158        sql,
159        &[(":page_url", &String::from(url.clone()))],
160        FetchedPageInfo::from_row,
161        true,
162    )
163}
164
165fn new_page_info(db: &PlacesDb, url: &Url, new_guid: Option<SyncGuid>) -> Result<PageInfo> {
166    let guid = match new_guid {
167        Some(guid) => guid,
168        None => SyncGuid::random(),
169    };
170    let url_str = url.as_str();
171    if url_str.len() > URL_LENGTH_MAX {
172        // Generally callers check this first (bookmarks don't, history does).
173        return Err(Error::InvalidPlaceInfo(InvalidPlaceInfo::UrlTooLong));
174    }
175    let sql = "INSERT INTO moz_places (guid, url, url_hash)
176               VALUES (:guid, :url, hash(:url))";
177    db.execute_cached(sql, &[(":guid", &guid as &dyn ToSql), (":url", &url_str)])?;
178    Ok(PageInfo {
179        url: url.clone(),
180        guid,
181        row_id: RowId(db.conn().last_insert_rowid()),
182        title: "".into(),
183        hidden: true, // will be set to false as soon as a non-hidden visit appears.
184        preview_image_url: None,
185        typed: 0,
186        frecency: -1,
187        visit_count_local: 0,
188        visit_count_remote: 0,
189        last_visit_date_local: Timestamp(0),
190        last_visit_date_remote: Timestamp(0),
191        sync_status: SyncStatus::New,
192        sync_change_counter: 0,
193        unknown_fields: UnknownFields::new(),
194    })
195}
196
197impl HistoryVisitInfo {
198    fn from_row(row: &rusqlite::Row<'_>) -> Result<Self> {
199        let visit_type = VisitType::from_primitive(row.get::<_, u8>("visit_type")?)
200            // Do we have an existing error we use for this? For now they
201            // probably don't care too much about VisitType, so this
202            // is fine.
203            .unwrap_or(VisitType::Link);
204        let visit_date: Timestamp = row.get("visit_date")?;
205        let url: String = row.get("url")?;
206        let preview_image_url: Option<String> = row.get("preview_image_url")?;
207        Ok(Self {
208            url: Url::parse(&url)?,
209            title: row.get("title")?,
210            timestamp: visit_date,
211            visit_type,
212            is_hidden: row.get("hidden")?,
213            preview_image_url: match preview_image_url {
214                Some(s) => Some(Url::parse(&s)?),
215                None => None,
216            },
217            is_remote: !row.get("is_local")?,
218        })
219    }
220}
221
222impl TopFrecentSiteInfo {
223    pub(crate) fn from_row(row: &rusqlite::Row<'_>) -> Result<Self> {
224        let url: String = row.get("url")?;
225        Ok(Self {
226            url: Url::parse(&url)?,
227            title: row.get("title")?,
228        })
229    }
230}
231
232#[derive(Debug)]
233pub struct RunMaintenanceMetrics {
234    pub pruned_visits: bool,
235    pub db_size_before: u32,
236    pub db_size_after: u32,
237}
238
239/// Run maintenance on the places DB (prune step)
240///
241/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
242/// to clean up / shrink the database.  They're split up so that we can time each one in the
243/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
244/// it supports a stop-watch style API, not recording specific values).
245///
246/// db_size_limit is the approximate storage limit in bytes.  If the database is using more space
247/// than this, some older visits will be deleted to free up space.  Pass in a 0 to skip this.
248///
249/// prune_limit is the maximum number of visits to prune if the database is over db_size_limit
250pub fn run_maintenance_prune(
251    conn: &PlacesDb,
252    db_size_limit: u32,
253    prune_limit: u32,
254) -> Result<RunMaintenanceMetrics> {
255    let db_size_before = conn.get_db_size()?;
256    let should_prune = db_size_limit > 0 && db_size_before > db_size_limit;
257    if should_prune {
258        history::prune_older_visits(conn, prune_limit)?;
259    }
260    let db_size_after = conn.get_db_size()?;
261    Ok(RunMaintenanceMetrics {
262        pruned_visits: should_prune,
263        db_size_before,
264        db_size_after,
265    })
266}
267
268/// Run maintenance on the places DB (vacuum step)
269///
270/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
271/// to clean up / shrink the database.  They're split up so that we can time each one in the
272/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
273/// it supports a stop-watch style API, not recording specific values).
274pub fn run_maintenance_vacuum(conn: &PlacesDb) -> Result<()> {
275    sql_support::maintenance::vacuum(conn)?;
276    Ok(())
277}
278
279/// Run maintenance on the places DB (optimize step)
280///
281/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
282/// to clean up / shrink the database.  They're split up so that we can time each one in the
283/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
284/// it supports a stop-watch style API, not recording specific values).
285pub fn run_maintenance_optimize(conn: &PlacesDb) -> Result<()> {
286    // 0x10012: run ANALYZE on tables that might benefit (0x02), with a row limit to keep
287    // runtime bounded (0x10), including tables not queried during this connection (0x10000).
288    // The 0x10000 bit lets maintenance refresh stats for tables the writer never queries;
289    // desktop added this alongside the Bug 2017227 shutdown fix.
290    conn.execute_one("PRAGMA optimize(0x10012)")?;
291    Ok(())
292}
293
294/// Run maintenance on the places DB (checkpoint step)
295///
296/// The `run_maintenance_*()` functions are intended to be run during idle time and will take steps
297/// to clean up / shrink the database.  They're split up so that we can time each one in the
298/// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and
299/// it supports a stop-watch style API, not recording specific values).
300pub fn run_maintenance_checkpoint(conn: &PlacesDb) -> Result<()> {
301    conn.execute_one("PRAGMA wal_checkpoint(PASSIVE)")?;
302    Ok(())
303}
304
305pub fn update_all_frecencies_at_once(db: &PlacesDb, scope: &SqlInterruptScope) -> Result<()> {
306    let tx = db.begin_transaction()?;
307
308    let need_frecency_update = tx.query_rows_and_then(
309        "SELECT place_id FROM moz_places_stale_frecencies",
310        [],
311        |r| r.get::<_, i64>(0),
312    )?;
313    scope.err_if_interrupted()?;
314    let frecencies = need_frecency_update
315        .iter()
316        .map(|places_id| {
317            scope.err_if_interrupted()?;
318            Ok((
319                *places_id,
320                calculate_frecency(db, &DEFAULT_FRECENCY_SETTINGS, *places_id, Some(false))?,
321            ))
322        })
323        .collect::<Result<Vec<(i64, i32)>>>()?;
324
325    if frecencies.is_empty() {
326        return Ok(());
327    }
328    // Update all frecencies in one fell swoop
329    tx.execute_batch(&format!(
330        "WITH frecencies(id, frecency) AS (
331            VALUES {}
332            )
333            UPDATE moz_places SET
334            frecency = (SELECT frecency FROM frecencies f
335                        WHERE f.id = id)
336            WHERE id IN (SELECT f.id FROM frecencies f)",
337        sql_support::repeat_display(frecencies.len(), ",", |index, f| {
338            let (id, frecency) = frecencies[index];
339            write!(f, "({}, {})", id, frecency)
340        })
341    ))?;
342
343    scope.err_if_interrupted()?;
344
345    // ...And remove them from the stale table.
346    tx.execute_batch(&format!(
347        "DELETE FROM moz_places_stale_frecencies
348         WHERE place_id IN ({})",
349        sql_support::repeat_display(frecencies.len(), ",", |index, f| {
350            let (id, _) = frecencies[index];
351            write!(f, "{}", id)
352        })
353    ))?;
354    tx.commit()?;
355
356    Ok(())
357}
358
359pub(crate) fn put_meta(conn: &Connection, key: &str, value: &dyn ToSql) -> Result<()> {
360    conn.execute_cached(
361        "REPLACE INTO moz_meta (key, value) VALUES (:key, :value)",
362        &[(":key", &key as &dyn ToSql), (":value", value)],
363    )?;
364    Ok(())
365}
366
367pub(crate) fn get_meta<T: FromSql>(db: &PlacesDb, key: &str) -> Result<Option<T>> {
368    let res = db.try_query_one(
369        "SELECT value FROM moz_meta WHERE key = :key",
370        &[(":key", &key)],
371        true,
372    )?;
373    Ok(res)
374}
375
376pub(crate) fn delete_meta(db: &PlacesDb, key: &str) -> Result<()> {
377    db.execute_cached("DELETE FROM moz_meta WHERE key = :key", &[(":key", &key)])?;
378    Ok(())
379}
380
381/// Delete all items in the temp tables we use for staging changes.
382pub fn delete_pending_temp_tables(conn: &PlacesDb) -> Result<()> {
383    conn.execute_batch(
384        "DELETE FROM moz_updateoriginsinsert_temp;
385         DELETE FROM moz_updateoriginsupdate_temp;
386         DELETE FROM moz_updateoriginsdelete_temp;",
387    )?;
388    Ok(())
389}
390
391#[cfg(test)]
392mod tests {
393    use super::*;
394    use crate::api::places_api::test::new_mem_connection;
395    use crate::observation::VisitObservation;
396    use bookmarks::{
397        delete_bookmark, insert_bookmark, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
398        InsertableItem,
399    };
400    use history::apply_observation;
401
402    #[test]
403    fn test_meta() {
404        let conn = new_mem_connection();
405        let value1 = "value 1".to_string();
406        let value2 = "value 2".to_string();
407        assert!(get_meta::<String>(&conn, "foo")
408            .expect("should get")
409            .is_none());
410        put_meta(&conn, "foo", &value1).expect("should put");
411        assert_eq!(
412            get_meta(&conn, "foo").expect("should get new val"),
413            Some(value1)
414        );
415        put_meta(&conn, "foo", &value2).expect("should put an existing value");
416        assert_eq!(get_meta(&conn, "foo").expect("should get"), Some(value2));
417        delete_meta(&conn, "foo").expect("should delete");
418        assert!(get_meta::<String>(&conn, "foo")
419            .expect("should get non-existing")
420            .is_none());
421        delete_meta(&conn, "foo").expect("delete non-existing should work");
422    }
423
424    // Here we try and test that we replicate desktop behaviour, which isn't that obvious.
425    // * create a bookmark
426    // * remove the bookmark - this doesn't remove the place or origin - probably because in
427    //   real browsers there will be visits for the URL existing, but this still smells like
428    //   a bug - see https://bugzilla.mozilla.org/show_bug.cgi?id=1650511#c34
429    // * Arrange for history for that item to be removed, via various means
430    // At this point the origin and place should be removed. The only code (in desktop and here) which
431    // removes places with a foreign_count of zero is that history removal!
432
433    #[test]
434    fn test_removal_delete_visits_between() {
435        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
436            history::delete_visits_between(conn, Timestamp::EARLIEST, Timestamp::now())
437        })
438    }
439
440    #[test]
441    fn test_removal_delete_visits_for() {
442        do_test_removal_places_and_origins(|conn: &PlacesDb, guid: &SyncGuid| {
443            history::delete_visits_for(conn, guid)
444        })
445    }
446
447    #[test]
448    fn test_removal_prune() {
449        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
450            history::prune_older_visits(conn, 6)
451        })
452    }
453
454    #[test]
455    fn test_removal_visit_at_time() {
456        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
457            let url = Url::parse("http://example.com/foo").unwrap();
458            let visit = Timestamp::from(727_747_200_001);
459            history::delete_place_visit_at_time(conn, &url, visit)
460        })
461    }
462
463    #[test]
464    fn test_removal_everything() {
465        do_test_removal_places_and_origins(|conn: &PlacesDb, _guid: &SyncGuid| {
466            history::delete_everything(conn)
467        })
468    }
469
470    // The core test - takes a function which deletes history.
471    fn do_test_removal_places_and_origins<F>(removal_fn: F)
472    where
473        F: FnOnce(&PlacesDb, &SyncGuid) -> Result<()>,
474    {
475        let conn = new_mem_connection();
476        let url = Url::parse("http://example.com/foo").unwrap();
477        let bm = InsertableItem::Bookmark {
478            b: InsertableBookmark {
479                parent_guid: BookmarkRootGuid::Unfiled.into(),
480                position: BookmarkPosition::Append,
481                date_added: None,
482                last_modified: None,
483                guid: None,
484                url: url.clone(),
485                title: Some("the title".into()),
486            },
487        };
488        assert_eq!(
489            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
490                .unwrap(),
491            5
492        ); // our 5 roots.
493        let bookmark_guid = insert_bookmark(&conn, bm).unwrap();
494        let place_guid = fetch_page_info(&conn, &url)
495            .expect("should work")
496            .expect("must exist")
497            .page
498            .guid;
499        // the place should exist with a foreign_count of 1.
500        assert_eq!(
501            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
502                .unwrap(),
503            6
504        ); // our 5 roots + new bookmark
505        assert_eq!(
506            conn.conn_ext_query_one::<i64>(
507                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
508            )
509            .unwrap(),
510            1
511        );
512        // visit the bookmark.
513        assert!(apply_observation(
514            &conn,
515            VisitObservation::new(url)
516                .with_at(Timestamp::from(727_747_200_001))
517                .with_visit_type(VisitType::Link)
518        )
519        .unwrap()
520        .is_some());
521
522        delete_bookmark(&conn, &bookmark_guid).unwrap();
523        assert_eq!(
524            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
525                .unwrap(),
526            5
527        ); // our 5 roots
528           // the place should have no foreign references, but still exists.
529        assert_eq!(
530            conn.conn_ext_query_one::<i64>(
531                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
532            )
533            .unwrap(),
534            0
535        );
536        removal_fn(&conn, &place_guid).expect("removal function should work");
537        assert_eq!(
538            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_places;")
539                .unwrap(),
540            0
541        );
542        assert_eq!(
543            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_origins;")
544                .unwrap(),
545            0
546        );
547    }
548
549    // Similar to the above, but if the bookmark has no visits the place/origin should die
550    // without requiring history removal
551    #[test]
552    fn test_visitless_removal_places_and_origins() {
553        let conn = new_mem_connection();
554        let url = Url::parse("http://example.com/foo").unwrap();
555        let bm = InsertableItem::Bookmark {
556            b: InsertableBookmark {
557                parent_guid: BookmarkRootGuid::Unfiled.into(),
558                position: BookmarkPosition::Append,
559                date_added: None,
560                last_modified: None,
561                guid: None,
562                url,
563                title: Some("the title".into()),
564            },
565        };
566        assert_eq!(
567            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
568                .unwrap(),
569            5
570        ); // our 5 roots.
571        let bookmark_guid = insert_bookmark(&conn, bm).unwrap();
572        // the place should exist with a foreign_count of 1.
573        assert_eq!(
574            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
575                .unwrap(),
576            6
577        ); // our 5 roots + new bookmark
578        assert_eq!(
579            conn.conn_ext_query_one::<i64>(
580                "SELECT foreign_count FROM moz_places WHERE url = \"http://example.com/foo\";"
581            )
582            .unwrap(),
583            1
584        );
585        // Delete it.
586        delete_bookmark(&conn, &bookmark_guid).unwrap();
587        assert_eq!(
588            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_bookmarks;")
589                .unwrap(),
590            5
591        ); // our 5 roots
592           // should be gone from places and origins.
593        assert_eq!(
594            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_places;")
595                .unwrap(),
596            0
597        );
598        assert_eq!(
599            conn.conn_ext_query_one::<i64>("SELECT COUNT(*) FROM moz_origins;")
600                .unwrap(),
601            0
602        );
603    }
604}
places/storage/mod.rs

places/storage/
mod.rs
