places/db/
db.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5use super::schema;
6use crate::api::places_api::ConnectionType;
7use crate::error::*;
8use interrupt_support::{SqlInterruptHandle, SqlInterruptScope};
9use lazy_static::lazy_static;
10use parking_lot::Mutex;
11use rusqlite::{self, Connection, Transaction};
12use sql_support::{
13    open_database::{self, open_database_with_flags, ConnectionInitializer},
14    ConnExt,
15};
16use std::collections::HashMap;
17use std::ops::Deref;
18use std::path::Path;
19
20use std::sync::{
21    atomic::{AtomicI64, Ordering},
22    Arc, RwLock,
23};
24
25pub const MAX_VARIABLE_NUMBER: usize = 999;
26
27lazy_static! {
28    // Each API has a single bookmark change counter shared across all connections.
29    // This hashmap indexes them by the "api id" of the API.
30    pub static ref GLOBAL_BOOKMARK_CHANGE_COUNTERS: RwLock<HashMap<usize, AtomicI64>> = RwLock::new(HashMap::new());
31}
32
33pub struct PlacesInitializer {
34    api_id: usize,
35    conn_type: ConnectionType,
36}
37
38impl PlacesInitializer {
39    #[cfg(test)]
40    pub fn new_for_test() -> Self {
41        Self {
42            api_id: 0,
43            conn_type: ConnectionType::ReadWrite,
44        }
45    }
46}
47
48impl ConnectionInitializer for PlacesInitializer {
49    const NAME: &'static str = "places";
50    const END_VERSION: u32 = schema::VERSION;
51
52    fn init(&self, tx: &Transaction<'_>) -> open_database::Result<()> {
53        Ok(schema::init(tx)?)
54    }
55
56    fn upgrade_from(&self, tx: &Transaction<'_>, version: u32) -> open_database::Result<()> {
57        Ok(schema::upgrade_from(tx, version)?)
58    }
59
60    fn prepare(&self, conn: &Connection, db_empty: bool) -> open_database::Result<()> {
61        // If this is an empty DB, setup incremental auto-vacuum now rather than wait for the first
62        // run_maintenance_vacuum() call.  It should be much faster now with an empty DB.
63        if db_empty && !matches!(self.conn_type, ConnectionType::ReadOnly) {
64            conn.execute_one("PRAGMA auto_vacuum=incremental")?;
65            conn.execute_one("VACUUM")?;
66        }
67
68        let initial_pragmas = "
69            -- The value we use was taken from Desktop Firefox, and seems necessary to
70            -- help ensure good performance on autocomplete-style queries.
71            -- Modern default value is 4096, but as reported in
72            -- https://bugzilla.mozilla.org/show_bug.cgi?id=1782283, desktop places saw
73            -- a nice improvement with this value.
74            PRAGMA page_size = 32768;
75
76            -- Disable calling mlock/munlock for every malloc/free.
77            -- In practice this results in a massive speedup, especially
78            -- for insert-heavy workloads.
79            PRAGMA cipher_memory_security = false;
80
81            -- `temp_store = 2` is required on Android to force the DB to keep temp
82            -- files in memory, since on Android there's no tmp partition. See
83            -- https://github.com/mozilla/mentat/issues/505. Ideally we'd only
84            -- do this on Android, and/or allow caller to configure it.
85            -- (although see also bug 1313021, where Firefox enabled it for both
86            -- Android and 64bit desktop builds)
87            PRAGMA temp_store = 2;
88
89            -- 6MiB, same as the value used for `promiseLargeCacheDBConnection` in PlacesUtils,
90            -- which is used to improve query performance for autocomplete-style queries (by
91            -- UnifiedComplete). Note that SQLite uses a negative value for this pragma to indicate
92            -- that it's in units of KiB.
93            PRAGMA cache_size = -6144;
94
95            -- We want foreign-key support.
96            PRAGMA foreign_keys = ON;
97
98            -- we unconditionally want write-ahead-logging mode
99            PRAGMA journal_mode=WAL;
100
101            -- How often to autocheckpoint (in units of pages).
102            -- 2048000 (our max desired WAL size) / 32760 (page size).
103            PRAGMA wal_autocheckpoint=62;
104
105            -- How long to wait for a lock before returning SQLITE_BUSY (in ms)
106            -- See `doc/sql_concurrency.md` for details.
107            PRAGMA busy_timeout = 5000;
108        ";
109        conn.execute_batch(initial_pragmas)?;
110        define_functions(conn, self.api_id)?;
111        sql_support::debug_tools::define_debug_functions(conn)?;
112        conn.set_prepared_statement_cache_capacity(128);
113        Ok(())
114    }
115
116    fn finish(&self, conn: &Connection) -> open_database::Result<()> {
117        Ok(schema::finish(conn, self.conn_type)?)
118    }
119}
120
121#[derive(Debug)]
122pub struct PlacesDb {
123    pub db: Connection,
124    conn_type: ConnectionType,
125    interrupt_handle: Arc<SqlInterruptHandle>,
126    api_id: usize,
127    pub(super) coop_tx_lock: Arc<Mutex<()>>,
128}
129
130impl PlacesDb {
131    fn with_connection(
132        db: Connection,
133        conn_type: ConnectionType,
134        api_id: usize,
135        coop_tx_lock: Arc<Mutex<()>>,
136    ) -> Self {
137        Self {
138            interrupt_handle: Arc::new(SqlInterruptHandle::new(&db)),
139            db,
140            conn_type,
141            // The API sets this explicitly.
142            api_id,
143            coop_tx_lock,
144        }
145    }
146
147    pub fn open(
148        path: impl AsRef<Path>,
149        conn_type: ConnectionType,
150        api_id: usize,
151        coop_tx_lock: Arc<Mutex<()>>,
152    ) -> Result<Self> {
153        let initializer = PlacesInitializer { api_id, conn_type };
154        let conn = open_database_with_flags(path, conn_type.rusqlite_flags(), &initializer)?;
155        Ok(Self::with_connection(conn, conn_type, api_id, coop_tx_lock))
156    }
157
158    #[cfg(test)]
159    // Useful for some tests (although most tests should use helper functions
160    // in api::places_api::test)
161    pub fn open_in_memory(conn_type: ConnectionType) -> Result<Self> {
162        let initializer = PlacesInitializer {
163            api_id: 0,
164            conn_type,
165        };
166        let conn = open_database::open_memory_database_with_flags(
167            conn_type.rusqlite_flags(),
168            &initializer,
169        )?;
170        Ok(Self::with_connection(
171            conn,
172            conn_type,
173            0,
174            Arc::new(Mutex::new(())),
175        ))
176    }
177
178    pub fn new_interrupt_handle(&self) -> Arc<SqlInterruptHandle> {
179        Arc::clone(&self.interrupt_handle)
180    }
181
182    #[inline]
183    pub fn begin_interrupt_scope(&self) -> Result<SqlInterruptScope> {
184        Ok(self.interrupt_handle.begin_interrupt_scope()?)
185    }
186
187    #[inline]
188    pub fn conn_type(&self) -> ConnectionType {
189        self.conn_type
190    }
191
192    /// Returns an object that can tell you whether any changes have been made
193    /// to bookmarks since this was called.
194    /// While this conceptually should live on the PlacesApi, the things that
195    /// need this typically only have a PlacesDb, so we expose it here.
196    pub fn global_bookmark_change_tracker(&self) -> GlobalChangeCounterTracker {
197        GlobalChangeCounterTracker::new(self.api_id)
198    }
199
200    #[inline]
201    pub fn api_id(&self) -> usize {
202        self.api_id
203    }
204}
205
206impl Drop for PlacesDb {
207    fn drop(&mut self) {
208        // In line with both the recommendations from SQLite and the behavior of places in
209        // Database.cpp, we run `PRAGMA optimize` before closing the connection.
210        if let ConnectionType::ReadOnly = self.conn_type() {
211            // A reader connection can't execute an optimize
212            return;
213        }
214        // The 0x12 flags mean: run ANALYZE on tables that might benefit (0x02), with a row
215        // limit to keep runtime bounded (0x10). Mirrors the flags used on desktop since
216        // Bug 2017227.
217        let res = self.db.execute_batch("PRAGMA optimize(0x12);");
218        if let Err(e) = res {
219            warn!("Failed to execute pragma optimize (DB locked?): {}", e);
220        }
221    }
222}
223
224impl ConnExt for PlacesDb {
225    #[inline]
226    fn conn(&self) -> &Connection {
227        &self.db
228    }
229}
230
231impl Deref for PlacesDb {
232    type Target = Connection;
233    #[inline]
234    fn deref(&self) -> &Connection {
235        &self.db
236    }
237}
238
239/// PlacesDB that's behind a Mutex so it can be shared between threads
240pub struct SharedPlacesDb {
241    db: Mutex<PlacesDb>,
242    interrupt_handle: Arc<SqlInterruptHandle>,
243}
244
245impl SharedPlacesDb {
246    pub fn new(db: PlacesDb) -> Self {
247        Self {
248            interrupt_handle: db.new_interrupt_handle(),
249            db: Mutex::new(db),
250        }
251    }
252
253    pub fn begin_interrupt_scope(&self) -> Result<SqlInterruptScope> {
254        Ok(self.interrupt_handle.begin_interrupt_scope()?)
255    }
256}
257
258// Deref to a Mutex<PlacesDb>, which is how we will use SharedPlacesDb most of the time
259impl Deref for SharedPlacesDb {
260    type Target = Mutex<PlacesDb>;
261
262    #[inline]
263    fn deref(&self) -> &Mutex<PlacesDb> {
264        &self.db
265    }
266}
267
268// Also implement AsRef<SqlInterruptHandle> so that we can interrupt this at shutdown
269impl AsRef<SqlInterruptHandle> for SharedPlacesDb {
270    fn as_ref(&self) -> &SqlInterruptHandle {
271        &self.interrupt_handle
272    }
273}
274
275/// An object that can tell you whether a bookmark changing operation has
276/// happened since the object was created.
277pub struct GlobalChangeCounterTracker {
278    api_id: usize,
279    start_value: i64,
280}
281
282impl GlobalChangeCounterTracker {
283    pub fn new(api_id: usize) -> Self {
284        GlobalChangeCounterTracker {
285            api_id,
286            start_value: Self::cur_value(api_id),
287        }
288    }
289
290    // The value is an implementation detail, so just expose what we care
291    // about - ie, "has it changed?"
292    pub fn changed(&self) -> bool {
293        Self::cur_value(self.api_id) != self.start_value
294    }
295
296    fn cur_value(api_id: usize) -> i64 {
297        let map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
298            .read()
299            .expect("gbcc poisoned");
300        match map.get(&api_id) {
301            Some(counter) => counter.load(Ordering::Acquire),
302            None => 0,
303        }
304    }
305}
306
307#[derive(Clone, Copy)]
308pub enum Pragma {
309    IgnoreCheckConstraints,
310    ForeignKeys,
311    WritableSchema,
312}
313
314impl Pragma {
315    pub fn name(&self) -> &str {
316        match self {
317            Self::IgnoreCheckConstraints => "ignore_check_constraints",
318            Self::ForeignKeys => "foreign_keys",
319            Self::WritableSchema => "writable_schema",
320        }
321    }
322}
323
324/// A scope guard that sets a Boolean PRAGMA to a new value, and
325/// restores the inverse of the value when dropped.
326pub struct PragmaGuard<'a> {
327    conn: &'a Connection,
328    pragma: Pragma,
329    old_value: bool,
330}
331
332impl<'a> PragmaGuard<'a> {
333    pub fn new(conn: &'a Connection, pragma: Pragma, new_value: bool) -> rusqlite::Result<Self> {
334        conn.pragma_update(
335            None,
336            pragma.name(),
337            match new_value {
338                true => "ON",
339                false => "OFF",
340            },
341        )?;
342        Ok(Self {
343            conn,
344            pragma,
345            old_value: !new_value,
346        })
347    }
348}
349
350impl Drop for PragmaGuard<'_> {
351    fn drop(&mut self) {
352        let _ = self.conn.pragma_update(
353            None,
354            self.pragma.name(),
355            match self.old_value {
356                true => "ON",
357                false => "OFF",
358            },
359        );
360    }
361}
362
363fn define_functions(c: &Connection, api_id: usize) -> rusqlite::Result<()> {
364    use rusqlite::functions::FunctionFlags;
365    c.create_scalar_function(
366        "get_prefix",
367        1,
368        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
369        sql_fns::get_prefix,
370    )?;
371    c.create_scalar_function(
372        "get_host_and_port",
373        1,
374        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
375        sql_fns::get_host_and_port,
376    )?;
377    c.create_scalar_function(
378        "strip_prefix_and_userinfo",
379        1,
380        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
381        sql_fns::strip_prefix_and_userinfo,
382    )?;
383    c.create_scalar_function(
384        "reverse_host",
385        1,
386        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
387        sql_fns::reverse_host,
388    )?;
389    c.create_scalar_function(
390        "autocomplete_match",
391        10,
392        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
393        sql_fns::autocomplete_match,
394    )?;
395    c.create_scalar_function(
396        "hash",
397        -1,
398        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
399        sql_fns::hash,
400    )?;
401    c.create_scalar_function("now", 0, FunctionFlags::SQLITE_UTF8, sql_fns::now)?;
402    c.create_scalar_function(
403        "generate_guid",
404        0,
405        FunctionFlags::SQLITE_UTF8,
406        sql_fns::generate_guid,
407    )?;
408    c.create_scalar_function(
409        "note_bookmarks_sync_change",
410        0,
411        FunctionFlags::SQLITE_UTF8,
412        move |ctx| -> rusqlite::Result<i64> { sql_fns::note_bookmarks_sync_change(ctx, api_id) },
413    )?;
414    c.create_scalar_function("throw", 1, FunctionFlags::SQLITE_UTF8, move |ctx| {
415        sql_fns::throw(ctx, api_id)
416    })?;
417    Ok(())
418}
419
420pub(crate) mod sql_fns {
421    use super::GLOBAL_BOOKMARK_CHANGE_COUNTERS;
422    use crate::api::matcher::{split_after_host_and_port, split_after_prefix};
423    use crate::hash;
424    use crate::match_impl::{AutocompleteMatch, MatchBehavior, SearchBehavior};
425    use rusqlite::types::Null;
426    use rusqlite::{functions::Context, types::ValueRef, Error, Result};
427    use std::sync::atomic::Ordering;
428    use sync_guid::Guid as SyncGuid;
429    use types::Timestamp;
430
431    // Helpers for define_functions
432    fn get_raw_str<'a>(ctx: &'a Context<'_>, fname: &'static str, idx: usize) -> Result<&'a str> {
433        ctx.get_raw(idx).as_str().map_err(|e| {
434            Error::UserFunctionError(format!("Bad arg {} to '{}': {}", idx, fname, e).into())
435        })
436    }
437
438    fn get_raw_opt_str<'a>(
439        ctx: &'a Context<'_>,
440        fname: &'static str,
441        idx: usize,
442    ) -> Result<Option<&'a str>> {
443        let raw = ctx.get_raw(idx);
444        if raw == ValueRef::Null {
445            return Ok(None);
446        }
447        Ok(Some(raw.as_str().map_err(|e| {
448            Error::UserFunctionError(format!("Bad arg {} to '{}': {}", idx, fname, e).into())
449        })?))
450    }
451
452    // Note: The compiler can't meaningfully inline these, but if we don't put
453    // #[inline(never)] on them they get "inlined" into a temporary Box<FnMut>,
454    // which doesn't have a name (and itself doesn't get inlined). Adding
455    // #[inline(never)] ensures they show up in profiles.
456
457    #[inline(never)]
458    pub fn hash(ctx: &Context<'_>) -> rusqlite::Result<Option<i64>> {
459        Ok(match ctx.len() {
460            1 => {
461                // This is a deterministic function, which means sqlite
462                // does certain optimizations which means hash() may be called
463                // with a null value even though the query prevents the null
464                // value from actually being used. As a special case, we return
465                // null when the input is NULL. We return NULL instead of zero
466                // because the hash columns are NOT NULL, so attempting to
467                // actually use the null should fail.
468                get_raw_opt_str(ctx, "hash", 0)?.map(|value| hash::hash_url(value) as i64)
469            }
470            2 => {
471                let value = get_raw_opt_str(ctx, "hash", 0)?;
472                let mode = get_raw_str(ctx, "hash", 1)?;
473                if let Some(value) = value {
474                    Some(match mode {
475                        "" => hash::hash_url(value),
476                        "prefix_lo" => hash::hash_url_prefix(value, hash::PrefixMode::Lo),
477                        "prefix_hi" => hash::hash_url_prefix(value, hash::PrefixMode::Hi),
478                        arg => {
479                            return Err(rusqlite::Error::UserFunctionError(format!(
480                                "`hash` second argument must be either '', 'prefix_lo', or 'prefix_hi', got {:?}.",
481                                arg).into()));
482                        }
483                    } as i64)
484                } else {
485                    None
486                }
487            }
488            n => {
489                return Err(rusqlite::Error::UserFunctionError(
490                    format!("`hash` expects 1 or 2 arguments, got {}.", n).into(),
491                ));
492            }
493        })
494    }
495
496    #[inline(never)]
497    pub fn autocomplete_match(ctx: &Context<'_>) -> Result<bool> {
498        let search_str = get_raw_str(ctx, "autocomplete_match", 0)?;
499        let url_str = get_raw_str(ctx, "autocomplete_match", 1)?;
500        let title_str = get_raw_opt_str(ctx, "autocomplete_match", 2)?.unwrap_or_default();
501        let tags = get_raw_opt_str(ctx, "autocomplete_match", 3)?.unwrap_or_default();
502        let visit_count = ctx.get::<u32>(4)?;
503        let typed = ctx.get::<bool>(5)?;
504        let bookmarked = ctx.get::<bool>(6)?;
505        let open_page_count = ctx.get::<Option<u32>>(7)?.unwrap_or(0);
506        let match_behavior = ctx.get::<MatchBehavior>(8)?;
507        let search_behavior = ctx.get::<SearchBehavior>(9)?;
508
509        let matcher = AutocompleteMatch {
510            search_str,
511            url_str,
512            title_str,
513            tags,
514            visit_count,
515            typed,
516            bookmarked,
517            open_page_count,
518            match_behavior,
519            search_behavior,
520        };
521        Ok(matcher.invoke())
522    }
523
524    #[inline(never)]
525    pub fn reverse_host(ctx: &Context<'_>) -> Result<String> {
526        // We reuse this memory so no need for get_raw.
527        let mut host = ctx.get::<String>(0)?;
528        debug_assert!(host.is_ascii(), "Hosts must be Punycoded");
529
530        host.make_ascii_lowercase();
531        let mut rev_host_bytes = host.into_bytes();
532        rev_host_bytes.reverse();
533        rev_host_bytes.push(b'.');
534
535        let rev_host = String::from_utf8(rev_host_bytes).map_err(|_err| {
536            rusqlite::Error::UserFunctionError("non-punycode host provided to reverse_host!".into())
537        })?;
538        Ok(rev_host)
539    }
540
541    #[inline(never)]
542    pub fn get_prefix(ctx: &Context<'_>) -> Result<String> {
543        let href = get_raw_str(ctx, "get_prefix", 0)?;
544        let (prefix, _) = split_after_prefix(href);
545        Ok(prefix.to_owned())
546    }
547
548    #[inline(never)]
549    pub fn get_host_and_port(ctx: &Context<'_>) -> Result<String> {
550        let href = get_raw_str(ctx, "get_host_and_port", 0)?;
551        let (host_and_port, _) = split_after_host_and_port(href);
552        Ok(host_and_port.to_owned())
553    }
554
555    #[inline(never)]
556    pub fn strip_prefix_and_userinfo(ctx: &Context<'_>) -> Result<String> {
557        let href = get_raw_str(ctx, "strip_prefix_and_userinfo", 0)?;
558        let (host_and_port, remainder) = split_after_host_and_port(href);
559        let mut res = String::with_capacity(host_and_port.len() + remainder.len() + 1);
560        res += host_and_port;
561        res += remainder;
562        Ok(res)
563    }
564
565    #[inline(never)]
566    pub fn now(_ctx: &Context<'_>) -> Result<Timestamp> {
567        Ok(Timestamp::now())
568    }
569
570    #[inline(never)]
571    pub fn generate_guid(_ctx: &Context<'_>) -> Result<SyncGuid> {
572        Ok(SyncGuid::random())
573    }
574
575    #[inline(never)]
576    pub fn throw(ctx: &Context<'_>, api_id: usize) -> Result<Null> {
577        Err(rusqlite::Error::UserFunctionError(
578            format!("{} (#{})", ctx.get::<String>(0)?, api_id).into(),
579        ))
580    }
581
582    #[inline(never)]
583    pub fn note_bookmarks_sync_change(_ctx: &Context<'_>, api_id: usize) -> Result<i64> {
584        let map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
585            .read()
586            .expect("gbcc poisoned");
587        if let Some(counter) = map.get(&api_id) {
588            // Because we only ever check for equality, we can use Relaxed ordering.
589            return Ok(counter.fetch_add(1, Ordering::Relaxed));
590        }
591        // Need to add the counter to the map - drop the read lock before
592        // taking the write lock.
593        drop(map);
594        let mut map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
595            .write()
596            .expect("gbcc poisoned");
597        let counter = map.entry(api_id).or_default();
598        // Because we only ever check for equality, we can use Relaxed ordering.
599        Ok(counter.fetch_add(1, Ordering::Relaxed))
600    }
601}
602
603#[cfg(test)]
604mod tests {
605    use super::*;
606
607    // Sanity check that we can create a database.
608    #[test]
609    fn test_open() {
610        PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
611    }
612
613    #[test]
614    fn test_reverse_host() {
615        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
616        let rev_host: String = conn
617            .db
618            .query_row("SELECT reverse_host('www.mozilla.org')", [], |row| {
619                row.get(0)
620            })
621            .unwrap();
622        assert_eq!(rev_host, "gro.allizom.www.");
623
624        let rev_host: String = conn
625            .db
626            .query_row("SELECT reverse_host('')", [], |row| row.get(0))
627            .unwrap();
628        assert_eq!(rev_host, ".");
629    }
630}
places/db/db.rs

places/db/
db.rs
