places/db/
db.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5use super::schema;
6use crate::api::places_api::ConnectionType;
7use crate::error::*;
8use interrupt_support::{SqlInterruptHandle, SqlInterruptScope};
9use lazy_static::lazy_static;
10use parking_lot::Mutex;
11use rusqlite::{self, Connection, Transaction};
12use sql_support::{
13    open_database::{self, open_database_with_flags, ConnectionInitializer},
14    ConnExt,
15};
16use std::collections::HashMap;
17use std::ops::Deref;
18use std::path::Path;
19
20use std::sync::{
21    atomic::{AtomicI64, Ordering},
22    Arc, RwLock,
23};
24
25pub const MAX_VARIABLE_NUMBER: usize = 999;
26
27lazy_static! {
28    // Each API has a single bookmark change counter shared across all connections.
29    // This hashmap indexes them by the "api id" of the API.
30    pub static ref GLOBAL_BOOKMARK_CHANGE_COUNTERS: RwLock<HashMap<usize, AtomicI64>> = RwLock::new(HashMap::new());
31}
32
33pub struct PlacesInitializer {
34    api_id: usize,
35    conn_type: ConnectionType,
36}
37
38impl PlacesInitializer {
39    #[cfg(test)]
40    pub fn new_for_test() -> Self {
41        Self {
42            api_id: 0,
43            conn_type: ConnectionType::ReadWrite,
44        }
45    }
46}
47
48impl ConnectionInitializer for PlacesInitializer {
49    const NAME: &'static str = "places";
50    const END_VERSION: u32 = schema::VERSION;
51
52    fn init(&self, tx: &Transaction<'_>) -> open_database::Result<()> {
53        Ok(schema::init(tx)?)
54    }
55
56    fn upgrade_from(&self, tx: &Transaction<'_>, version: u32) -> open_database::Result<()> {
57        Ok(schema::upgrade_from(tx, version)?)
58    }
59
60    fn prepare(&self, conn: &Connection, db_empty: bool) -> open_database::Result<()> {
61        // If this is an empty DB, setup incremental auto-vacuum now rather than wait for the first
62        // run_maintenance_vacuum() call.  It should be much faster now with an empty DB.
63        if db_empty && !matches!(self.conn_type, ConnectionType::ReadOnly) {
64            conn.execute_one("PRAGMA auto_vacuum=incremental")?;
65            conn.execute_one("VACUUM")?;
66        }
67
68        let initial_pragmas = "
69            -- The value we use was taken from Desktop Firefox, and seems necessary to
70            -- help ensure good performance on autocomplete-style queries.
71            -- Modern default value is 4096, but as reported in
72            -- https://bugzilla.mozilla.org/show_bug.cgi?id=1782283, desktop places saw
73            -- a nice improvement with this value.
74            PRAGMA page_size = 32768;
75
76            -- Disable calling mlock/munlock for every malloc/free.
77            -- In practice this results in a massive speedup, especially
78            -- for insert-heavy workloads.
79            PRAGMA cipher_memory_security = false;
80
81            -- `temp_store = 2` is required on Android to force the DB to keep temp
82            -- files in memory, since on Android there's no tmp partition. See
83            -- https://github.com/mozilla/mentat/issues/505. Ideally we'd only
84            -- do this on Android, and/or allow caller to configure it.
85            -- (although see also bug 1313021, where Firefox enabled it for both
86            -- Android and 64bit desktop builds)
87            PRAGMA temp_store = 2;
88
89            -- 6MiB, same as the value used for `promiseLargeCacheDBConnection` in PlacesUtils,
90            -- which is used to improve query performance for autocomplete-style queries (by
91            -- UnifiedComplete). Note that SQLite uses a negative value for this pragma to indicate
92            -- that it's in units of KiB.
93            PRAGMA cache_size = -6144;
94
95            -- We want foreign-key support.
96            PRAGMA foreign_keys = ON;
97
98            -- we unconditionally want write-ahead-logging mode
99            PRAGMA journal_mode=WAL;
100
101            -- How often to autocheckpoint (in units of pages).
102            -- 2048000 (our max desired WAL size) / 32760 (page size).
103            PRAGMA wal_autocheckpoint=62;
104
105            -- How long to wait for a lock before returning SQLITE_BUSY (in ms)
106            -- See `doc/sql_concurrency.md` for details.
107            PRAGMA busy_timeout = 5000;
108        ";
109        conn.execute_batch(initial_pragmas)?;
110        define_functions(conn, self.api_id)?;
111        sql_support::debug_tools::define_debug_functions(conn)?;
112        conn.set_prepared_statement_cache_capacity(128);
113        Ok(())
114    }
115
116    fn finish(&self, conn: &Connection) -> open_database::Result<()> {
117        Ok(schema::finish(conn, self.conn_type)?)
118    }
119}
120
121#[derive(Debug)]
122pub struct PlacesDb {
123    pub db: Connection,
124    conn_type: ConnectionType,
125    interrupt_handle: Arc<SqlInterruptHandle>,
126    api_id: usize,
127    pub(super) coop_tx_lock: Arc<Mutex<()>>,
128}
129
130impl PlacesDb {
131    fn with_connection(
132        db: Connection,
133        conn_type: ConnectionType,
134        api_id: usize,
135        coop_tx_lock: Arc<Mutex<()>>,
136    ) -> Self {
137        Self {
138            interrupt_handle: Arc::new(SqlInterruptHandle::new(&db)),
139            db,
140            conn_type,
141            // The API sets this explicitly.
142            api_id,
143            coop_tx_lock,
144        }
145    }
146
147    pub fn open(
148        path: impl AsRef<Path>,
149        conn_type: ConnectionType,
150        api_id: usize,
151        coop_tx_lock: Arc<Mutex<()>>,
152    ) -> Result<Self> {
153        let initializer = PlacesInitializer { api_id, conn_type };
154        let conn = open_database_with_flags(path, conn_type.rusqlite_flags(), &initializer)?;
155        Ok(Self::with_connection(conn, conn_type, api_id, coop_tx_lock))
156    }
157
158    #[cfg(test)]
159    // Useful for some tests (although most tests should use helper functions
160    // in api::places_api::test)
161    pub fn open_in_memory(conn_type: ConnectionType) -> Result<Self> {
162        let initializer = PlacesInitializer {
163            api_id: 0,
164            conn_type,
165        };
166        let conn = open_database::open_memory_database_with_flags(
167            conn_type.rusqlite_flags(),
168            &initializer,
169        )?;
170        Ok(Self::with_connection(
171            conn,
172            conn_type,
173            0,
174            Arc::new(Mutex::new(())),
175        ))
176    }
177
178    pub fn new_interrupt_handle(&self) -> Arc<SqlInterruptHandle> {
179        Arc::clone(&self.interrupt_handle)
180    }
181
182    #[inline]
183    pub fn begin_interrupt_scope(&self) -> Result<SqlInterruptScope> {
184        Ok(self.interrupt_handle.begin_interrupt_scope()?)
185    }
186
187    #[inline]
188    pub fn conn_type(&self) -> ConnectionType {
189        self.conn_type
190    }
191
192    /// Returns an object that can tell you whether any changes have been made
193    /// to bookmarks since this was called.
194    /// While this conceptually should live on the PlacesApi, the things that
195    /// need this typically only have a PlacesDb, so we expose it here.
196    pub fn global_bookmark_change_tracker(&self) -> GlobalChangeCounterTracker {
197        GlobalChangeCounterTracker::new(self.api_id)
198    }
199
200    #[inline]
201    pub fn api_id(&self) -> usize {
202        self.api_id
203    }
204}
205
206impl Drop for PlacesDb {
207    fn drop(&mut self) {
208        // In line with both the recommendations from SQLite and the behavior of places in
209        // Database.cpp, we run `PRAGMA optimize` before closing the connection.
210        if let ConnectionType::ReadOnly = self.conn_type() {
211            // A reader connection can't execute an optimize
212            return;
213        }
214        let res = self.db.execute_batch("PRAGMA optimize(0x02);");
215        if let Err(e) = res {
216            warn!("Failed to execute pragma optimize (DB locked?): {}", e);
217        }
218    }
219}
220
221impl ConnExt for PlacesDb {
222    #[inline]
223    fn conn(&self) -> &Connection {
224        &self.db
225    }
226}
227
228impl Deref for PlacesDb {
229    type Target = Connection;
230    #[inline]
231    fn deref(&self) -> &Connection {
232        &self.db
233    }
234}
235
236/// PlacesDB that's behind a Mutex so it can be shared between threads
237pub struct SharedPlacesDb {
238    db: Mutex<PlacesDb>,
239    interrupt_handle: Arc<SqlInterruptHandle>,
240}
241
242impl SharedPlacesDb {
243    pub fn new(db: PlacesDb) -> Self {
244        Self {
245            interrupt_handle: db.new_interrupt_handle(),
246            db: Mutex::new(db),
247        }
248    }
249
250    pub fn begin_interrupt_scope(&self) -> Result<SqlInterruptScope> {
251        Ok(self.interrupt_handle.begin_interrupt_scope()?)
252    }
253}
254
255// Deref to a Mutex<PlacesDb>, which is how we will use SharedPlacesDb most of the time
256impl Deref for SharedPlacesDb {
257    type Target = Mutex<PlacesDb>;
258
259    #[inline]
260    fn deref(&self) -> &Mutex<PlacesDb> {
261        &self.db
262    }
263}
264
265// Also implement AsRef<SqlInterruptHandle> so that we can interrupt this at shutdown
266impl AsRef<SqlInterruptHandle> for SharedPlacesDb {
267    fn as_ref(&self) -> &SqlInterruptHandle {
268        &self.interrupt_handle
269    }
270}
271
272/// An object that can tell you whether a bookmark changing operation has
273/// happened since the object was created.
274pub struct GlobalChangeCounterTracker {
275    api_id: usize,
276    start_value: i64,
277}
278
279impl GlobalChangeCounterTracker {
280    pub fn new(api_id: usize) -> Self {
281        GlobalChangeCounterTracker {
282            api_id,
283            start_value: Self::cur_value(api_id),
284        }
285    }
286
287    // The value is an implementation detail, so just expose what we care
288    // about - ie, "has it changed?"
289    pub fn changed(&self) -> bool {
290        Self::cur_value(self.api_id) != self.start_value
291    }
292
293    fn cur_value(api_id: usize) -> i64 {
294        let map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
295            .read()
296            .expect("gbcc poisoned");
297        match map.get(&api_id) {
298            Some(counter) => counter.load(Ordering::Acquire),
299            None => 0,
300        }
301    }
302}
303
304#[derive(Clone, Copy)]
305pub enum Pragma {
306    IgnoreCheckConstraints,
307    ForeignKeys,
308    WritableSchema,
309}
310
311impl Pragma {
312    pub fn name(&self) -> &str {
313        match self {
314            Self::IgnoreCheckConstraints => "ignore_check_constraints",
315            Self::ForeignKeys => "foreign_keys",
316            Self::WritableSchema => "writable_schema",
317        }
318    }
319}
320
321/// A scope guard that sets a Boolean PRAGMA to a new value, and
322/// restores the inverse of the value when dropped.
323pub struct PragmaGuard<'a> {
324    conn: &'a Connection,
325    pragma: Pragma,
326    old_value: bool,
327}
328
329impl<'a> PragmaGuard<'a> {
330    pub fn new(conn: &'a Connection, pragma: Pragma, new_value: bool) -> rusqlite::Result<Self> {
331        conn.pragma_update(
332            None,
333            pragma.name(),
334            match new_value {
335                true => "ON",
336                false => "OFF",
337            },
338        )?;
339        Ok(Self {
340            conn,
341            pragma,
342            old_value: !new_value,
343        })
344    }
345}
346
347impl Drop for PragmaGuard<'_> {
348    fn drop(&mut self) {
349        let _ = self.conn.pragma_update(
350            None,
351            self.pragma.name(),
352            match self.old_value {
353                true => "ON",
354                false => "OFF",
355            },
356        );
357    }
358}
359
360fn define_functions(c: &Connection, api_id: usize) -> rusqlite::Result<()> {
361    use rusqlite::functions::FunctionFlags;
362    c.create_scalar_function(
363        "get_prefix",
364        1,
365        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
366        sql_fns::get_prefix,
367    )?;
368    c.create_scalar_function(
369        "get_host_and_port",
370        1,
371        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
372        sql_fns::get_host_and_port,
373    )?;
374    c.create_scalar_function(
375        "strip_prefix_and_userinfo",
376        1,
377        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
378        sql_fns::strip_prefix_and_userinfo,
379    )?;
380    c.create_scalar_function(
381        "reverse_host",
382        1,
383        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
384        sql_fns::reverse_host,
385    )?;
386    c.create_scalar_function(
387        "autocomplete_match",
388        10,
389        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
390        sql_fns::autocomplete_match,
391    )?;
392    c.create_scalar_function(
393        "hash",
394        -1,
395        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
396        sql_fns::hash,
397    )?;
398    c.create_scalar_function("now", 0, FunctionFlags::SQLITE_UTF8, sql_fns::now)?;
399    c.create_scalar_function(
400        "generate_guid",
401        0,
402        FunctionFlags::SQLITE_UTF8,
403        sql_fns::generate_guid,
404    )?;
405    c.create_scalar_function(
406        "note_bookmarks_sync_change",
407        0,
408        FunctionFlags::SQLITE_UTF8,
409        move |ctx| -> rusqlite::Result<i64> { sql_fns::note_bookmarks_sync_change(ctx, api_id) },
410    )?;
411    c.create_scalar_function("throw", 1, FunctionFlags::SQLITE_UTF8, move |ctx| {
412        sql_fns::throw(ctx, api_id)
413    })?;
414    Ok(())
415}
416
417pub(crate) mod sql_fns {
418    use super::GLOBAL_BOOKMARK_CHANGE_COUNTERS;
419    use crate::api::matcher::{split_after_host_and_port, split_after_prefix};
420    use crate::hash;
421    use crate::match_impl::{AutocompleteMatch, MatchBehavior, SearchBehavior};
422    use rusqlite::types::Null;
423    use rusqlite::{functions::Context, types::ValueRef, Error, Result};
424    use std::sync::atomic::Ordering;
425    use sync_guid::Guid as SyncGuid;
426    use types::Timestamp;
427
428    // Helpers for define_functions
429    fn get_raw_str<'a>(ctx: &'a Context<'_>, fname: &'static str, idx: usize) -> Result<&'a str> {
430        ctx.get_raw(idx).as_str().map_err(|e| {
431            Error::UserFunctionError(format!("Bad arg {} to '{}': {}", idx, fname, e).into())
432        })
433    }
434
435    fn get_raw_opt_str<'a>(
436        ctx: &'a Context<'_>,
437        fname: &'static str,
438        idx: usize,
439    ) -> Result<Option<&'a str>> {
440        let raw = ctx.get_raw(idx);
441        if raw == ValueRef::Null {
442            return Ok(None);
443        }
444        Ok(Some(raw.as_str().map_err(|e| {
445            Error::UserFunctionError(format!("Bad arg {} to '{}': {}", idx, fname, e).into())
446        })?))
447    }
448
449    // Note: The compiler can't meaningfully inline these, but if we don't put
450    // #[inline(never)] on them they get "inlined" into a temporary Box<FnMut>,
451    // which doesn't have a name (and itself doesn't get inlined). Adding
452    // #[inline(never)] ensures they show up in profiles.
453
454    #[inline(never)]
455    pub fn hash(ctx: &Context<'_>) -> rusqlite::Result<Option<i64>> {
456        Ok(match ctx.len() {
457            1 => {
458                // This is a deterministic function, which means sqlite
459                // does certain optimizations which means hash() may be called
460                // with a null value even though the query prevents the null
461                // value from actually being used. As a special case, we return
462                // null when the input is NULL. We return NULL instead of zero
463                // because the hash columns are NOT NULL, so attempting to
464                // actually use the null should fail.
465                get_raw_opt_str(ctx, "hash", 0)?.map(|value| hash::hash_url(value) as i64)
466            }
467            2 => {
468                let value = get_raw_opt_str(ctx, "hash", 0)?;
469                let mode = get_raw_str(ctx, "hash", 1)?;
470                if let Some(value) = value {
471                    Some(match mode {
472                        "" => hash::hash_url(value),
473                        "prefix_lo" => hash::hash_url_prefix(value, hash::PrefixMode::Lo),
474                        "prefix_hi" => hash::hash_url_prefix(value, hash::PrefixMode::Hi),
475                        arg => {
476                            return Err(rusqlite::Error::UserFunctionError(format!(
477                                "`hash` second argument must be either '', 'prefix_lo', or 'prefix_hi', got {:?}.",
478                                arg).into()));
479                        }
480                    } as i64)
481                } else {
482                    None
483                }
484            }
485            n => {
486                return Err(rusqlite::Error::UserFunctionError(
487                    format!("`hash` expects 1 or 2 arguments, got {}.", n).into(),
488                ));
489            }
490        })
491    }
492
493    #[inline(never)]
494    pub fn autocomplete_match(ctx: &Context<'_>) -> Result<bool> {
495        let search_str = get_raw_str(ctx, "autocomplete_match", 0)?;
496        let url_str = get_raw_str(ctx, "autocomplete_match", 1)?;
497        let title_str = get_raw_opt_str(ctx, "autocomplete_match", 2)?.unwrap_or_default();
498        let tags = get_raw_opt_str(ctx, "autocomplete_match", 3)?.unwrap_or_default();
499        let visit_count = ctx.get::<u32>(4)?;
500        let typed = ctx.get::<bool>(5)?;
501        let bookmarked = ctx.get::<bool>(6)?;
502        let open_page_count = ctx.get::<Option<u32>>(7)?.unwrap_or(0);
503        let match_behavior = ctx.get::<MatchBehavior>(8)?;
504        let search_behavior = ctx.get::<SearchBehavior>(9)?;
505
506        let matcher = AutocompleteMatch {
507            search_str,
508            url_str,
509            title_str,
510            tags,
511            visit_count,
512            typed,
513            bookmarked,
514            open_page_count,
515            match_behavior,
516            search_behavior,
517        };
518        Ok(matcher.invoke())
519    }
520
521    #[inline(never)]
522    pub fn reverse_host(ctx: &Context<'_>) -> Result<String> {
523        // We reuse this memory so no need for get_raw.
524        let mut host = ctx.get::<String>(0)?;
525        debug_assert!(host.is_ascii(), "Hosts must be Punycoded");
526
527        host.make_ascii_lowercase();
528        let mut rev_host_bytes = host.into_bytes();
529        rev_host_bytes.reverse();
530        rev_host_bytes.push(b'.');
531
532        let rev_host = String::from_utf8(rev_host_bytes).map_err(|_err| {
533            rusqlite::Error::UserFunctionError("non-punycode host provided to reverse_host!".into())
534        })?;
535        Ok(rev_host)
536    }
537
538    #[inline(never)]
539    pub fn get_prefix(ctx: &Context<'_>) -> Result<String> {
540        let href = get_raw_str(ctx, "get_prefix", 0)?;
541        let (prefix, _) = split_after_prefix(href);
542        Ok(prefix.to_owned())
543    }
544
545    #[inline(never)]
546    pub fn get_host_and_port(ctx: &Context<'_>) -> Result<String> {
547        let href = get_raw_str(ctx, "get_host_and_port", 0)?;
548        let (host_and_port, _) = split_after_host_and_port(href);
549        Ok(host_and_port.to_owned())
550    }
551
552    #[inline(never)]
553    pub fn strip_prefix_and_userinfo(ctx: &Context<'_>) -> Result<String> {
554        let href = get_raw_str(ctx, "strip_prefix_and_userinfo", 0)?;
555        let (host_and_port, remainder) = split_after_host_and_port(href);
556        let mut res = String::with_capacity(host_and_port.len() + remainder.len() + 1);
557        res += host_and_port;
558        res += remainder;
559        Ok(res)
560    }
561
562    #[inline(never)]
563    pub fn now(_ctx: &Context<'_>) -> Result<Timestamp> {
564        Ok(Timestamp::now())
565    }
566
567    #[inline(never)]
568    pub fn generate_guid(_ctx: &Context<'_>) -> Result<SyncGuid> {
569        Ok(SyncGuid::random())
570    }
571
572    #[inline(never)]
573    pub fn throw(ctx: &Context<'_>, api_id: usize) -> Result<Null> {
574        Err(rusqlite::Error::UserFunctionError(
575            format!("{} (#{})", ctx.get::<String>(0)?, api_id).into(),
576        ))
577    }
578
579    #[inline(never)]
580    pub fn note_bookmarks_sync_change(_ctx: &Context<'_>, api_id: usize) -> Result<i64> {
581        let map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
582            .read()
583            .expect("gbcc poisoned");
584        if let Some(counter) = map.get(&api_id) {
585            // Because we only ever check for equality, we can use Relaxed ordering.
586            return Ok(counter.fetch_add(1, Ordering::Relaxed));
587        }
588        // Need to add the counter to the map - drop the read lock before
589        // taking the write lock.
590        drop(map);
591        let mut map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
592            .write()
593            .expect("gbcc poisoned");
594        let counter = map.entry(api_id).or_default();
595        // Because we only ever check for equality, we can use Relaxed ordering.
596        Ok(counter.fetch_add(1, Ordering::Relaxed))
597    }
598}
599
600#[cfg(test)]
601mod tests {
602    use super::*;
603
604    // Sanity check that we can create a database.
605    #[test]
606    fn test_open() {
607        PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
608    }
609
610    #[test]
611    fn test_reverse_host() {
612        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
613        let rev_host: String = conn
614            .db
615            .query_row("SELECT reverse_host('www.mozilla.org')", [], |row| {
616                row.get(0)
617            })
618            .unwrap();
619        assert_eq!(rev_host, "gro.allizom.www.");
620
621        let rev_host: String = conn
622            .db
623            .query_row("SELECT reverse_host('')", [], |row| row.get(0))
624            .unwrap();
625        assert_eq!(rev_host, ".");
626    }
627}