places/db/
db.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5use super::schema;
6use crate::api::places_api::ConnectionType;
7use crate::error::*;
8use interrupt_support::{SqlInterruptHandle, SqlInterruptScope};
9use lazy_static::lazy_static;
10use parking_lot::Mutex;
11use rusqlite::{self, Connection, Transaction};
12use sql_support::{
13    open_database::{self, open_database_with_flags, ConnectionInitializer},
14    ConnExt,
15};
16use std::collections::HashMap;
17use std::ops::Deref;
18use std::path::Path;
19
20use std::sync::{
21    atomic::{AtomicI64, Ordering},
22    Arc, RwLock,
23};
24
25pub const MAX_VARIABLE_NUMBER: usize = 999;
26
27lazy_static! {
28    // Each API has a single bookmark change counter shared across all connections.
29    // This hashmap indexes them by the "api id" of the API.
30    pub static ref GLOBAL_BOOKMARK_CHANGE_COUNTERS: RwLock<HashMap<usize, AtomicI64>> = RwLock::new(HashMap::new());
31}
32
33pub struct PlacesInitializer {
34    api_id: usize,
35    conn_type: ConnectionType,
36}
37
38impl PlacesInitializer {
39    #[cfg(test)]
40    pub fn new_for_test() -> Self {
41        Self {
42            api_id: 0,
43            conn_type: ConnectionType::ReadWrite,
44        }
45    }
46}
47
48impl ConnectionInitializer for PlacesInitializer {
49    const NAME: &'static str = "places";
50    const END_VERSION: u32 = schema::VERSION;
51
52    fn init(&self, tx: &Transaction<'_>) -> open_database::Result<()> {
53        Ok(schema::init(tx)?)
54    }
55
56    fn upgrade_from(&self, tx: &Transaction<'_>, version: u32) -> open_database::Result<()> {
57        Ok(schema::upgrade_from(tx, version)?)
58    }
59
60    fn prepare(&self, conn: &Connection, _db_empty: bool) -> open_database::Result<()> {
61        let initial_pragmas = "
62            -- The value we use was taken from Desktop Firefox, and seems necessary to
63            -- help ensure good performance on autocomplete-style queries.
64            -- Modern default value is 4096, but as reported in
65            -- https://bugzilla.mozilla.org/show_bug.cgi?id=1782283, desktop places saw
66            -- a nice improvement with this value.
67            PRAGMA page_size = 32768;
68
69            -- Disable calling mlock/munlock for every malloc/free.
70            -- In practice this results in a massive speedup, especially
71            -- for insert-heavy workloads.
72            PRAGMA cipher_memory_security = false;
73
74            -- `temp_store = 2` is required on Android to force the DB to keep temp
75            -- files in memory, since on Android there's no tmp partition. See
76            -- https://github.com/mozilla/mentat/issues/505. Ideally we'd only
77            -- do this on Android, and/or allow caller to configure it.
78            -- (although see also bug 1313021, where Firefox enabled it for both
79            -- Android and 64bit desktop builds)
80            PRAGMA temp_store = 2;
81
82            -- 6MiB, same as the value used for `promiseLargeCacheDBConnection` in PlacesUtils,
83            -- which is used to improve query performance for autocomplete-style queries (by
84            -- UnifiedComplete). Note that SQLite uses a negative value for this pragma to indicate
85            -- that it's in units of KiB.
86            PRAGMA cache_size = -6144;
87
88            -- We want foreign-key support.
89            PRAGMA foreign_keys = ON;
90
91            -- we unconditionally want write-ahead-logging mode
92            PRAGMA journal_mode=WAL;
93
94            -- How often to autocheckpoint (in units of pages).
95            -- 2048000 (our max desired WAL size) / 32760 (page size).
96            PRAGMA wal_autocheckpoint=62;
97
98            -- How long to wait for a lock before returning SQLITE_BUSY (in ms)
99            -- See `doc/sql_concurrency.md` for details.
100            PRAGMA busy_timeout = 5000;
101        ";
102        conn.execute_batch(initial_pragmas)?;
103        define_functions(conn, self.api_id)?;
104        sql_support::debug_tools::define_debug_functions(conn)?;
105        conn.set_prepared_statement_cache_capacity(128);
106        Ok(())
107    }
108
109    fn finish(&self, conn: &Connection) -> open_database::Result<()> {
110        Ok(schema::finish(conn, self.conn_type)?)
111    }
112}
113
114#[derive(Debug)]
115pub struct PlacesDb {
116    pub db: Connection,
117    conn_type: ConnectionType,
118    interrupt_handle: Arc<SqlInterruptHandle>,
119    api_id: usize,
120    pub(super) coop_tx_lock: Arc<Mutex<()>>,
121}
122
123impl PlacesDb {
124    fn with_connection(
125        db: Connection,
126        conn_type: ConnectionType,
127        api_id: usize,
128        coop_tx_lock: Arc<Mutex<()>>,
129    ) -> Self {
130        Self {
131            interrupt_handle: Arc::new(SqlInterruptHandle::new(&db)),
132            db,
133            conn_type,
134            // The API sets this explicitly.
135            api_id,
136            coop_tx_lock,
137        }
138    }
139
140    pub fn open(
141        path: impl AsRef<Path>,
142        conn_type: ConnectionType,
143        api_id: usize,
144        coop_tx_lock: Arc<Mutex<()>>,
145    ) -> Result<Self> {
146        let initializer = PlacesInitializer { api_id, conn_type };
147        let conn = open_database_with_flags(path, conn_type.rusqlite_flags(), &initializer)?;
148        Ok(Self::with_connection(conn, conn_type, api_id, coop_tx_lock))
149    }
150
151    #[cfg(test)]
152    // Useful for some tests (although most tests should use helper functions
153    // in api::places_api::test)
154    pub fn open_in_memory(conn_type: ConnectionType) -> Result<Self> {
155        let initializer = PlacesInitializer {
156            api_id: 0,
157            conn_type,
158        };
159        let conn = open_database::open_memory_database_with_flags(
160            conn_type.rusqlite_flags(),
161            &initializer,
162        )?;
163        Ok(Self::with_connection(
164            conn,
165            conn_type,
166            0,
167            Arc::new(Mutex::new(())),
168        ))
169    }
170
171    pub fn new_interrupt_handle(&self) -> Arc<SqlInterruptHandle> {
172        Arc::clone(&self.interrupt_handle)
173    }
174
175    #[inline]
176    pub fn begin_interrupt_scope(&self) -> Result<SqlInterruptScope> {
177        Ok(self.interrupt_handle.begin_interrupt_scope()?)
178    }
179
180    #[inline]
181    pub fn conn_type(&self) -> ConnectionType {
182        self.conn_type
183    }
184
185    /// Returns an object that can tell you whether any changes have been made
186    /// to bookmarks since this was called.
187    /// While this conceptually should live on the PlacesApi, the things that
188    /// need this typically only have a PlacesDb, so we expose it here.
189    pub fn global_bookmark_change_tracker(&self) -> GlobalChangeCounterTracker {
190        GlobalChangeCounterTracker::new(self.api_id)
191    }
192
193    #[inline]
194    pub fn api_id(&self) -> usize {
195        self.api_id
196    }
197}
198
199impl Drop for PlacesDb {
200    fn drop(&mut self) {
201        // In line with both the recommendations from SQLite and the behavior of places in
202        // Database.cpp, we run `PRAGMA optimize` before closing the connection.
203        if let ConnectionType::ReadOnly = self.conn_type() {
204            // A reader connection can't execute an optimize
205            return;
206        }
207        // The 0x12 flags mean: run ANALYZE on tables that might benefit (0x02), with a row
208        // limit to keep runtime bounded (0x10). Mirrors the flags used on desktop since
209        // Bug 2017227.
210        let res = self.db.execute_batch("PRAGMA optimize(0x12);");
211        if let Err(e) = res {
212            warn!("Failed to execute pragma optimize (DB locked?): {}", e);
213        }
214    }
215}
216
217impl ConnExt for PlacesDb {
218    #[inline]
219    fn conn(&self) -> &Connection {
220        &self.db
221    }
222}
223
224impl Deref for PlacesDb {
225    type Target = Connection;
226    #[inline]
227    fn deref(&self) -> &Connection {
228        &self.db
229    }
230}
231
232/// PlacesDB that's behind a Mutex so it can be shared between threads
233pub struct SharedPlacesDb {
234    db: Mutex<PlacesDb>,
235    interrupt_handle: Arc<SqlInterruptHandle>,
236}
237
238impl SharedPlacesDb {
239    pub fn new(db: PlacesDb) -> Self {
240        Self {
241            interrupt_handle: db.new_interrupt_handle(),
242            db: Mutex::new(db),
243        }
244    }
245
246    pub fn begin_interrupt_scope(&self) -> Result<SqlInterruptScope> {
247        Ok(self.interrupt_handle.begin_interrupt_scope()?)
248    }
249}
250
251// Deref to a Mutex<PlacesDb>, which is how we will use SharedPlacesDb most of the time
252impl Deref for SharedPlacesDb {
253    type Target = Mutex<PlacesDb>;
254
255    #[inline]
256    fn deref(&self) -> &Mutex<PlacesDb> {
257        &self.db
258    }
259}
260
261// Also implement AsRef<SqlInterruptHandle> so that we can interrupt this at shutdown
262impl AsRef<SqlInterruptHandle> for SharedPlacesDb {
263    fn as_ref(&self) -> &SqlInterruptHandle {
264        &self.interrupt_handle
265    }
266}
267
268/// An object that can tell you whether a bookmark changing operation has
269/// happened since the object was created.
270pub struct GlobalChangeCounterTracker {
271    api_id: usize,
272    start_value: i64,
273}
274
275impl GlobalChangeCounterTracker {
276    pub fn new(api_id: usize) -> Self {
277        GlobalChangeCounterTracker {
278            api_id,
279            start_value: Self::cur_value(api_id),
280        }
281    }
282
283    // The value is an implementation detail, so just expose what we care
284    // about - ie, "has it changed?"
285    pub fn changed(&self) -> bool {
286        Self::cur_value(self.api_id) != self.start_value
287    }
288
289    fn cur_value(api_id: usize) -> i64 {
290        let map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
291            .read()
292            .expect("gbcc poisoned");
293        match map.get(&api_id) {
294            Some(counter) => counter.load(Ordering::Acquire),
295            None => 0,
296        }
297    }
298}
299
300#[derive(Clone, Copy)]
301pub enum Pragma {
302    IgnoreCheckConstraints,
303    ForeignKeys,
304    WritableSchema,
305}
306
307impl Pragma {
308    pub fn name(&self) -> &str {
309        match self {
310            Self::IgnoreCheckConstraints => "ignore_check_constraints",
311            Self::ForeignKeys => "foreign_keys",
312            Self::WritableSchema => "writable_schema",
313        }
314    }
315}
316
317/// A scope guard that sets a Boolean PRAGMA to a new value, and
318/// restores the inverse of the value when dropped.
319pub struct PragmaGuard<'a> {
320    conn: &'a Connection,
321    pragma: Pragma,
322    old_value: bool,
323}
324
325impl<'a> PragmaGuard<'a> {
326    pub fn new(conn: &'a Connection, pragma: Pragma, new_value: bool) -> rusqlite::Result<Self> {
327        conn.pragma_update(
328            None,
329            pragma.name(),
330            match new_value {
331                true => "ON",
332                false => "OFF",
333            },
334        )?;
335        Ok(Self {
336            conn,
337            pragma,
338            old_value: !new_value,
339        })
340    }
341}
342
343impl Drop for PragmaGuard<'_> {
344    fn drop(&mut self) {
345        let _ = self.conn.pragma_update(
346            None,
347            self.pragma.name(),
348            match self.old_value {
349                true => "ON",
350                false => "OFF",
351            },
352        );
353    }
354}
355
356fn define_functions(c: &Connection, api_id: usize) -> rusqlite::Result<()> {
357    use rusqlite::functions::FunctionFlags;
358    c.create_scalar_function(
359        "get_prefix",
360        1,
361        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
362        sql_fns::get_prefix,
363    )?;
364    c.create_scalar_function(
365        "get_host_and_port",
366        1,
367        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
368        sql_fns::get_host_and_port,
369    )?;
370    c.create_scalar_function(
371        "strip_prefix_and_userinfo",
372        1,
373        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
374        sql_fns::strip_prefix_and_userinfo,
375    )?;
376    c.create_scalar_function(
377        "reverse_host",
378        1,
379        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
380        sql_fns::reverse_host,
381    )?;
382    c.create_scalar_function(
383        "autocomplete_match",
384        10,
385        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
386        sql_fns::autocomplete_match,
387    )?;
388    c.create_scalar_function(
389        "hash",
390        -1,
391        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
392        sql_fns::hash,
393    )?;
394    c.create_scalar_function("now", 0, FunctionFlags::SQLITE_UTF8, sql_fns::now)?;
395    c.create_scalar_function(
396        "generate_guid",
397        0,
398        FunctionFlags::SQLITE_UTF8,
399        sql_fns::generate_guid,
400    )?;
401    c.create_scalar_function(
402        "note_bookmarks_sync_change",
403        0,
404        FunctionFlags::SQLITE_UTF8,
405        move |ctx| -> rusqlite::Result<i64> { sql_fns::note_bookmarks_sync_change(ctx, api_id) },
406    )?;
407    c.create_scalar_function("throw", 1, FunctionFlags::SQLITE_UTF8, move |ctx| {
408        sql_fns::throw(ctx, api_id)
409    })?;
410    Ok(())
411}
412
413pub(crate) mod sql_fns {
414    use super::GLOBAL_BOOKMARK_CHANGE_COUNTERS;
415    use crate::api::matcher::{split_after_host_and_port, split_after_prefix};
416    use crate::hash;
417    use crate::match_impl::{AutocompleteMatch, MatchBehavior, SearchBehavior};
418    use rusqlite::types::Null;
419    use rusqlite::{functions::Context, types::ValueRef, Error, Result};
420    use std::sync::atomic::Ordering;
421    use sync_guid::Guid as SyncGuid;
422    use types::Timestamp;
423
424    // Helpers for define_functions
425    fn get_raw_str<'a>(ctx: &'a Context<'_>, fname: &'static str, idx: usize) -> Result<&'a str> {
426        ctx.get_raw(idx).as_str().map_err(|e| {
427            Error::UserFunctionError(format!("Bad arg {} to '{}': {}", idx, fname, e).into())
428        })
429    }
430
431    fn get_raw_opt_str<'a>(
432        ctx: &'a Context<'_>,
433        fname: &'static str,
434        idx: usize,
435    ) -> Result<Option<&'a str>> {
436        let raw = ctx.get_raw(idx);
437        if raw == ValueRef::Null {
438            return Ok(None);
439        }
440        Ok(Some(raw.as_str().map_err(|e| {
441            Error::UserFunctionError(format!("Bad arg {} to '{}': {}", idx, fname, e).into())
442        })?))
443    }
444
445    // Note: The compiler can't meaningfully inline these, but if we don't put
446    // #[inline(never)] on them they get "inlined" into a temporary Box<FnMut>,
447    // which doesn't have a name (and itself doesn't get inlined). Adding
448    // #[inline(never)] ensures they show up in profiles.
449
450    #[inline(never)]
451    pub fn hash(ctx: &Context<'_>) -> rusqlite::Result<Option<i64>> {
452        Ok(match ctx.len() {
453            1 => {
454                // This is a deterministic function, which means sqlite
455                // does certain optimizations which means hash() may be called
456                // with a null value even though the query prevents the null
457                // value from actually being used. As a special case, we return
458                // null when the input is NULL. We return NULL instead of zero
459                // because the hash columns are NOT NULL, so attempting to
460                // actually use the null should fail.
461                get_raw_opt_str(ctx, "hash", 0)?.map(|value| hash::hash_url(value) as i64)
462            }
463            2 => {
464                let value = get_raw_opt_str(ctx, "hash", 0)?;
465                let mode = get_raw_str(ctx, "hash", 1)?;
466                if let Some(value) = value {
467                    Some(match mode {
468                        "" => hash::hash_url(value),
469                        "prefix_lo" => hash::hash_url_prefix(value, hash::PrefixMode::Lo),
470                        "prefix_hi" => hash::hash_url_prefix(value, hash::PrefixMode::Hi),
471                        arg => {
472                            return Err(rusqlite::Error::UserFunctionError(format!(
473                                "`hash` second argument must be either '', 'prefix_lo', or 'prefix_hi', got {:?}.",
474                                arg).into()));
475                        }
476                    } as i64)
477                } else {
478                    None
479                }
480            }
481            n => {
482                return Err(rusqlite::Error::UserFunctionError(
483                    format!("`hash` expects 1 or 2 arguments, got {}.", n).into(),
484                ));
485            }
486        })
487    }
488
489    #[inline(never)]
490    pub fn autocomplete_match(ctx: &Context<'_>) -> Result<bool> {
491        let search_str = get_raw_str(ctx, "autocomplete_match", 0)?;
492        let url_str = get_raw_str(ctx, "autocomplete_match", 1)?;
493        let title_str = get_raw_opt_str(ctx, "autocomplete_match", 2)?.unwrap_or_default();
494        let tags = get_raw_opt_str(ctx, "autocomplete_match", 3)?.unwrap_or_default();
495        let visit_count = ctx.get::<u32>(4)?;
496        let typed = ctx.get::<bool>(5)?;
497        let bookmarked = ctx.get::<bool>(6)?;
498        let open_page_count = ctx.get::<Option<u32>>(7)?.unwrap_or(0);
499        let match_behavior = ctx.get::<MatchBehavior>(8)?;
500        let search_behavior = ctx.get::<SearchBehavior>(9)?;
501
502        let matcher = AutocompleteMatch {
503            search_str,
504            url_str,
505            title_str,
506            tags,
507            visit_count,
508            typed,
509            bookmarked,
510            open_page_count,
511            match_behavior,
512            search_behavior,
513        };
514        Ok(matcher.invoke())
515    }
516
517    #[inline(never)]
518    pub fn reverse_host(ctx: &Context<'_>) -> Result<String> {
519        // We reuse this memory so no need for get_raw.
520        let mut host = ctx.get::<String>(0)?;
521        debug_assert!(host.is_ascii(), "Hosts must be Punycoded");
522
523        host.make_ascii_lowercase();
524        let mut rev_host_bytes = host.into_bytes();
525        rev_host_bytes.reverse();
526        rev_host_bytes.push(b'.');
527
528        let rev_host = String::from_utf8(rev_host_bytes).map_err(|_err| {
529            rusqlite::Error::UserFunctionError("non-punycode host provided to reverse_host!".into())
530        })?;
531        Ok(rev_host)
532    }
533
534    #[inline(never)]
535    pub fn get_prefix(ctx: &Context<'_>) -> Result<String> {
536        let href = get_raw_str(ctx, "get_prefix", 0)?;
537        let (prefix, _) = split_after_prefix(href);
538        Ok(prefix.to_owned())
539    }
540
541    #[inline(never)]
542    pub fn get_host_and_port(ctx: &Context<'_>) -> Result<String> {
543        let href = get_raw_str(ctx, "get_host_and_port", 0)?;
544        let (host_and_port, _) = split_after_host_and_port(href);
545        Ok(host_and_port.to_owned())
546    }
547
548    #[inline(never)]
549    pub fn strip_prefix_and_userinfo(ctx: &Context<'_>) -> Result<String> {
550        let href = get_raw_str(ctx, "strip_prefix_and_userinfo", 0)?;
551        let (host_and_port, remainder) = split_after_host_and_port(href);
552        let mut res = String::with_capacity(host_and_port.len() + remainder.len() + 1);
553        res += host_and_port;
554        res += remainder;
555        Ok(res)
556    }
557
558    #[inline(never)]
559    pub fn now(_ctx: &Context<'_>) -> Result<Timestamp> {
560        Ok(Timestamp::now())
561    }
562
563    #[inline(never)]
564    pub fn generate_guid(_ctx: &Context<'_>) -> Result<SyncGuid> {
565        Ok(SyncGuid::random())
566    }
567
568    #[inline(never)]
569    pub fn throw(ctx: &Context<'_>, api_id: usize) -> Result<Null> {
570        Err(rusqlite::Error::UserFunctionError(
571            format!("{} (#{})", ctx.get::<String>(0)?, api_id).into(),
572        ))
573    }
574
575    #[inline(never)]
576    pub fn note_bookmarks_sync_change(_ctx: &Context<'_>, api_id: usize) -> Result<i64> {
577        let map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
578            .read()
579            .expect("gbcc poisoned");
580        if let Some(counter) = map.get(&api_id) {
581            // Because we only ever check for equality, we can use Relaxed ordering.
582            return Ok(counter.fetch_add(1, Ordering::Relaxed));
583        }
584        // Need to add the counter to the map - drop the read lock before
585        // taking the write lock.
586        drop(map);
587        let mut map = GLOBAL_BOOKMARK_CHANGE_COUNTERS
588            .write()
589            .expect("gbcc poisoned");
590        let counter = map.entry(api_id).or_default();
591        // Because we only ever check for equality, we can use Relaxed ordering.
592        Ok(counter.fetch_add(1, Ordering::Relaxed))
593    }
594}
595
596#[cfg(test)]
597mod tests {
598    use super::*;
599
600    // Sanity check that we can create a database.
601    #[test]
602    fn test_open() {
603        PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
604    }
605
606    #[test]
607    fn test_reverse_host() {
608        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
609        let rev_host: String = conn
610            .db
611            .query_row("SELECT reverse_host('www.mozilla.org')", [], |row| {
612                row.get(0)
613            })
614            .unwrap();
615        assert_eq!(rev_host, "gro.allizom.www.");
616
617        let rev_host: String = conn
618            .db
619            .query_row("SELECT reverse_host('')", [], |row| row.get(0))
620            .unwrap();
621        assert_eq!(rev_host, ".");
622    }
623}
places/db/db.rs

places/db/
db.rs
