1use std::{
33 borrow::Cow,
34 path::Path,
35 sync::atomic::{AtomicUsize, Ordering},
36};
37
38use rusqlite::{
39 Connection, Error as RusqliteError, ErrorCode, OpenFlags, Transaction, TransactionBehavior,
40};
41use thiserror::Error;
42
43use crate::ConnExt;
44use crate::{debug, info, warn};
45
46#[derive(Error, Debug)]
47pub enum Error {
48 #[error("Incompatible database version: {0}")]
49 IncompatibleVersion(u32),
50 #[error("Database is corrupt")]
51 Corrupt,
52 #[error("Error executing SQL: {0}")]
53 SqlError(rusqlite::Error),
54 #[error("Failed to recover a corrupt database due to an error deleting the file: {0}")]
55 RecoveryError(std::io::Error),
56 #[error("In shutdown mode")]
57 Shutdown,
58}
59
60impl From<rusqlite::Error> for Error {
61 fn from(value: rusqlite::Error) -> Self {
62 match value {
63 RusqliteError::SqliteFailure(e, _)
64 if matches!(e.code, ErrorCode::DatabaseCorrupt | ErrorCode::NotADatabase) =>
65 {
66 Self::Corrupt
67 }
68 _ => Self::SqlError(value),
69 }
70 }
71}
72
73pub type Result<T> = std::result::Result<T, Error>;
74
75pub trait ConnectionInitializer {
76 const NAME: &'static str;
78
79 const END_VERSION: u32;
81
82 fn init(&self, tx: &Transaction<'_>) -> Result<()>;
85
86 fn upgrade_from(&self, conn: &Transaction<'_>, version: u32) -> Result<()>;
88
89 fn prepare(&self, _conn: &Connection, _db_empty: bool) -> Result<()> {
92 Ok(())
93 }
94
95 fn finish(&self, _conn: &Connection) -> Result<()> {
99 Ok(())
100 }
101}
102
103pub fn open_database<CI: ConnectionInitializer, P: AsRef<Path>>(
104 path: P,
105 connection_initializer: &CI,
106) -> Result<Connection> {
107 open_database_with_flags(path, OpenFlags::default(), connection_initializer)
108}
109
110pub fn open_memory_database<CI: ConnectionInitializer>(
111 conn_initializer: &CI,
112) -> Result<Connection> {
113 open_memory_database_with_flags(OpenFlags::default(), conn_initializer)
114}
115
116pub fn open_database_with_flags<CI: ConnectionInitializer, P: AsRef<Path>>(
117 path: P,
118 open_flags: OpenFlags,
119 connection_initializer: &CI,
120) -> Result<Connection> {
121 do_open_database_with_flags(&path, open_flags, connection_initializer).or_else(|e| {
122 try_handle_db_failure(&path, open_flags, connection_initializer, e)?;
124 do_open_database_with_flags(&path, open_flags, connection_initializer)
125 })
126}
127
128pub fn read_write_flags() -> OpenFlags {
130 OpenFlags::SQLITE_OPEN_URI
131 | OpenFlags::SQLITE_OPEN_NO_MUTEX
132 | OpenFlags::SQLITE_OPEN_CREATE
133 | OpenFlags::SQLITE_OPEN_READ_WRITE
134}
135
136pub fn read_only_flags() -> OpenFlags {
138 OpenFlags::SQLITE_OPEN_URI | OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY
139}
140
141fn do_open_database_with_flags<CI: ConnectionInitializer, P: AsRef<Path>>(
142 path: P,
143 open_flags: OpenFlags,
144 connection_initializer: &CI,
145) -> Result<Connection> {
146 debug!("{}: opening database", CI::NAME);
148 let mut conn = Connection::open_with_flags(path, open_flags)?;
149 debug!("{}: checking if initialization is necessary", CI::NAME);
150 let db_empty = is_db_empty(&conn)?;
151
152 debug!("{}: preparing", CI::NAME);
153 connection_initializer.prepare(&conn, db_empty)?;
154
155 if open_flags.contains(OpenFlags::SQLITE_OPEN_READ_WRITE) {
156 let mut write_schema_version = true;
157 if db_empty {
158 init_for_maintenance(&conn)?;
160 }
161 let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
162 if db_empty {
163 debug!("{}: initializing new database", CI::NAME);
164 connection_initializer.init(&tx)?;
165 } else {
166 let mut current_version = get_schema_version(&tx)?;
167 if current_version > CI::END_VERSION {
168 return Err(Error::IncompatibleVersion(current_version));
169 } else if current_version == CI::END_VERSION {
170 write_schema_version = false;
171 } else {
172 while current_version < CI::END_VERSION {
173 debug!(
174 "{}: upgrading database to {}",
175 CI::NAME,
176 current_version + 1
177 );
178 connection_initializer.upgrade_from(&tx, current_version)?;
179 current_version += 1;
180 }
181 }
182 }
183 debug!("{}: finishing writable database open", CI::NAME);
184 connection_initializer.finish(&tx)?;
185 if write_schema_version {
186 set_schema_version(&tx, CI::END_VERSION)?;
187 }
188 tx.commit()?;
189 } else {
190 assert!(!db_empty, "existing writer must have initialized");
193 assert!(
194 get_schema_version(&conn)? == CI::END_VERSION,
195 "existing writer must have migrated"
196 );
197 debug!("{}: finishing readonly database open", CI::NAME);
198 connection_initializer.finish(&conn)?;
199 }
200 debug!("{}: database open successful", CI::NAME);
201 Ok(conn)
202}
203
204pub fn open_memory_database_with_flags<CI: ConnectionInitializer>(
205 flags: OpenFlags,
206 conn_initializer: &CI,
207) -> Result<Connection> {
208 open_database_with_flags(":memory:", flags, conn_initializer)
209}
210
211fn init_for_maintenance(conn: &Connection) -> Result<()> {
212 conn.execute_one("PRAGMA auto_vacuum=incremental")?;
220 conn.execute_one("VACUUM")?;
224 Ok(())
225}
226
227fn try_handle_db_failure<CI: ConnectionInitializer, P: AsRef<Path>>(
233 path: P,
234 open_flags: OpenFlags,
235 _connection_initializer: &CI,
236 err: Error,
237) -> Result<()> {
238 if !open_flags.contains(OpenFlags::SQLITE_OPEN_CREATE)
239 && matches!(err, Error::SqlError(rusqlite::Error::SqliteFailure(code, _)) if code.code == rusqlite::ErrorCode::CannotOpen)
240 {
241 info!(
242 "{}: database doesn't exist, but we weren't requested to create it",
243 CI::NAME
244 );
245 return Err(err);
246 }
247 warn!("{}: database operation failed: {}", CI::NAME, err);
248 if !open_flags.contains(OpenFlags::SQLITE_OPEN_READ_WRITE) {
249 warn!(
250 "{}: not attempting recovery as this is a read-only connection request",
251 CI::NAME
252 );
253 return Err(err);
254 }
255
256 let delete = matches!(err, Error::Corrupt);
257 if delete {
258 info!(
259 "{}: the database is fatally damaged; deleting and starting fresh",
260 CI::NAME
261 );
262 if let Err(io_err) = std::fs::remove_file(path) {
266 return Err(Error::RecoveryError(io_err));
267 }
268 Ok(())
269 } else {
270 Err(err)
271 }
272}
273
274fn is_db_empty(conn: &Connection) -> Result<bool> {
275 Ok(conn.conn_ext_query_one::<u32>("SELECT COUNT(*) FROM sqlite_master")? == 0)
276}
277
278fn get_schema_version(conn: &Connection) -> Result<u32> {
279 let version = conn.query_row_and_then("PRAGMA user_version", [], |row| row.get(0))?;
280 Ok(version)
281}
282
283fn set_schema_version(conn: &Connection, version: u32) -> Result<()> {
284 conn.set_pragma("user_version", version)?;
285 Ok(())
286}
287
288pub fn unique_in_memory_db_path() -> String {
292 static COUNTER: AtomicUsize = AtomicUsize::new(0);
293 format!(
294 "file:in-memory-db-{}?mode=memory&cache=shared",
295 COUNTER.fetch_add(1, Ordering::Relaxed)
296 )
297}
298
299pub mod test_utils {
302 use super::*;
303 use std::{
304 cell::RefCell,
305 collections::{HashMap, HashSet},
306 path::PathBuf,
307 };
308 use tempfile::TempDir;
309
310 pub struct TestConnectionInitializer {
311 pub calls: RefCell<Vec<&'static str>>,
312 pub buggy_v3_upgrade: bool,
313 }
314
315 impl Default for TestConnectionInitializer {
316 fn default() -> Self {
317 Self::new()
318 }
319 }
320
321 impl TestConnectionInitializer {
322 pub fn new() -> Self {
323 Self {
324 calls: RefCell::new(Vec::new()),
325 buggy_v3_upgrade: false,
326 }
327 }
328 pub fn new_with_buggy_logic() -> Self {
329 Self {
330 calls: RefCell::new(Vec::new()),
331 buggy_v3_upgrade: true,
332 }
333 }
334
335 pub fn clear_calls(&self) {
336 self.calls.borrow_mut().clear();
337 }
338
339 pub fn push_call(&self, call: &'static str) {
340 self.calls.borrow_mut().push(call);
341 }
342
343 pub fn check_calls(&self, expected: Vec<&'static str>) {
344 assert_eq!(*self.calls.borrow(), expected);
345 }
346 }
347
348 impl ConnectionInitializer for TestConnectionInitializer {
349 const NAME: &'static str = "test db";
350 const END_VERSION: u32 = 4;
351
352 fn prepare(&self, conn: &Connection, _: bool) -> Result<()> {
353 self.push_call("prep");
354 conn.execute_batch(
355 "
356 PRAGMA journal_mode = wal;
357 ",
358 )?;
359 Ok(())
360 }
361
362 fn init(&self, conn: &Transaction<'_>) -> Result<()> {
363 self.push_call("init");
364 conn.execute_batch(
365 "
366 CREATE TABLE prep_table(col);
367 INSERT INTO prep_table(col) VALUES ('correct-value');
368 CREATE TABLE my_table(col);
369 ",
370 )
371 .map_err(|e| e.into())
372 }
373
374 fn upgrade_from(&self, conn: &Transaction<'_>, version: u32) -> Result<()> {
375 match version {
376 1 => {
379 self.push_call("upgrade_from_v1");
380 Err(Error::Corrupt)
381 }
382 2 => {
383 self.push_call("upgrade_from_v2");
384 conn.execute_batch(
385 "
386 ALTER TABLE my_old_table_name RENAME TO my_table;
387 ",
388 )?;
389 Ok(())
390 }
391 3 => {
392 self.push_call("upgrade_from_v3");
393
394 if self.buggy_v3_upgrade {
395 conn.execute_batch("ILLEGAL_SQL_CODE")?;
396 }
397
398 conn.execute_batch(
399 "
400 ALTER TABLE my_table RENAME COLUMN old_col to col;
401 ",
402 )?;
403 Ok(())
404 }
405 _ => {
406 panic!("Unexpected version: {}", version);
407 }
408 }
409 }
410
411 fn finish(&self, conn: &Connection) -> Result<()> {
412 self.push_call("finish");
413 conn.execute_batch(
414 "
415 INSERT INTO my_table(col) SELECT col FROM prep_table;
416 ",
417 )?;
418 Ok(())
419 }
420 }
421
422 pub struct MigratedDatabaseFile<CI: ConnectionInitializer> {
428 _tempdir: TempDir,
431 pub connection_initializer: CI,
432 pub path: PathBuf,
433 }
434
435 impl<CI: ConnectionInitializer> MigratedDatabaseFile<CI> {
436 pub fn new(connection_initializer: CI, init_sql: &str) -> Self {
437 Self::new_with_flags(connection_initializer, init_sql, OpenFlags::default())
438 }
439
440 pub fn new_with_flags(
441 connection_initializer: CI,
442 init_sql: &str,
443 open_flags: OpenFlags,
444 ) -> Self {
445 let tempdir = tempfile::tempdir().unwrap();
446 let path = tempdir.path().join(Path::new("db.sql"));
447 let conn = Connection::open_with_flags(&path, open_flags).unwrap();
448 conn.execute_batch(init_sql).unwrap();
449 Self {
450 _tempdir: tempdir,
451 connection_initializer,
452 path,
453 }
454 }
455
456 pub fn upgrade_to(&self, version: u32) {
460 let mut conn = self.open();
461 let tx = conn.transaction().unwrap();
462 let mut current_version = get_schema_version(&tx).unwrap();
463 while current_version < version {
464 self.connection_initializer
465 .upgrade_from(&tx, current_version)
466 .unwrap();
467 current_version += 1;
468 }
469 set_schema_version(&tx, current_version).unwrap();
470 self.connection_initializer.finish(&tx).unwrap();
471 tx.commit().unwrap();
472 }
473
474 pub fn run_all_upgrades(&self) {
478 let current_version = get_schema_version(&self.open()).unwrap();
479 for version in current_version..CI::END_VERSION {
480 self.upgrade_to(version + 1);
481 }
482 }
483
484 pub fn assert_schema_matches_new_database(&self) {
485 let db = self.open();
486 let new_db = match open_memory_database(&self.connection_initializer) {
487 Ok(db) => db,
488 Err(e) => panic!("Creating new database failed:\n{e}"),
489 };
490
491 compare_sql_maps("table", get_sql(&db, "table"), get_sql(&new_db, "table"));
492 compare_sql_maps("index", get_sql(&db, "index"), get_sql(&new_db, "index"));
493 compare_sql_maps(
494 "trigger",
495 get_sql(&db, "trigger"),
496 get_sql(&new_db, "trigger"),
497 );
498 }
499
500 pub fn open(&self) -> Connection {
501 Connection::open(&self.path).unwrap()
502 }
503 }
504
505 fn get_sql(conn: &Connection, type_: &str) -> HashMap<String, Option<String>> {
506 conn.query_rows_and_then(
507 "SELECT name, sql FROM sqlite_master WHERE type=?",
508 (type_,),
509 |row| -> rusqlite::Result<(String, Option<String>)> { Ok((row.get(0)?, row.get(1)?)) },
510 )
511 .unwrap()
512 .into_iter()
513 .collect()
514 }
515
516 fn compare_sql_maps(
517 type_: &str,
518 old_items: HashMap<String, Option<String>>,
519 new_items: HashMap<String, Option<String>>,
520 ) {
521 let old_db_keys: HashSet<&String> = old_items.keys().collect();
522 let new_db_keys: HashSet<&String> = new_items.keys().collect();
523
524 let old_db_extra_keys = Vec::from_iter(old_db_keys.difference(&new_db_keys));
525 if !old_db_extra_keys.is_empty() {
526 panic!("Extra keys not present in new database for {type_}: {old_db_extra_keys:?}");
527 }
528 let new_db_extra_keys = Vec::from_iter(new_db_keys.difference(&old_db_keys));
529 if !new_db_extra_keys.is_empty() {
530 panic!("Extra keys only present in new database for {type_}: {new_db_extra_keys:?}");
531 }
532 for key in old_db_keys {
533 assert_eq!(
534 old_items.get(key).unwrap().as_deref().map(normalize),
535 new_items.get(key).unwrap().as_deref().map(normalize),
536 "sql differs for {type_} {key}"
537 );
538 }
539 }
540
541 fn normalize(sql: &str) -> String {
543 sql.split('\'')
544 .enumerate()
545 .map(|(i, part)| {
546 if (i % 2) == 0 {
551 Cow::Owned(part.split_whitespace().collect::<Vec<_>>().join(" "))
552 } else {
553 Cow::Borrowed(part)
554 }
555 })
556 .collect()
557 }
558}
559
560#[cfg(test)]
561mod test {
562 use super::test_utils::{MigratedDatabaseFile, TestConnectionInitializer};
563 use super::*;
564 use std::io::Write;
565
566 static INIT_V1: &str = "
569 CREATE TABLE prep_table(col);
570 PRAGMA user_version=1;
571 ";
572
573 static INIT_V2: &str = "
575 CREATE TABLE prep_table(col);
576 INSERT INTO prep_table(col) VALUES ('correct-value');
577 CREATE TABLE my_old_table_name(old_col);
578 PRAGMA user_version=2;
579 ";
580
581 fn check_final_data(conn: &Connection) {
582 let value: String = conn
583 .query_row("SELECT col FROM my_table", [], |r| r.get(0))
584 .unwrap();
585 assert_eq!(value, "correct-value");
586 assert_eq!(get_schema_version(conn).unwrap(), 4);
587 }
588
589 #[test]
590 fn test_init() {
591 let connection_initializer = TestConnectionInitializer::new();
592 let conn = open_memory_database(&connection_initializer).unwrap();
593 check_final_data(&conn);
594 connection_initializer.check_calls(vec!["prep", "init", "finish"]);
595 }
596
597 #[test]
598 fn test_upgrades() {
599 let db_file = MigratedDatabaseFile::new(TestConnectionInitializer::new(), INIT_V2);
600 let conn = open_database(db_file.path.clone(), &db_file.connection_initializer).unwrap();
601 check_final_data(&conn);
602 db_file.connection_initializer.check_calls(vec![
603 "prep",
604 "upgrade_from_v2",
605 "upgrade_from_v3",
606 "finish",
607 ]);
608 }
609
610 #[test]
611 fn test_open_current_version() {
612 let db_file = MigratedDatabaseFile::new(TestConnectionInitializer::new(), INIT_V2);
613 db_file.upgrade_to(4);
614 db_file.connection_initializer.clear_calls();
615 let conn = open_database(db_file.path.clone(), &db_file.connection_initializer).unwrap();
616 check_final_data(&conn);
617 db_file
618 .connection_initializer
619 .check_calls(vec!["prep", "finish"]);
620 }
621
622 #[test]
623 fn test_pragmas() {
624 let db_file = MigratedDatabaseFile::new(TestConnectionInitializer::new(), INIT_V2);
625 let conn = open_database(db_file.path.clone(), &db_file.connection_initializer).unwrap();
626 assert_eq!(
627 conn.conn_ext_query_one::<String>("PRAGMA journal_mode")
628 .unwrap(),
629 "wal"
630 );
631 }
632
633 #[test]
634 fn test_migration_error() {
635 let db_file =
636 MigratedDatabaseFile::new(TestConnectionInitializer::new_with_buggy_logic(), INIT_V2);
637 db_file
638 .open()
639 .execute(
640 "INSERT INTO my_old_table_name(old_col) VALUES ('I should not be deleted')",
641 [],
642 )
643 .unwrap();
644
645 open_database(db_file.path.clone(), &db_file.connection_initializer).unwrap_err();
646 assert_eq!(
649 db_file
650 .open()
651 .conn_ext_query_one::<i32>("SELECT COUNT(*) FROM my_old_table_name")
652 .unwrap(),
653 1
654 );
655 }
656
657 #[test]
658 fn test_version_too_new() {
659 let db_file = MigratedDatabaseFile::new(TestConnectionInitializer::new(), INIT_V2);
660 set_schema_version(&db_file.open(), 5).unwrap();
661
662 db_file
663 .open()
664 .execute(
665 "INSERT INTO my_old_table_name(old_col) VALUES ('I should not be deleted')",
666 [],
667 )
668 .unwrap();
669
670 assert!(matches!(
671 open_database(db_file.path.clone(), &db_file.connection_initializer,),
672 Err(Error::IncompatibleVersion(5))
673 ));
674 assert_eq!(
677 db_file
678 .open()
679 .conn_ext_query_one::<i32>("SELECT COUNT(*) FROM my_old_table_name")
680 .unwrap(),
681 1
682 );
683 }
684
685 #[test]
686 fn test_corrupt_db() {
687 let tempdir = tempfile::tempdir().unwrap();
688 let path = tempdir.path().join(Path::new("corrupt-db.sql"));
689 let mut file = std::fs::File::create(path.clone()).unwrap();
690 file.write_all(b"not sql").unwrap();
695 let metadata = std::fs::metadata(path.clone()).unwrap();
696 assert_eq!(metadata.len(), 7);
697 drop(file);
698 open_database(path.clone(), &TestConnectionInitializer::new()).unwrap();
699 let metadata = std::fs::metadata(path).unwrap();
700 assert_ne!(metadata.len(), 7);
702 }
703
704 #[test]
705 fn test_force_replace() {
706 let db_file = MigratedDatabaseFile::new(TestConnectionInitializer::new(), INIT_V1);
707 let conn = open_database(db_file.path.clone(), &db_file.connection_initializer).unwrap();
708 check_final_data(&conn);
709 db_file.connection_initializer.check_calls(vec![
710 "prep",
711 "upgrade_from_v1",
712 "prep",
713 "init",
714 "finish",
715 ]);
716 }
717}