1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
45use std::time::Instant;
67use crate::error::{info, Result};
8use crate::history_sync::engine::LAST_SYNC_META_KEY;
9use crate::import::common::{
10 attached_database, define_history_migration_functions, select_count, HistoryMigrationResult,
11};
12use crate::storage::{put_meta, update_all_frecencies_at_once};
13use crate::PlacesDb;
14use types::Timestamp;
15use url::Url;
1617/// This import is used for iOS users migrating from `browser.db`-based
18/// history storage to the new rust-places store.
19///
20/// The goal of this import is to persist all local browser.db items into places database
21///
22///
23/// ### Basic process
24///
25/// - Attach the iOS database.
26/// - Slurp records into a temp table "iOSHistoryStaging" from iOS database.
27/// - This is mostly done for convenience, to punycode the URLs and some performance benefits over
28/// using a view or reading things into Rust
29/// - Add any entries to moz_places that are needed (in practice, most are
30/// needed, users in practice don't have nearly as many bookmarks as history entries)
31/// - Use iosHistoryStaging and the browser.db to migrate visits to the places visits table.
32/// - Update frecency for new items.
33/// - Cleanup (detach iOS database, etc).
34pub fn import(
35 conn: &PlacesDb,
36 path: impl AsRef<std::path::Path>,
37 last_sync_timestamp: i64,
38) -> Result<HistoryMigrationResult> {
39let url = crate::util::ensure_url_path(path)?;
40 do_import(conn, url, last_sync_timestamp)
41}
4243fn do_import(
44 conn: &PlacesDb,
45 ios_db_file_url: Url,
46 last_sync_timestamp: i64,
47) -> Result<HistoryMigrationResult> {
48let scope = conn.begin_interrupt_scope()?;
49 define_history_migration_functions(conn)?;
50// TODO: for some reason opening the db as read-only in **iOS** causes
51 // the migration to fail with an "attempting to write to a read-only database"
52 // when the migration is **not** writing to the BrowserDB database.
53 // this only happens in the simulator with artifacts built for iOS and not
54 // in unit tests.
5556 // ios_db_file_url.query_pairs_mut().append_pair("mode", "ro");
57let import_start = Instant::now();
58info!("Attaching database {}", ios_db_file_url);
59let auto_detach = attached_database(conn, &ios_db_file_url, "ios")?;
60let tx = conn.begin_transaction()?;
61let num_total = select_count(conn, &COUNT_IOS_HISTORY_VISITS)?;
62info!("The number of visits is: {:?}", num_total);
6364info!("Creating and populating staging table");
6566 tx.execute_batch(&CREATE_TEMP_VISIT_TABLE)?;
67 tx.execute_batch(&FILL_VISIT_TABLE)?;
68 tx.execute_batch(&CREATE_STAGING_TABLE)?;
69 tx.execute_batch(&FILL_STAGING)?;
70 scope.err_if_interrupted()?;
7172info!("Updating old titles that may be missing, but now are available");
73 tx.execute_batch(&UPDATE_PLACES_TITLES)?;
74 scope.err_if_interrupted()?;
7576info!("Populating missing entries in moz_places");
77 tx.execute_batch(&FILL_MOZ_PLACES)?;
78 scope.err_if_interrupted()?;
7980info!("Inserting the history visits");
81 tx.execute_batch(&INSERT_HISTORY_VISITS)?;
82 scope.err_if_interrupted()?;
8384info!("Insert all new entries into stale frecencies");
85let now = Timestamp::now().as_millis();
86 tx.execute(&ADD_TO_STALE_FRECENCIES, &[(":now", &now)])?;
87 scope.err_if_interrupted()?;
8889// Once the migration is done, we also migrate the sync timestamp if we have one
90 // this prevents us from having to do a **full** sync
91put_meta(conn, LAST_SYNC_META_KEY, &last_sync_timestamp)?;
9293 tx.commit()?;
94info!("Successfully imported history visits!");
9596info!("Counting Places history visits");
9798let num_succeeded = select_count(conn, &COUNT_PLACES_HISTORY_VISITS)?;
99let num_failed = num_total.saturating_sub(num_succeeded);
100101// We now update the frecencies as its own transaction
102 // this is desired because we want reader connections to
103 // read the migrated data and not have to wait for the
104 // frecencies to be up to date
105info!("Updating all frecencies");
106 update_all_frecencies_at_once(conn, &scope)?;
107info!("Frecencies updated!");
108 auto_detach.execute_now()?;
109110Ok(HistoryMigrationResult {
111 num_total,
112 num_succeeded,
113 num_failed,
114 total_duration: import_start.elapsed().as_millis() as u64,
115 })
116}
117118lazy_static::lazy_static! {
119// Count IOS history visits
120static ref COUNT_IOS_HISTORY_VISITS: &'static str =
121"SELECT COUNT(*) FROM ios.visits v
122 LEFT JOIN ios.history h on v.siteID = h.id
123 WHERE h.is_deleted = 0"
124;
125126// Create a temporary table for visists
127static ref CREATE_TEMP_VISIT_TABLE: &'static str = "
128 CREATE TEMP TABLE IF NOT EXISTS temp.latestVisits(
129 id INTEGER PRIMARY KEY,
130 siteID INTEGER NOT NULL,
131 date REAL NOT NULL,
132 type INTEGER NOT NULL,
133 is_local TINYINT NOT NULL
134 ) WITHOUT ROWID;
135 ";
136137// Insert into temp visit table
138static ref FILL_VISIT_TABLE: &'static str = "
139 INSERT OR IGNORE INTO temp.latestVisits(id, siteID, date, type, is_local)
140 SELECT
141 id,
142 siteID,
143 date,
144 type,
145 is_local
146 FROM ios.visits
147 ORDER BY date DESC
148 LIMIT 10000
149 ";
150151// We use a staging table purely so that we can normalize URLs (and
152 // specifically, punycode them)
153static ref CREATE_STAGING_TABLE: &'static str = "
154 CREATE TEMP TABLE IF NOT EXISTS temp.iOSHistoryStaging(
155 id INTEGER PRIMARY KEY,
156 url TEXT,
157 url_hash INTEGER NOT NULL,
158 title TEXT
159 ) WITHOUT ROWID;";
160161static ref FILL_STAGING: &'static str = "
162 INSERT OR IGNORE INTO temp.iOSHistoryStaging(id, url, url_hash, title)
163 SELECT
164 h.id,
165 validate_url(h.url),
166 hash(validate_url(h.url)),
167 sanitize_utf8(h.title)
168 FROM temp.latestVisits v
169 JOIN ios.history h on v.siteID = h.id
170 WHERE h.url IS NOT NULL
171 AND h.is_deleted = 0
172 "
173;
174175// Unfortunately UPDATE FROM is not available until sqlite 3.33
176 // however, iOS does not ship with 3.33 yet as of the time of writing.
177static ref UPDATE_PLACES_TITLES: &'static str =
178"UPDATE main.moz_places
179 SET title = IFNULL((SELECT t.title
180 FROM temp.iOSHistoryStaging t
181 WHERE t.url_hash = main.moz_places.url_hash AND t.url = main.moz_places.url), title)"
182;
183184// Insert any missing entries into moz_places that we'll need for this.
185static ref FILL_MOZ_PLACES: &'static str =
186"INSERT OR IGNORE INTO main.moz_places(guid, url, url_hash, title, frecency, sync_change_counter)
187 SELECT
188 IFNULL(
189 (SELECT p.guid FROM main.moz_places p WHERE p.url_hash = t.url_hash AND p.url = t.url),
190 generate_guid()
191 ),
192 t.url,
193 t.url_hash,
194 t.title,
195 -1,
196 1
197 FROM temp.iOSHistoryStaging t
198 "
199;
200201// Insert history visits
202static ref INSERT_HISTORY_VISITS: &'static str =
203"INSERT OR IGNORE INTO main.moz_historyvisits(from_visit, place_id, visit_date, visit_type, is_local)
204 SELECT
205 NULL, -- iOS does not store enough information to rebuild redirect chains.
206 (SELECT p.id FROM main.moz_places p WHERE p.url_hash = t.url_hash AND p.url = t.url),
207 sanitize_float_timestamp(v.date),
208 v.type, -- iOS stores visit types that map 1:1 to ours.
209 v.is_local
210 FROM temp.latestVisits v
211 JOIN temp.iOSHistoryStaging t on v.siteID = t.id
212 "
213;
214215216// Count places history visits
217static ref COUNT_PLACES_HISTORY_VISITS: &'static str =
218"SELECT COUNT(*) FROM main.moz_historyvisits"
219;
220221// Adds newly modified places entries into the stale frecencies table
222static ref ADD_TO_STALE_FRECENCIES: &'static str =
223"INSERT OR IGNORE INTO main.moz_places_stale_frecencies(place_id, stale_at)
224 SELECT
225 p.id,
226 :now
227 FROM main.moz_places p
228 WHERE p.frecency = -1"
229;
230}