places/import/ios/
history.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5use std::time::Instant;
6
7use crate::error::{info, Result};
8use crate::history_sync::engine::LAST_SYNC_META_KEY;
9use crate::import::common::{
10    attached_database, define_history_migration_functions, select_count, HistoryMigrationResult,
11};
12use crate::storage::{put_meta, update_all_frecencies_at_once};
13use crate::PlacesDb;
14use types::Timestamp;
15use url::Url;
16
17/// This import is used for iOS users migrating from `browser.db`-based
18/// history storage to the new rust-places store.
19///
20/// The goal of this import is to persist all local browser.db items into places database
21///
22///
23/// ### Basic process
24///
25/// - Attach the iOS database.
26/// - Slurp records into a temp table "iOSHistoryStaging" from iOS database.
27///   - This is mostly done for convenience, to punycode the URLs and some performance benefits over
28///     using a view or reading things into Rust
29/// - Add any entries to moz_places that are needed (in practice, most are
30///   needed, users in practice don't have nearly as many bookmarks as history entries)
31/// - Use iosHistoryStaging and the browser.db to migrate visits to the places visits table.
32/// - Update frecency for new items.
33/// - Cleanup (detach iOS database, etc).
34pub fn import(
35    conn: &PlacesDb,
36    path: impl AsRef<std::path::Path>,
37    last_sync_timestamp: i64,
38) -> Result<HistoryMigrationResult> {
39    let url = crate::util::ensure_url_path(path)?;
40    do_import(conn, url, last_sync_timestamp)
41}
42
43fn do_import(
44    conn: &PlacesDb,
45    ios_db_file_url: Url,
46    last_sync_timestamp: i64,
47) -> Result<HistoryMigrationResult> {
48    let scope = conn.begin_interrupt_scope()?;
49    define_history_migration_functions(conn)?;
50    // TODO: for some reason opening the db as read-only in **iOS** causes
51    // the migration to fail with an "attempting to write to a read-only database"
52    // when the migration is **not** writing to the BrowserDB database.
53    // this only happens in the simulator with artifacts built for iOS and not
54    // in unit tests.
55
56    // ios_db_file_url.query_pairs_mut().append_pair("mode", "ro");
57    let import_start = Instant::now();
58    info!("Attaching database {}", ios_db_file_url);
59    let auto_detach = attached_database(conn, &ios_db_file_url, "ios")?;
60    let tx = conn.begin_transaction()?;
61    let num_total = select_count(conn, &COUNT_IOS_HISTORY_VISITS)?;
62    info!("The number of visits is: {:?}", num_total);
63
64    info!("Creating and populating staging table");
65
66    tx.execute_batch(&CREATE_TEMP_VISIT_TABLE)?;
67    tx.execute_batch(&FILL_VISIT_TABLE)?;
68    tx.execute_batch(&CREATE_STAGING_TABLE)?;
69    tx.execute_batch(&FILL_STAGING)?;
70    scope.err_if_interrupted()?;
71
72    info!("Updating old titles that may be missing, but now are available");
73    tx.execute_batch(&UPDATE_PLACES_TITLES)?;
74    scope.err_if_interrupted()?;
75
76    info!("Populating missing entries in moz_places");
77    tx.execute_batch(&FILL_MOZ_PLACES)?;
78    scope.err_if_interrupted()?;
79
80    info!("Inserting the history visits");
81    tx.execute_batch(&INSERT_HISTORY_VISITS)?;
82    scope.err_if_interrupted()?;
83
84    info!("Insert all new entries into stale frecencies");
85    let now = Timestamp::now().as_millis();
86    tx.execute(&ADD_TO_STALE_FRECENCIES, &[(":now", &now)])?;
87    scope.err_if_interrupted()?;
88
89    // Once the migration is done, we also migrate the sync timestamp if we have one
90    // this prevents us from having to do a **full** sync
91    put_meta(conn, LAST_SYNC_META_KEY, &last_sync_timestamp)?;
92
93    tx.commit()?;
94    info!("Successfully imported history visits!");
95
96    info!("Counting Places history visits");
97
98    let num_succeeded = select_count(conn, &COUNT_PLACES_HISTORY_VISITS)?;
99    let num_failed = num_total.saturating_sub(num_succeeded);
100
101    // We now update the frecencies as its own transaction
102    // this is desired because we want reader connections to
103    // read the migrated data and not have to wait for the
104    // frecencies to be up to date
105    info!("Updating all frecencies");
106    update_all_frecencies_at_once(conn, &scope)?;
107    info!("Frecencies updated!");
108    auto_detach.execute_now()?;
109
110    Ok(HistoryMigrationResult {
111        num_total,
112        num_succeeded,
113        num_failed,
114        total_duration: import_start.elapsed().as_millis() as u64,
115    })
116}
117
118lazy_static::lazy_static! {
119   // Count IOS history visits
120   static ref COUNT_IOS_HISTORY_VISITS: &'static str =
121       "SELECT COUNT(*) FROM ios.visits v
122        LEFT JOIN ios.history h on v.siteID = h.id
123        WHERE h.is_deleted = 0"
124   ;
125
126   // Create a temporary table for visists
127   static ref CREATE_TEMP_VISIT_TABLE: &'static str = "
128    CREATE TEMP TABLE IF NOT EXISTS temp.latestVisits(
129        id INTEGER PRIMARY KEY,
130        siteID INTEGER NOT NULL,
131        date REAL NOT NULL,
132        type INTEGER NOT NULL,
133        is_local TINYINT NOT NULL
134    ) WITHOUT ROWID;
135   ";
136
137   // Insert into temp visit table
138   static ref FILL_VISIT_TABLE: &'static str = "
139    INSERT OR IGNORE INTO temp.latestVisits(id, siteID, date, type, is_local)
140        SELECT
141            id,
142            siteID,
143            date,
144            type,
145            is_local
146        FROM ios.visits
147        ORDER BY date DESC
148        LIMIT 10000
149   ";
150
151   // We use a staging table purely so that we can normalize URLs (and
152   // specifically, punycode them)
153   static ref CREATE_STAGING_TABLE: &'static str = "
154        CREATE TEMP TABLE IF NOT EXISTS temp.iOSHistoryStaging(
155            id INTEGER PRIMARY KEY,
156            url TEXT,
157            url_hash INTEGER NOT NULL,
158            title TEXT
159        ) WITHOUT ROWID;";
160
161   static ref FILL_STAGING: &'static str = "
162    INSERT OR IGNORE INTO temp.iOSHistoryStaging(id, url, url_hash, title)
163        SELECT
164            h.id,
165            validate_url(h.url),
166            hash(validate_url(h.url)),
167            sanitize_utf8(h.title)
168        FROM temp.latestVisits v
169        JOIN ios.history h on v.siteID = h.id
170        WHERE h.url IS NOT NULL
171        AND h.is_deleted = 0
172        "
173   ;
174
175    // Unfortunately UPDATE FROM is not available until sqlite 3.33
176   // however, iOS does not ship with 3.33 yet as of the time of writing.
177   static ref UPDATE_PLACES_TITLES: &'static str =
178   "UPDATE main.moz_places
179        SET title = IFNULL((SELECT t.title
180                            FROM temp.iOSHistoryStaging t
181                            WHERE t.url_hash = main.moz_places.url_hash AND t.url = main.moz_places.url), title)"
182    ;
183
184   // Insert any missing entries into moz_places that we'll need for this.
185   static ref FILL_MOZ_PLACES: &'static str =
186   "INSERT OR IGNORE INTO main.moz_places(guid, url, url_hash, title, frecency, sync_change_counter)
187        SELECT
188            IFNULL(
189                (SELECT p.guid FROM main.moz_places p WHERE p.url_hash = t.url_hash AND p.url = t.url),
190                generate_guid()
191            ),
192            t.url,
193            t.url_hash,
194            t.title,
195            -1,
196            1
197        FROM temp.iOSHistoryStaging t
198   "
199   ;
200
201   // Insert history visits
202   static ref INSERT_HISTORY_VISITS: &'static str =
203   "INSERT OR IGNORE INTO main.moz_historyvisits(from_visit, place_id, visit_date, visit_type, is_local)
204        SELECT
205            NULL, -- iOS does not store enough information to rebuild redirect chains.
206            (SELECT p.id FROM main.moz_places p WHERE p.url_hash = t.url_hash AND p.url = t.url),
207            sanitize_float_timestamp(v.date),
208            v.type, -- iOS stores visit types that map 1:1 to ours.
209            v.is_local
210        FROM temp.latestVisits v
211        JOIN temp.iOSHistoryStaging t on v.siteID = t.id
212    "
213   ;
214
215
216   // Count places history visits
217   static ref COUNT_PLACES_HISTORY_VISITS: &'static str =
218       "SELECT COUNT(*) FROM main.moz_historyvisits"
219   ;
220
221   // Adds newly modified places entries into the stale frecencies table
222   static ref ADD_TO_STALE_FRECENCIES: &'static str =
223   "INSERT OR IGNORE INTO main.moz_places_stale_frecencies(place_id, stale_at)
224    SELECT
225        p.id,
226        :now
227    FROM main.moz_places p
228    WHERE p.frecency = -1"
229    ;
230}
places/import/ios/history.rs

places/import/ios/
history.rs
