places/storage/
history.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5mod actions;
6
7use super::{fetch_page_info, new_page_info, PageInfo, RowId};
8use crate::db::PlacesDb;
9use crate::error::{debug, trace, warn, Result};
10use crate::ffi::{HistoryVisitInfo, HistoryVisitInfosWithBound, TopFrecentSiteInfo};
11use crate::frecency;
12use crate::hash;
13use crate::history_sync::engine::{
14    COLLECTION_SYNCID_META_KEY, GLOBAL_SYNCID_META_KEY, LAST_SYNC_META_KEY,
15};
16use crate::observation::VisitObservation;
17use crate::storage::{
18    delete_meta, delete_pending_temp_tables, get_meta, history_metadata, put_meta,
19};
20use crate::types::{
21    serialize_unknown_fields, SyncStatus, UnknownFields, VisitTransitionSet, VisitType,
22};
23use actions::*;
24use rusqlite::types::ToSql;
25use rusqlite::Result as RusqliteResult;
26use rusqlite::Row;
27use sql_support::{self, ConnExt};
28use std::collections::HashSet;
29use std::time::Duration;
30use sync15::bso::OutgoingBso;
31use sync15::engine::EngineSyncAssociation;
32use sync_guid::Guid as SyncGuid;
33use types::Timestamp;
34use url::Url;
35
36/// When `delete_everything` is called (to perform a permanent local deletion), in
37/// addition to performing the deletion as requested, we make a note of the time
38/// when it occurred, and refuse to sync incoming visits from before this time.
39///
40/// This allows us to avoid these visits trickling back in as other devices
41/// add visits to them remotely.
42static DELETION_HIGH_WATER_MARK_META_KEY: &str = "history_deleted_hwm";
43
44/// Returns the RowId of a new visit in moz_historyvisits, or None if no new visit was added.
45pub fn apply_observation(db: &PlacesDb, visit_ob: VisitObservation) -> Result<Option<RowId>> {
46    let tx = db.begin_transaction()?;
47    let result = apply_observation_direct(db, visit_ob)?;
48    delete_pending_temp_tables(db)?;
49    tx.commit()?;
50    Ok(result)
51}
52
53/// Returns the RowId of a new visit in moz_historyvisits, or None if no new visit was added.
54pub fn apply_observation_direct(
55    db: &PlacesDb,
56    visit_ob: VisitObservation,
57) -> Result<Option<RowId>> {
58    // Don't insert urls larger than our length max.
59    if visit_ob.url.as_str().len() > super::URL_LENGTH_MAX {
60        return Ok(None);
61    }
62    // Make sure we have a valid preview URL - it should parse, and not exceed max size.
63    // In case the URL is too long, ignore it and proceed with the rest of the observation.
64    // In case the URL is entirely invalid, let the caller know by failing.
65    let preview_image_url = if let Some(ref piu) = visit_ob.preview_image_url {
66        if piu.as_str().len() > super::URL_LENGTH_MAX {
67            None
68        } else {
69            Some(piu.clone())
70        }
71    } else {
72        None
73    };
74    let mut page_info = match fetch_page_info(db, &visit_ob.url)? {
75        Some(info) => info.page,
76        None => new_page_info(db, &visit_ob.url, None)?,
77    };
78    let mut update_change_counter = false;
79    let mut update_frec = false;
80    let mut updates: Vec<(&str, &str, &dyn ToSql)> = Vec::new();
81
82    if let Some(ref title) = visit_ob.title {
83        page_info.title = crate::util::slice_up_to(title, super::TITLE_LENGTH_MAX).into();
84        updates.push(("title", ":title", &page_info.title));
85        update_change_counter = true;
86    }
87    let preview_image_url_str;
88    if let Some(ref preview_image_url) = preview_image_url {
89        preview_image_url_str = preview_image_url.as_str();
90        updates.push((
91            "preview_image_url",
92            ":preview_image_url",
93            &preview_image_url_str,
94        ));
95    }
96    // There's a new visit, so update everything that implies. To help with
97    // testing we return the rowid of the visit we added.
98    let visit_row_id = match visit_ob.visit_type {
99        Some(visit_type) => {
100            // A single non-hidden visit makes the place non-hidden.
101            if !visit_ob.get_is_hidden() {
102                updates.push(("hidden", ":hidden", &false));
103            }
104            if visit_type == VisitType::Typed {
105                page_info.typed += 1;
106                updates.push(("typed", ":typed", &page_info.typed));
107            }
108
109            let at = visit_ob.at.unwrap_or_else(Timestamp::now);
110            let is_remote = visit_ob.is_remote.unwrap_or(false);
111            let row_id = add_visit(db, page_info.row_id, None, at, visit_type, !is_remote, None)?;
112            // a new visit implies new frecency except in error cases.
113            if !visit_ob.is_error.unwrap_or(false) {
114                update_frec = true;
115            }
116            update_change_counter = true;
117            Some(row_id)
118        }
119        None => None,
120    };
121
122    if update_change_counter {
123        page_info.sync_change_counter += 1;
124        updates.push((
125            "sync_change_counter",
126            ":sync_change_counter",
127            &page_info.sync_change_counter,
128        ));
129    }
130
131    if !updates.is_empty() {
132        let mut params: Vec<(&str, &dyn ToSql)> = Vec::with_capacity(updates.len() + 1);
133        let mut sets: Vec<String> = Vec::with_capacity(updates.len());
134        for (col, name, val) in updates {
135            sets.push(format!("{} = {}", col, name));
136            params.push((name, val))
137        }
138        params.push((":row_id", &page_info.row_id.0));
139        let sql = format!(
140            "UPDATE moz_places
141                          SET {}
142                          WHERE id == :row_id",
143            sets.join(",")
144        );
145        db.execute(&sql, &params[..])?;
146    }
147    // This needs to happen after the other updates.
148    if update_frec {
149        update_frecency(
150            db,
151            page_info.row_id,
152            Some(visit_ob.get_redirect_frecency_boost()),
153        )?;
154    }
155    Ok(visit_row_id)
156}
157
158pub fn update_frecency(db: &PlacesDb, id: RowId, redirect_boost: Option<bool>) -> Result<()> {
159    let score = frecency::calculate_frecency(
160        db.conn(),
161        &frecency::DEFAULT_FRECENCY_SETTINGS,
162        id.0, // TODO: calculate_frecency should take a RowId here.
163        redirect_boost,
164    )?;
165
166    db.execute(
167        "
168        UPDATE moz_places
169            SET frecency = :frecency
170        WHERE id = :page_id",
171        &[
172            (":frecency", &score as &dyn rusqlite::ToSql),
173            (":page_id", &id.0),
174        ],
175    )?;
176
177    Ok(())
178}
179
180/// Indicates if and when a URL's frecency was marked as stale.
181pub fn frecency_stale_at(db: &PlacesDb, url: &Url) -> Result<Option<Timestamp>> {
182    let result = db.try_query_row(
183        "SELECT stale_at FROM moz_places_stale_frecencies s
184         JOIN moz_places h ON h.id = s.place_id
185         WHERE h.url_hash = hash(:url) AND
186               h.url = :url",
187        &[(":url", &url.as_str())],
188        |row| -> rusqlite::Result<_> { row.get::<_, Timestamp>(0) },
189        true,
190    )?;
191    Ok(result)
192}
193
194// Add a single visit - you must know the page rowid. Does not update the
195// page info - if you are calling this, you will also need to update the
196// parent page with an updated change counter etc.
197fn add_visit(
198    db: &PlacesDb,
199    page_id: RowId,
200    from_visit: Option<RowId>,
201    visit_date: Timestamp,
202    visit_type: VisitType,
203    is_local: bool,
204    unknown_fields: Option<String>,
205) -> Result<RowId> {
206    let sql = "INSERT INTO moz_historyvisits
207            (from_visit, place_id, visit_date, visit_type, is_local, unknown_fields)
208        VALUES (:from_visit, :page_id, :visit_date, :visit_type, :is_local, :unknown_fields)";
209    db.execute_cached(
210        sql,
211        &[
212            (":from_visit", &from_visit as &dyn rusqlite::ToSql),
213            (":page_id", &page_id),
214            (":visit_date", &visit_date),
215            (":visit_type", &visit_type),
216            (":is_local", &is_local),
217            (":unknown_fields", &unknown_fields),
218        ],
219    )?;
220    let rid = db.conn().last_insert_rowid();
221    // Delete any tombstone that exists.
222    db.execute_cached(
223        "DELETE FROM moz_historyvisit_tombstones
224         WHERE place_id = :place_id
225           AND visit_date = :visit_date",
226        &[
227            (":place_id", &page_id as &dyn rusqlite::ToSql),
228            (":visit_date", &visit_date),
229        ],
230    )?;
231    Ok(RowId(rid))
232}
233
234/// Returns the GUID for the specified Url, or None if it doesn't exist.
235pub fn url_to_guid(db: &PlacesDb, url: &Url) -> Result<Option<SyncGuid>> {
236    href_to_guid(db, url.clone().as_str())
237}
238
239/// Returns the GUID for the specified Url String, or None if it doesn't exist.
240pub fn href_to_guid(db: &PlacesDb, url: &str) -> Result<Option<SyncGuid>> {
241    let sql = "SELECT guid FROM moz_places WHERE url_hash = hash(:url) AND url = :url";
242    let result: Option<SyncGuid> = db.try_query_row(
243        sql,
244        &[(":url", &url.to_owned())],
245        // subtle: we explicitly need to specify rusqlite::Result or the compiler
246        // struggles to work out what error type to return from try_query_row.
247        |row| -> rusqlite::Result<_> { row.get::<_, SyncGuid>(0) },
248        true,
249    )?;
250    Ok(result)
251}
252
253/// Internal function for deleting a page, creating a tombstone if necessary.
254/// Assumes a transaction is already set up by the caller.
255fn delete_visits_for_in_tx(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
256    // We only create tombstones for history which exists and with sync_status
257    // == SyncStatus::Normal
258    let to_clean = db.conn().try_query_row(
259        "SELECT id,
260                (foreign_count != 0) AS has_foreign,
261                1 as has_visits,
262                sync_status
263        FROM moz_places
264        WHERE guid = :guid",
265        &[(":guid", guid)],
266        PageToClean::from_row,
267        true,
268    )?;
269    // Note that history metadata has an `ON DELETE CASCADE` for the place ID - so if we
270    // call `delete_page` here, we assume history metadata dies too. Otherwise we
271    // explicitly delete the metadata after we delete the visits themselves.
272    match to_clean {
273        Some(PageToClean {
274            id,
275            has_foreign: true,
276            sync_status: SyncStatus::Normal,
277            ..
278        }) => {
279            // If our page is syncing, and has foreign key references (like
280            // local or synced bookmarks, keywords, and tags), we can't delete
281            // its row from `moz_places` directly; that would cause a constraint
282            // violation. Instead, we must insert tombstones for all visits, and
283            // then delete just the visits, keeping the page in place (pun most
284            // definitely intended).
285            insert_tombstones_for_all_page_visits(db, id)?;
286            delete_all_visits_for_page(db, id)?;
287            history_metadata::delete_all_metadata_for_page(db, id)?;
288        }
289        Some(PageToClean {
290            id,
291            has_foreign: false,
292            sync_status: SyncStatus::Normal,
293            ..
294        }) => {
295            // However, if our page is syncing and _doesn't_ have any foreign
296            // key references, we can delete it from `moz_places` outright, and
297            // write a tombstone for the page instead of all the visits.
298            insert_tombstone_for_page(db, guid)?;
299            delete_page(db, id)?;
300        }
301        Some(PageToClean {
302            id,
303            has_foreign: true,
304            ..
305        }) => {
306            // If our page has foreign key references but _isn't_ syncing,
307            // we still can't delete it; we must delete its visits. But we
308            // don't need to write any tombstones for those deleted visits.
309            delete_all_visits_for_page(db, id)?;
310            // and we need to delete all history metadata.
311            history_metadata::delete_all_metadata_for_page(db, id)?;
312        }
313        Some(PageToClean {
314            id,
315            has_foreign: false,
316            ..
317        }) => {
318            // And, finally, the easiest case: not syncing, and no foreign
319            // key references, so just delete the page.
320            delete_page(db, id)?;
321        }
322        None => {}
323    }
324    delete_pending_temp_tables(db)?;
325    Ok(())
326}
327
328/// Inserts Sync tombstones for all of a page's visits.
329fn insert_tombstones_for_all_page_visits(db: &PlacesDb, page_id: RowId) -> Result<()> {
330    db.execute_cached(
331        "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date)
332         SELECT place_id, visit_date
333         FROM moz_historyvisits
334         WHERE place_id = :page_id",
335        &[(":page_id", &page_id)],
336    )?;
337    Ok(())
338}
339
340/// Removes all visits from a page. DOES NOT remove history_metadata - use
341/// `history_metadata::delete_all_metadata_for_page` for that.
342fn delete_all_visits_for_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
343    db.execute_cached(
344        "DELETE FROM moz_historyvisits
345         WHERE place_id = :page_id",
346        &[(":page_id", &page_id)],
347    )?;
348    Ok(())
349}
350
351/// Inserts a Sync tombstone for a page.
352fn insert_tombstone_for_page(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
353    db.execute_cached(
354        "INSERT OR IGNORE INTO moz_places_tombstones (guid)
355         VALUES(:guid)",
356        &[(":guid", guid)],
357    )?;
358    Ok(())
359}
360
361/// Deletes a page. Note that this throws a constraint violation if the page is
362/// bookmarked, or has a keyword or tags.
363fn delete_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
364    db.execute_cached(
365        "DELETE FROM moz_places
366         WHERE id = :page_id",
367        &[(":page_id", &page_id)],
368    )?;
369    Ok(())
370}
371
372/// Deletes all visits for a page given its GUID, creating tombstones if
373/// necessary.
374pub fn delete_visits_for(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
375    let tx = db.begin_transaction()?;
376    let result = delete_visits_for_in_tx(db, guid);
377    tx.commit()?;
378    result
379}
380
381/// Delete all visits in a date range.
382pub fn delete_visits_between(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
383    let tx = db.begin_transaction()?;
384    delete_visits_between_in_tx(db, start, end)?;
385    tx.commit()?;
386    Ok(())
387}
388
389pub fn delete_place_visit_at_time(db: &PlacesDb, place: &Url, visit: Timestamp) -> Result<()> {
390    delete_place_visit_at_time_by_href(db, place.as_str(), visit)
391}
392
393pub fn delete_place_visit_at_time_by_href(
394    db: &PlacesDb,
395    place: &str,
396    visit: Timestamp,
397) -> Result<()> {
398    let tx = db.begin_transaction()?;
399    delete_place_visit_at_time_in_tx(db, place, visit)?;
400    tx.commit()?;
401    Ok(())
402}
403
404pub fn prune_older_visits(db: &PlacesDb, limit: u32) -> Result<()> {
405    let tx = db.begin_transaction()?;
406
407    let result = DbAction::apply_all(
408        db,
409        db_actions_from_visits_to_delete(find_visits_to_prune(
410            db,
411            limit as usize,
412            Timestamp::now(),
413        )?),
414    );
415    tx.commit()?;
416    result
417}
418
419fn find_visits_to_prune(db: &PlacesDb, limit: usize, now: Timestamp) -> Result<Vec<VisitToDelete>> {
420    // Start with the exotic visits
421    let mut to_delete: HashSet<_> = find_exotic_visits_to_prune(db, limit, now)?
422        .into_iter()
423        .collect();
424    // If we still have more visits to prune, then add them from find_normal_visits_to_prune,
425    // leveraging the HashSet to ensure we don't add a duplicate item.
426    if to_delete.len() < limit {
427        for delete_visit in find_normal_visits_to_prune(db, limit, now)? {
428            to_delete.insert(delete_visit);
429            if to_delete.len() >= limit {
430                break;
431            }
432        }
433    }
434    Ok(Vec::from_iter(to_delete))
435}
436
437fn find_normal_visits_to_prune(
438    db: &PlacesDb,
439    limit: usize,
440    now: Timestamp,
441) -> Result<Vec<VisitToDelete>> {
442    // 7 days ago
443    let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 7));
444    db.query_rows_and_then(
445        "
446        SELECT v.id, v.place_id
447        FROM moz_places p
448        JOIN moz_historyvisits v ON v.place_id = p.id
449        WHERE v.visit_date < :visit_date_cuttoff
450        ORDER BY v.visit_date
451        LIMIT :limit
452        ",
453        rusqlite::named_params! {
454            ":visit_date_cuttoff": visit_date_cutoff,
455            ":limit": limit,
456        },
457        VisitToDelete::from_row,
458    )
459}
460
461/// Find "exotic" visits to prune.  These are visits visits that should be pruned first because
462/// they are less useful to the user because:
463///   - They're very old
464///   - They're not useful in the awesome bar because they're either a long URL or a download
465///
466/// This is based on the desktop pruning logic:
467/// https://searchfox.org/mozilla-central/search?q=QUERY_FIND_EXOTIC_VISITS_TO_EXPIRE
468fn find_exotic_visits_to_prune(
469    db: &PlacesDb,
470    limit: usize,
471    now: Timestamp,
472) -> Result<Vec<VisitToDelete>> {
473    // 60 days ago
474    let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 60));
475    db.query_rows_and_then(
476        "
477        SELECT v.id, v.place_id
478        FROM moz_places p
479        JOIN moz_historyvisits v ON v.place_id = p.id
480        WHERE v.visit_date < :visit_date_cuttoff
481        AND (LENGTH(p.url) > 255 OR v.visit_type = :download_visit_type)
482        ORDER BY v.visit_date
483        LIMIT :limit
484        ",
485        rusqlite::named_params! {
486            ":visit_date_cuttoff": visit_date_cutoff,
487            ":download_visit_type": VisitType::Download,
488            ":limit": limit,
489        },
490        VisitToDelete::from_row,
491    )
492}
493
494fn wipe_local_in_tx(db: &PlacesDb) -> Result<()> {
495    use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
496    db.execute_all(&[
497        "DELETE FROM moz_places WHERE foreign_count == 0",
498        "DELETE FROM moz_places_metadata",
499        "DELETE FROM moz_places_metadata_search_queries",
500        "DELETE FROM moz_historyvisits",
501        "DELETE FROM moz_places_tombstones",
502        "DELETE FROM moz_inputhistory AS i WHERE NOT EXISTS(
503             SELECT 1 FROM moz_places h
504             WHERE h.id = i.place_id)",
505        "DELETE FROM moz_historyvisit_tombstones",
506        "DELETE FROM moz_origins
507         WHERE id NOT IN (SELECT origin_id FROM moz_places)",
508        &format!(
509            r#"UPDATE moz_places SET
510                frecency = (CASE WHEN url_hash BETWEEN hash("place", "prefix_lo") AND
511                                                       hash("place", "prefix_hi")
512                                 THEN 0
513                                 ELSE {unvisited_bookmark_frec}
514                            END),
515                sync_change_counter = 0"#,
516            unvisited_bookmark_frec = DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
517        ),
518    ])?;
519
520    let need_frecency_update =
521        db.query_rows_and_then("SELECT id FROM moz_places", [], |r| r.get::<_, RowId>(0))?;
522    // Update the frecency for any remaining items, which basically means just
523    // for the bookmarks.
524    for row_id in need_frecency_update {
525        update_frecency(db, row_id, None)?;
526    }
527    delete_pending_temp_tables(db)?;
528    Ok(())
529}
530
531pub fn delete_everything(db: &PlacesDb) -> Result<()> {
532    let tx = db.begin_transaction()?;
533
534    // Remote visits could have a higher date than `now` if our clock is weird.
535    let most_recent_known_visit_time = db
536        .try_query_one::<Timestamp, _>("SELECT MAX(visit_date) FROM moz_historyvisits", [], false)?
537        .unwrap_or_default();
538
539    // Check the old value (if any) for the same reason
540    let previous_mark =
541        get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
542
543    let new_mark = Timestamp::now()
544        .max(previous_mark)
545        .max(most_recent_known_visit_time);
546
547    put_meta(db, DELETION_HIGH_WATER_MARK_META_KEY, &new_mark)?;
548
549    wipe_local_in_tx(db)?;
550
551    // Remove Sync metadata, too.
552    reset_in_tx(db, &EngineSyncAssociation::Disconnected)?;
553
554    tx.commit()?;
555
556    // Note: SQLite cannot VACUUM within a transaction.
557    db.execute_batch("VACUUM")?;
558    Ok(())
559}
560
561fn delete_place_visit_at_time_in_tx(db: &PlacesDb, url: &str, visit_date: Timestamp) -> Result<()> {
562    DbAction::apply_all(
563        db,
564        db_actions_from_visits_to_delete(db.query_rows_and_then(
565            "SELECT v.id, v.place_id
566                 FROM moz_places h
567                 JOIN moz_historyvisits v
568                   ON v.place_id = h.id
569                 WHERE v.visit_date = :visit_date
570                   AND h.url_hash = hash(:url)
571                   AND h.url = :url",
572            &[
573                (":url", &url as &dyn rusqlite::ToSql),
574                (":visit_date", &visit_date),
575            ],
576            VisitToDelete::from_row,
577        )?),
578    )
579}
580
581pub fn delete_visits_between_in_tx(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
582    // Like desktop's removeVisitsByFilter, we query the visit and place ids
583    // affected, then delete all visits, then delete all place ids in the set
584    // which are orphans after the delete.
585    let sql = "
586        SELECT id, place_id, visit_date
587        FROM moz_historyvisits
588        WHERE visit_date
589            BETWEEN :start AND :end
590    ";
591    let visits = db.query_rows_and_then(
592        sql,
593        &[(":start", &start), (":end", &end)],
594        |row| -> rusqlite::Result<_> {
595            Ok((
596                row.get::<_, RowId>(0)?,
597                row.get::<_, RowId>(1)?,
598                row.get::<_, Timestamp>(2)?,
599            ))
600        },
601    )?;
602
603    sql_support::each_chunk_mapped(
604        &visits,
605        |(visit_id, _, _)| visit_id,
606        |chunk, _| -> Result<()> {
607            db.conn().execute(
608                &format!(
609                    "DELETE from moz_historyvisits WHERE id IN ({})",
610                    sql_support::repeat_sql_vars(chunk.len()),
611                ),
612                rusqlite::params_from_iter(chunk),
613            )?;
614            Ok(())
615        },
616    )?;
617
618    // Insert tombstones for the deleted visits.
619    if !visits.is_empty() {
620        let sql = format!(
621            "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date) VALUES {}",
622            sql_support::repeat_display(visits.len(), ",", |i, f| {
623                let (_, place_id, visit_date) = visits[i];
624                write!(f, "({},{})", place_id.0, visit_date.0)
625            })
626        );
627        db.conn().execute(&sql, [])?;
628    }
629
630    // Find out which pages have been possibly orphaned and clean them up.
631    sql_support::each_chunk_mapped(
632        &visits,
633        |(_, place_id, _)| place_id.0,
634        |chunk, _| -> Result<()> {
635            let query = format!(
636                "SELECT id,
637                    (foreign_count != 0) AS has_foreign,
638                    ((last_visit_date_local + last_visit_date_remote) != 0) as has_visits,
639                    sync_status
640                FROM moz_places
641                WHERE id IN ({})",
642                sql_support::repeat_sql_vars(chunk.len()),
643            );
644
645            let mut stmt = db.conn().prepare(&query)?;
646            let page_results =
647                stmt.query_and_then(rusqlite::params_from_iter(chunk), PageToClean::from_row)?;
648            let pages: Vec<PageToClean> = page_results.collect::<Result<_>>()?;
649            cleanup_pages(db, &pages)
650        },
651    )?;
652
653    // Clean up history metadata between start and end
654    history_metadata::delete_between(db, start.as_millis_i64(), end.as_millis_i64())?;
655    delete_pending_temp_tables(db)?;
656    Ok(())
657}
658
659#[derive(Debug)]
660struct PageToClean {
661    id: RowId,
662    has_foreign: bool,
663    has_visits: bool,
664    sync_status: SyncStatus,
665}
666
667impl PageToClean {
668    pub fn from_row(row: &Row<'_>) -> Result<Self> {
669        Ok(Self {
670            id: row.get("id")?,
671            has_foreign: row.get("has_foreign")?,
672            has_visits: row.get("has_visits")?,
673            sync_status: row.get("sync_status")?,
674        })
675    }
676}
677
678/// Clean up pages whose history has been modified, by either
679/// removing them entirely (if they are marked for removal,
680/// typically because all visits have been removed and there
681/// are no more foreign keys such as bookmarks) or updating
682/// their frecency.
683fn cleanup_pages(db: &PlacesDb, pages: &[PageToClean]) -> Result<()> {
684    // desktop does this frecency work using a function in a single sql
685    // statement - we should see if we can do that too.
686    let frec_ids = pages
687        .iter()
688        .filter(|&p| p.has_foreign || p.has_visits)
689        .map(|p| p.id);
690
691    for id in frec_ids {
692        update_frecency(db, id, None)?;
693    }
694
695    // Like desktop, we do "AND foreign_count = 0 AND last_visit_date ISNULL"
696    // to creating orphans in case of async race conditions - in Desktop's
697    // case, it reads the pages before starting a write transaction, so that
698    // probably is possible. We don't currently do that, but might later, so
699    // we do it anyway.
700    let remove_ids: Vec<RowId> = pages
701        .iter()
702        .filter(|p| !p.has_foreign && !p.has_visits)
703        .map(|p| p.id)
704        .collect();
705    sql_support::each_chunk(&remove_ids, |chunk, _| -> Result<()> {
706        // tombstones first.
707        db.conn().execute(
708            &format!(
709                "
710                INSERT OR IGNORE INTO moz_places_tombstones (guid)
711                SELECT guid FROM moz_places
712                WHERE id in ({ids}) AND sync_status = {status}
713                    AND foreign_count = 0
714                    AND last_visit_date_local = 0
715                    AND last_visit_date_remote = 0",
716                ids = sql_support::repeat_sql_vars(chunk.len()),
717                status = SyncStatus::Normal as u8,
718            ),
719            rusqlite::params_from_iter(chunk),
720        )?;
721        db.conn().execute(
722            &format!(
723                "
724                DELETE FROM moz_places
725                WHERE id IN ({ids})
726                    AND foreign_count = 0
727                    AND last_visit_date_local = 0
728                    AND last_visit_date_remote = 0",
729                ids = sql_support::repeat_sql_vars(chunk.len())
730            ),
731            rusqlite::params_from_iter(chunk),
732        )?;
733        Ok(())
734    })?;
735
736    Ok(())
737}
738
739fn reset_in_tx(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
740    // Reset change counters and sync statuses for all URLs.
741    db.execute_cached(
742        &format!(
743            "
744            UPDATE moz_places
745                SET sync_change_counter = 0,
746                sync_status = {}",
747            (SyncStatus::New as u8)
748        ),
749        [],
750    )?;
751
752    // Reset the last sync time, so that the next sync fetches fresh records
753    // from the server.
754    put_meta(db, LAST_SYNC_META_KEY, &0)?;
755
756    // Clear the sync ID if we're signing out, or set it to whatever the
757    // server gave us if we're signing in.
758    match assoc {
759        EngineSyncAssociation::Disconnected => {
760            delete_meta(db, GLOBAL_SYNCID_META_KEY)?;
761            delete_meta(db, COLLECTION_SYNCID_META_KEY)?;
762        }
763        EngineSyncAssociation::Connected(ids) => {
764            put_meta(db, GLOBAL_SYNCID_META_KEY, &ids.global)?;
765            put_meta(db, COLLECTION_SYNCID_META_KEY, &ids.coll)?;
766        }
767    }
768
769    Ok(())
770}
771
772// Support for Sync - in its own module to try and keep a delineation
773pub mod history_sync {
774    use sync15::bso::OutgoingEnvelope;
775
776    use super::*;
777    use crate::history_sync::record::{HistoryRecord, HistoryRecordVisit};
778    use crate::history_sync::HISTORY_TTL;
779    use std::collections::HashSet;
780
781    #[derive(Debug, Clone, PartialEq, Eq)]
782    pub struct FetchedVisit {
783        pub is_local: bool,
784        pub visit_date: Timestamp,
785        pub visit_type: Option<VisitType>,
786    }
787
788    impl FetchedVisit {
789        pub fn from_row(row: &Row<'_>) -> Result<Self> {
790            Ok(Self {
791                is_local: row.get("is_local")?,
792                visit_date: row
793                    .get::<_, Option<Timestamp>>("visit_date")?
794                    .unwrap_or_default(),
795                visit_type: VisitType::from_primitive(
796                    row.get::<_, Option<u8>>("visit_type")?.unwrap_or(0),
797                ),
798            })
799        }
800    }
801
802    #[derive(Debug)]
803    pub struct FetchedVisitPage {
804        pub url: Url,
805        pub guid: SyncGuid,
806        pub row_id: RowId,
807        pub title: String,
808        pub unknown_fields: UnknownFields,
809    }
810
811    impl FetchedVisitPage {
812        pub fn from_row(row: &Row<'_>) -> Result<Self> {
813            Ok(Self {
814                url: Url::parse(&row.get::<_, String>("url")?)?,
815                guid: row.get::<_, String>("guid")?.into(),
816                row_id: row.get("id")?,
817                title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
818                unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
819                    None => UnknownFields::new(),
820                    Some(v) => serde_json::from_str(&v)?,
821                },
822            })
823        }
824    }
825
826    pub fn fetch_visits(
827        db: &PlacesDb,
828        url: &Url,
829        limit: usize,
830    ) -> Result<Option<(FetchedVisitPage, Vec<FetchedVisit>)>> {
831        // We do this in 2 steps - "do we have a page" then "get visits"
832        let page_sql = "
833          SELECT guid, url, id, title, unknown_fields
834          FROM moz_places h
835          WHERE url_hash = hash(:url) AND url = :url";
836
837        let page_info = match db.try_query_row(
838            page_sql,
839            &[(":url", &url.to_string())],
840            FetchedVisitPage::from_row,
841            true,
842        )? {
843            None => return Ok(None),
844            Some(pi) => pi,
845        };
846
847        let visits = db.query_rows_and_then(
848            "SELECT is_local, visit_type, visit_date
849            FROM moz_historyvisits
850            WHERE place_id = :place_id
851            LIMIT :limit",
852            &[
853                (":place_id", &page_info.row_id as &dyn rusqlite::ToSql),
854                (":limit", &(limit as u32)),
855            ],
856            FetchedVisit::from_row,
857        )?;
858        Ok(Some((page_info, visits)))
859    }
860
861    /// Apply history visit from sync. This assumes they have all been
862    /// validated, deduped, etc - it's just the storage we do here.
863    pub fn apply_synced_visits(
864        db: &PlacesDb,
865        incoming_guid: &SyncGuid,
866        url: &Url,
867        title: &Option<String>,
868        visits: &[HistoryRecordVisit],
869        unknown_fields: &UnknownFields,
870    ) -> Result<()> {
871        // At some point we may have done a local wipe of all visits. We skip applying
872        // incoming visits that could have been part of that deletion, to avoid them
873        // trickling back in.
874        let visit_ignored_mark =
875            get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
876
877        let visits = visits
878            .iter()
879            .filter(|v| Timestamp::from(v.date) > visit_ignored_mark)
880            .collect::<Vec<_>>();
881
882        let mut counter_incr = 0;
883        let page_info = match fetch_page_info(db, url)? {
884            Some(mut info) => {
885                // If the existing record has not yet been synced, then we will
886                // change the GUID to the incoming one. If it has been synced
887                // we keep the existing guid, but still apply the visits.
888                // See doc/history_duping.rst for more details.
889                if &info.page.guid != incoming_guid {
890                    if info.page.sync_status == SyncStatus::New {
891                        db.execute_cached(
892                            "UPDATE moz_places SET guid = :new_guid WHERE id = :row_id",
893                            &[
894                                (":new_guid", incoming_guid as &dyn rusqlite::ToSql),
895                                (":row_id", &info.page.row_id),
896                            ],
897                        )?;
898                        info.page.guid = incoming_guid.clone();
899                    }
900                    // Even if we didn't take the new guid, we are going to
901                    // take the new visits - so we want the change counter to
902                    // reflect there are changes.
903                    counter_incr = 1;
904                }
905                info.page
906            }
907            None => {
908                // Before we insert a new page_info, make sure we actually will
909                // have any visits to add.
910                if visits.is_empty() {
911                    return Ok(());
912                }
913                new_page_info(db, url, Some(incoming_guid.clone()))?
914            }
915        };
916
917        if !visits.is_empty() {
918            // Skip visits that are in tombstones, or that happen at the same time
919            // as visit that's already present. The 2nd lets us avoid inserting
920            // visits that we sent up to the server in the first place.
921            //
922            // It does cause us to ignore visits that legitimately happen
923            // at the same time, but that's probably fine and not worth
924            // worrying about.
925            let mut visits_to_skip: HashSet<Timestamp> = db.query_rows_into(
926                &format!(
927                    "SELECT t.visit_date AS visit_date
928                     FROM moz_historyvisit_tombstones t
929                     WHERE t.place_id = {place}
930                       AND t.visit_date IN ({dates})
931                     UNION ALL
932                     SELECT v.visit_date AS visit_date
933                     FROM moz_historyvisits v
934                     WHERE v.place_id = {place}
935                       AND v.visit_date IN ({dates})",
936                    place = page_info.row_id,
937                    dates = sql_support::repeat_display(visits.len(), ",", |i, f| write!(
938                        f,
939                        "{}",
940                        Timestamp::from(visits[i].date).0
941                    )),
942                ),
943                [],
944                |row| row.get::<_, Timestamp>(0),
945            )?;
946
947            visits_to_skip.reserve(visits.len());
948
949            for visit in visits {
950                let timestamp = Timestamp::from(visit.date);
951                // Don't insert visits that have been locally deleted.
952                if visits_to_skip.contains(&timestamp) {
953                    continue;
954                }
955                let transition = VisitType::from_primitive(visit.transition)
956                    .expect("these should already be validated");
957                add_visit(
958                    db,
959                    page_info.row_id,
960                    None,
961                    timestamp,
962                    transition,
963                    false,
964                    serialize_unknown_fields(&visit.unknown_fields)?,
965                )?;
966                // Make sure that even if a history entry weirdly has the same visit
967                // twice, we don't insert it twice. (This avoids us needing to
968                // recompute visits_to_skip in each step of the iteration)
969                visits_to_skip.insert(timestamp);
970            }
971        }
972        // XXX - we really need a better story for frecency-boost than
973        // Option<bool> - None vs Some(false) is confusing. We should use an enum.
974        update_frecency(db, page_info.row_id, None)?;
975
976        // and the place itself if necessary.
977        let new_title = title.as_ref().unwrap_or(&page_info.title);
978        // We set the Status to Normal, otherwise we will re-upload it as
979        // outgoing even if nothing has changed. Note that we *do not* reset
980        // the change counter - if it is non-zero now, we want it to remain
981        // as non-zero, so we do re-upload it if there were actual changes)
982        db.execute_cached(
983            "UPDATE moz_places
984             SET title = :title,
985                 unknown_fields = :unknown_fields,
986                 sync_status = :status,
987                 sync_change_counter = :sync_change_counter
988             WHERE id == :row_id",
989            &[
990                (":title", new_title as &dyn rusqlite::ToSql),
991                (":row_id", &page_info.row_id),
992                (":status", &SyncStatus::Normal),
993                (
994                    ":unknown_fields",
995                    &serialize_unknown_fields(unknown_fields)?,
996                ),
997                (
998                    ":sync_change_counter",
999                    &(page_info.sync_change_counter + counter_incr),
1000                ),
1001            ],
1002        )?;
1003
1004        Ok(())
1005    }
1006
1007    pub fn apply_synced_reconciliation(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1008        db.execute_cached(
1009            "UPDATE moz_places
1010                SET sync_status = :status,
1011                    sync_change_counter = 0
1012             WHERE guid == :guid",
1013            &[
1014                (":guid", guid as &dyn rusqlite::ToSql),
1015                (":status", &SyncStatus::Normal),
1016            ],
1017        )?;
1018        Ok(())
1019    }
1020
1021    pub fn apply_synced_deletion(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1022        // First we delete any visits for the page
1023        // because it's possible the moz_places foreign_count is not 0
1024        // and thus the moz_places entry won't be deleted.
1025        db.execute_cached(
1026            "DELETE FROM moz_historyvisits
1027              WHERE place_id IN (
1028                  SELECT id
1029                  FROM moz_places
1030                  WHERE guid = :guid
1031              )",
1032            &[(":guid", guid)],
1033        )?;
1034        db.execute_cached(
1035            "DELETE FROM moz_places WHERE guid = :guid AND foreign_count = 0",
1036            &[(":guid", guid)],
1037        )?;
1038        Ok(())
1039    }
1040
1041    pub fn fetch_outgoing(
1042        db: &PlacesDb,
1043        max_places: usize,
1044        max_visits: usize,
1045    ) -> Result<Vec<OutgoingBso>> {
1046        // Note that we want *all* "new" regardless of change counter,
1047        // so that we do the right thing after a "reset". We also
1048        // exclude hidden URLs from syncing, to match Desktop
1049        // (bug 1173359).
1050        let places_sql = format!(
1051            "
1052            SELECT guid, url, id, title, hidden, typed, frecency,
1053                visit_count_local, visit_count_remote,
1054                last_visit_date_local, last_visit_date_remote,
1055                sync_status, sync_change_counter, preview_image_url,
1056                unknown_fields
1057            FROM moz_places
1058            WHERE (sync_change_counter > 0 OR sync_status != {}) AND
1059                  NOT hidden
1060            ORDER BY frecency DESC
1061            LIMIT :max_places",
1062            (SyncStatus::Normal as u8)
1063        );
1064        let visits_sql = "
1065            SELECT visit_date as date, visit_type as transition, unknown_fields
1066            FROM moz_historyvisits
1067            WHERE place_id = :place_id
1068            ORDER BY visit_date DESC
1069            LIMIT :max_visits";
1070        // tombstones
1071        let tombstones_sql = "SELECT guid FROM moz_places_tombstones LIMIT :max_places";
1072
1073        let mut tombstone_ids = HashSet::new();
1074        let mut result = Vec::new();
1075
1076        // We want to limit to 5000 places - tombstones are arguably the
1077        // most important, so we fetch these first.
1078        let ts_rows = db.query_rows_and_then(
1079            tombstones_sql,
1080            &[(":max_places", &(max_places as u32))],
1081            |row| -> rusqlite::Result<SyncGuid> { Ok(row.get::<_, String>("guid")?.into()) },
1082        )?;
1083        // It's unfortunatee that query_rows_and_then returns a Vec instead of an iterator
1084        // (which would be very hard to do), but as long as we have it, we might as well make use
1085        // of it...
1086        result.reserve(ts_rows.len());
1087        tombstone_ids.reserve(ts_rows.len());
1088        for guid in ts_rows {
1089            trace!("outgoing tombstone {:?}", &guid);
1090            let envelope = OutgoingEnvelope {
1091                id: guid.clone(),
1092                ttl: Some(HISTORY_TTL),
1093                ..Default::default()
1094            };
1095            result.push(OutgoingBso::new_tombstone(envelope));
1096            tombstone_ids.insert(guid);
1097        }
1098
1099        // Max records is now limited by how many tombstones we found.
1100        let max_places_left = max_places - result.len();
1101
1102        // We write info about the records we are updating to a temp table.
1103        // While we could carry this around in memory, we'll need a temp table
1104        // in `finish_outgoing` anyway, because we execute a `NOT IN` query
1105        // there - which, in a worst-case scenario, is a very large `NOT IN`
1106        // set.
1107        db.execute(
1108            "CREATE TEMP TABLE IF NOT EXISTS temp_sync_updated_meta
1109                    (id INTEGER PRIMARY KEY,
1110                     change_delta INTEGER NOT NULL)",
1111            [],
1112        )?;
1113
1114        let insert_meta_sql = "
1115            INSERT INTO temp_sync_updated_meta VALUES (:row_id, :change_delta)";
1116
1117        let rows = db.query_rows_and_then(
1118            &places_sql,
1119            &[(":max_places", &(max_places_left as u32))],
1120            PageInfo::from_row,
1121        )?;
1122        result.reserve(rows.len());
1123        let mut ids_to_update = Vec::with_capacity(rows.len());
1124        for page in rows {
1125            let visits = db.query_rows_and_then_cached(
1126                visits_sql,
1127                &[
1128                    (":max_visits", &(max_visits as u32) as &dyn rusqlite::ToSql),
1129                    (":place_id", &page.row_id),
1130                ],
1131                |row| -> Result<_> {
1132                    Ok(HistoryRecordVisit {
1133                        date: row.get::<_, Timestamp>("date")?.into(),
1134                        transition: row.get::<_, u8>("transition")?,
1135                        unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
1136                            None => UnknownFields::new(),
1137                            Some(v) => serde_json::from_str(&v)?,
1138                        },
1139                    })
1140                },
1141            )?;
1142            if tombstone_ids.contains(&page.guid) {
1143                // should be impossible!
1144                warn!("Found {:?} in both tombstones and live records", &page.guid);
1145                continue;
1146            }
1147            if visits.is_empty() {
1148                // This will be true for things like bookmarks which haven't
1149                // had visits locally applied, and if we later prune old visits
1150                // we'll also hit it, so don't make much log noise.
1151                trace!(
1152                    "Page {:?} is flagged to be uploaded, but has no visits - skipping",
1153                    &page.guid
1154                );
1155                continue;
1156            }
1157            trace!("outgoing record {:?}", &page.guid);
1158            ids_to_update.push(page.row_id);
1159            db.execute_cached(
1160                insert_meta_sql,
1161                &[
1162                    (":row_id", &page.row_id as &dyn rusqlite::ToSql),
1163                    (":change_delta", &page.sync_change_counter),
1164                ],
1165            )?;
1166
1167            let content = HistoryRecord {
1168                id: page.guid.clone(),
1169                title: page.title,
1170                hist_uri: page.url.to_string(),
1171                visits,
1172                unknown_fields: page.unknown_fields,
1173            };
1174
1175            let envelope = OutgoingEnvelope {
1176                id: page.guid,
1177                sortindex: Some(page.frecency),
1178                ttl: Some(HISTORY_TTL),
1179            };
1180            let bso = OutgoingBso::from_content(envelope, content)?;
1181            result.push(bso);
1182        }
1183
1184        // We need to update the sync status of these items now rather than after
1185        // the upload, because if we are interrupted between upload and writing
1186        // we could end up with local items with state New even though we
1187        // uploaded them.
1188        sql_support::each_chunk(&ids_to_update, |chunk, _| -> Result<()> {
1189            db.conn().execute(
1190                &format!(
1191                    "UPDATE moz_places SET sync_status={status}
1192                                 WHERE id IN ({vars})",
1193                    vars = sql_support::repeat_sql_vars(chunk.len()),
1194                    status = SyncStatus::Normal as u8
1195                ),
1196                rusqlite::params_from_iter(chunk),
1197            )?;
1198            Ok(())
1199        })?;
1200
1201        Ok(result)
1202    }
1203
1204    pub fn finish_outgoing(db: &PlacesDb) -> Result<()> {
1205        // So all items *other* than those above must be set to "not dirty"
1206        // (ie, status=SyncStatus::Normal, change_counter=0). Otherwise every
1207        // subsequent sync will continue to add more and more local pages
1208        // until every page we have is uploaded. And we only want to do it
1209        // at the end of the sync because if we are interrupted, we'll end up
1210        // thinking we have nothing to upload.
1211        // BUT - this is potentially alot of rows! Because we want "NOT IN (...)"
1212        // we can't do chunking and building a literal string with the ids seems
1213        // wrong and likely to hit max sql length limits.
1214        // So we use a temp table.
1215        debug!("Updating all synced rows");
1216        // XXX - is there a better way to express this SQL? Multi-selects
1217        // doesn't seem ideal...
1218        db.conn().execute_cached(
1219            "
1220            UPDATE moz_places
1221                SET sync_change_counter = sync_change_counter -
1222                (SELECT change_delta FROM temp_sync_updated_meta m WHERE moz_places.id = m.id)
1223            WHERE id IN (SELECT id FROM temp_sync_updated_meta)
1224            ",
1225            [],
1226        )?;
1227
1228        debug!("Updating all non-synced rows");
1229        db.execute_all(&[
1230            &format!(
1231                "UPDATE moz_places
1232                    SET sync_change_counter = 0, sync_status = {}
1233                WHERE id NOT IN (SELECT id from temp_sync_updated_meta)",
1234                (SyncStatus::Normal as u8)
1235            ),
1236            "DELETE FROM temp_sync_updated_meta",
1237        ])?;
1238
1239        debug!("Removing local tombstones");
1240        db.conn()
1241            .execute_cached("DELETE from moz_places_tombstones", [])?;
1242
1243        Ok(())
1244    }
1245
1246    /// Resets all sync metadata, including change counters, sync statuses,
1247    /// the last sync time, and sync ID. This should be called when the user
1248    /// signs out of Sync.
1249    pub(crate) fn reset(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
1250        let tx = db.begin_transaction()?;
1251        reset_in_tx(db, assoc)?;
1252        tx.commit()?;
1253        Ok(())
1254    }
1255} // end of sync module.
1256
1257pub fn get_visited<I>(db: &PlacesDb, urls: I) -> Result<Vec<bool>>
1258where
1259    I: IntoIterator<Item = Url>,
1260    I::IntoIter: ExactSizeIterator,
1261{
1262    let iter = urls.into_iter();
1263    let mut result = vec![false; iter.len()];
1264    let url_idxs = iter.enumerate().collect::<Vec<_>>();
1265    get_visited_into(db, &url_idxs, &mut result)?;
1266    Ok(result)
1267}
1268
1269/// Low level api used to implement both get_visited and the FFI get_visited call.
1270/// Takes a slice where we should output the results, as well as a slice of
1271/// index/url pairs.
1272///
1273/// This is done so that the FFI can more easily support returning
1274/// false when asked if it's visited an invalid URL.
1275pub fn get_visited_into(
1276    db: &PlacesDb,
1277    urls_idxs: &[(usize, Url)],
1278    result: &mut [bool],
1279) -> Result<()> {
1280    sql_support::each_chunk_mapped(
1281        urls_idxs,
1282        |(_, url)| url.as_str(),
1283        |chunk, offset| -> Result<()> {
1284            let values_with_idx = sql_support::repeat_display(chunk.len(), ",", |i, f| {
1285                let (idx, url) = &urls_idxs[i + offset];
1286                write!(f, "({},{},?)", *idx, hash::hash_url(url.as_str()))
1287            });
1288            let sql = format!(
1289                "WITH to_fetch(fetch_url_index, url_hash, url) AS (VALUES {})
1290                 SELECT fetch_url_index
1291                 FROM moz_places h
1292                 JOIN to_fetch f ON h.url_hash = f.url_hash
1293                   AND h.url = f.url
1294                   AND (h.last_visit_date_local + h.last_visit_date_remote) != 0",
1295                values_with_idx
1296            );
1297            let mut stmt = db.prepare(&sql)?;
1298            for idx_r in stmt.query_and_then(
1299                rusqlite::params_from_iter(chunk),
1300                |row| -> rusqlite::Result<_> { Ok(row.get::<_, i64>(0)? as usize) },
1301            )? {
1302                let idx = idx_r?;
1303                result[idx] = true;
1304            }
1305            Ok(())
1306        },
1307    )?;
1308    Ok(())
1309}
1310
1311/// Get the set of urls that were visited between `start` and `end`. Only considers local visits
1312/// unless you pass in `include_remote`.
1313pub fn get_visited_urls(
1314    db: &PlacesDb,
1315    start: Timestamp,
1316    end: Timestamp,
1317    include_remote: bool,
1318) -> Result<Vec<String>> {
1319    // TODO: if `end` is >= now then we can probably just look at last_visit_date_{local,remote},
1320    // and avoid touching `moz_historyvisits` at all. That said, this query is taken more or less
1321    // from what places does so it's probably fine.
1322    let sql = format!(
1323        "SELECT h.url
1324        FROM moz_places h
1325        WHERE EXISTS (
1326            SELECT 1 FROM moz_historyvisits v
1327            WHERE place_id = h.id
1328                AND visit_date BETWEEN :start AND :end
1329                {and_is_local}
1330            LIMIT 1
1331        )",
1332        and_is_local = if include_remote { "" } else { "AND is_local" }
1333    );
1334    Ok(db.query_rows_and_then_cached(
1335        &sql,
1336        &[(":start", &start), (":end", &end)],
1337        |row| -> RusqliteResult<_> { row.get::<_, String>(0) },
1338    )?)
1339}
1340
1341pub fn get_top_frecent_site_infos(
1342    db: &PlacesDb,
1343    num_items: i32,
1344    frecency_threshold: i64,
1345) -> Result<Vec<TopFrecentSiteInfo>> {
1346    // Get the complement of the visit types that should be excluded.
1347    let allowed_types = VisitTransitionSet::for_specific(&[
1348        VisitType::Download,
1349        VisitType::Embed,
1350        VisitType::RedirectPermanent,
1351        VisitType::RedirectTemporary,
1352        VisitType::FramedLink,
1353        VisitType::Reload,
1354    ])
1355    .complement();
1356
1357    let infos = db.query_rows_and_then_cached(
1358        "SELECT h.frecency, h.title, h.url
1359        FROM moz_places h
1360        WHERE EXISTS (
1361            SELECT v.visit_type
1362            FROM moz_historyvisits v
1363            WHERE h.id = v.place_id
1364              AND (SUBSTR(h.url, 1, 6) == 'https:' OR SUBSTR(h.url, 1, 5) == 'http:')
1365              AND (h.last_visit_date_local + h.last_visit_date_remote) != 0
1366              AND ((1 << v.visit_type) & :allowed_types) != 0
1367              AND h.frecency >= :frecency_threshold AND
1368              NOT h.hidden
1369        )
1370        ORDER BY h.frecency DESC
1371        LIMIT :limit",
1372        rusqlite::named_params! {
1373            ":limit": num_items,
1374            ":allowed_types": allowed_types,
1375            ":frecency_threshold": frecency_threshold,
1376        },
1377        TopFrecentSiteInfo::from_row,
1378    )?;
1379    Ok(infos)
1380}
1381
1382pub fn get_visit_infos(
1383    db: &PlacesDb,
1384    start: Timestamp,
1385    end: Timestamp,
1386    exclude_types: VisitTransitionSet,
1387) -> Result<Vec<HistoryVisitInfo>> {
1388    let allowed_types = exclude_types.complement();
1389    let infos = db.query_rows_and_then_cached(
1390        "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1391                v.is_local
1392         FROM moz_places h
1393         JOIN moz_historyvisits v
1394           ON h.id = v.place_id
1395         WHERE v.visit_date BETWEEN :start AND :end
1396           AND ((1 << visit_type) & :allowed_types) != 0 AND
1397           NOT h.hidden
1398         ORDER BY v.visit_date",
1399        rusqlite::named_params! {
1400            ":start": start,
1401            ":end": end,
1402            ":allowed_types": allowed_types,
1403        },
1404        HistoryVisitInfo::from_row,
1405    )?;
1406    Ok(infos)
1407}
1408
1409pub fn get_visit_count(db: &PlacesDb, exclude_types: VisitTransitionSet) -> Result<i64> {
1410    let count = if exclude_types.is_empty() {
1411        db.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")?
1412    } else {
1413        let allowed_types = exclude_types.complement();
1414        db.query_row_and_then_cachable(
1415            "SELECT COUNT(*)
1416             FROM moz_historyvisits
1417             WHERE ((1 << visit_type) & :allowed_types) != 0",
1418            rusqlite::named_params! {
1419                ":allowed_types": allowed_types,
1420            },
1421            |r| r.get(0),
1422            true,
1423        )?
1424    };
1425    Ok(count)
1426}
1427
1428pub fn get_visit_count_for_host(
1429    db: &PlacesDb,
1430    host: &str,
1431    before: Timestamp,
1432    exclude_types: VisitTransitionSet,
1433) -> Result<i64> {
1434    let allowed_types = exclude_types.complement();
1435    let count = db.query_row_and_then_cachable(
1436        "SELECT COUNT(*)
1437        FROM moz_historyvisits
1438        JOIN moz_places ON moz_places.id = moz_historyvisits.place_id
1439        JOIN moz_origins ON moz_origins.id = moz_places.origin_id
1440        WHERE moz_origins.host = :host
1441          AND visit_date < :before
1442          AND ((1 << visit_type) & :allowed_types) != 0",
1443        rusqlite::named_params! {
1444            ":host": host,
1445            ":before": before,
1446            ":allowed_types": allowed_types,
1447        },
1448        |r| r.get(0),
1449        true,
1450    )?;
1451    Ok(count)
1452}
1453
1454pub fn get_visit_page(
1455    db: &PlacesDb,
1456    offset: i64,
1457    count: i64,
1458    exclude_types: VisitTransitionSet,
1459) -> Result<Vec<HistoryVisitInfo>> {
1460    let allowed_types = exclude_types.complement();
1461    let infos = db.query_rows_and_then_cached(
1462        "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1463                v.is_local
1464         FROM moz_places h
1465         JOIN moz_historyvisits v
1466           ON h.id = v.place_id
1467         WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1468               NOT h.hidden
1469         ORDER BY v.visit_date DESC, v.id
1470         LIMIT :count
1471         OFFSET :offset",
1472        rusqlite::named_params! {
1473            ":count": count,
1474            ":offset": offset,
1475            ":allowed_types": allowed_types,
1476        },
1477        HistoryVisitInfo::from_row,
1478    )?;
1479    Ok(infos)
1480}
1481
1482pub fn get_visit_page_with_bound(
1483    db: &PlacesDb,
1484    bound: i64,
1485    offset: i64,
1486    count: i64,
1487    exclude_types: VisitTransitionSet,
1488) -> Result<HistoryVisitInfosWithBound> {
1489    let allowed_types = exclude_types.complement();
1490    let infos = db.query_rows_and_then_cached(
1491        "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1492                v.is_local
1493         FROM moz_places h
1494         JOIN moz_historyvisits v
1495           ON h.id = v.place_id
1496         WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1497               NOT h.hidden
1498               AND v.visit_date <= :bound
1499         ORDER BY v.visit_date DESC, v.id
1500         LIMIT :count
1501         OFFSET :offset",
1502        rusqlite::named_params! {
1503            ":allowed_types": allowed_types,
1504            ":bound": bound,
1505            ":count": count,
1506            ":offset": offset,
1507        },
1508        HistoryVisitInfo::from_row,
1509    )?;
1510
1511    if let Some(l) = infos.last() {
1512        if l.timestamp.as_millis_i64() == bound {
1513            // all items' timestamp are equal to the previous bound
1514            let offset = offset + infos.len() as i64;
1515            Ok(HistoryVisitInfosWithBound {
1516                infos,
1517                bound,
1518                offset,
1519            })
1520        } else {
1521            let bound = l.timestamp;
1522            let offset = infos
1523                .iter()
1524                .rev()
1525                .take_while(|i| i.timestamp == bound)
1526                .count() as i64;
1527            Ok(HistoryVisitInfosWithBound {
1528                infos,
1529                bound: bound.as_millis_i64(),
1530                offset,
1531            })
1532        }
1533    } else {
1534        // infos is Empty
1535        Ok(HistoryVisitInfosWithBound {
1536            infos,
1537            bound: 0,
1538            offset: 0,
1539        })
1540    }
1541}
1542
1543#[cfg(test)]
1544mod tests {
1545    use super::history_sync::*;
1546    use super::*;
1547    use crate::history_sync::record::HistoryRecordVisit;
1548    use crate::storage::bookmarks::{insert_bookmark, InsertableItem};
1549    use crate::types::VisitTransitionSet;
1550    use crate::{api::places_api::ConnectionType, storage::bookmarks::BookmarkRootGuid};
1551    use std::time::{Duration, SystemTime};
1552    use sync15::engine::CollSyncIds;
1553    use types::Timestamp;
1554
1555    #[test]
1556    fn test_get_visited_urls() {
1557        use std::collections::HashSet;
1558        use std::time::SystemTime;
1559        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1560        let now: Timestamp = SystemTime::now().into();
1561        let now_u64 = now.0;
1562        // (url, when, is_remote, (expected_always, expected_only_local)
1563        let to_add = [
1564            (
1565                "https://www.example.com/1",
1566                now_u64 - 200_100,
1567                false,
1568                (false, false),
1569            ),
1570            (
1571                "https://www.example.com/12",
1572                now_u64 - 200_000,
1573                false,
1574                (true, true),
1575            ),
1576            (
1577                "https://www.example.com/123",
1578                now_u64 - 10_000,
1579                true,
1580                (true, false),
1581            ),
1582            (
1583                "https://www.example.com/1234",
1584                now_u64 - 1000,
1585                false,
1586                (true, true),
1587            ),
1588            (
1589                "https://www.mozilla.com",
1590                now_u64 - 500,
1591                false,
1592                (false, false),
1593            ),
1594        ];
1595
1596        for &(url, when, remote, _) in &to_add {
1597            apply_observation(
1598                &conn,
1599                VisitObservation::new(Url::parse(url).unwrap())
1600                    .with_at(Timestamp(when))
1601                    .with_is_remote(remote)
1602                    .with_visit_type(VisitType::Link),
1603            )
1604            .expect("Should apply visit");
1605        }
1606
1607        let visited_all = get_visited_urls(
1608            &conn,
1609            Timestamp(now_u64 - 200_000),
1610            Timestamp(now_u64 - 1000),
1611            true,
1612        )
1613        .unwrap()
1614        .into_iter()
1615        .collect::<HashSet<_>>();
1616
1617        let visited_local = get_visited_urls(
1618            &conn,
1619            Timestamp(now_u64 - 200_000),
1620            Timestamp(now_u64 - 1000),
1621            false,
1622        )
1623        .unwrap()
1624        .into_iter()
1625        .collect::<HashSet<_>>();
1626
1627        for &(url, ts, is_remote, (expected_in_all, expected_in_local)) in &to_add {
1628            // Make sure we format stuff the same way (in practice, just trailing slashes)
1629            let url = Url::parse(url).unwrap().to_string();
1630            assert_eq!(
1631                expected_in_local,
1632                visited_local.contains(&url),
1633                "Failed in local for {:?}",
1634                (url, ts, is_remote)
1635            );
1636            assert_eq!(
1637                expected_in_all,
1638                visited_all.contains(&url),
1639                "Failed in all for {:?}",
1640                (url, ts, is_remote)
1641            );
1642        }
1643    }
1644
1645    fn get_custom_observed_page<F>(conn: &mut PlacesDb, url: &str, custom: F) -> Result<PageInfo>
1646    where
1647        F: Fn(VisitObservation) -> VisitObservation,
1648    {
1649        let u = Url::parse(url)?;
1650        let obs = VisitObservation::new(u.clone()).with_visit_type(VisitType::Link);
1651        apply_observation(conn, custom(obs))?;
1652        Ok(fetch_page_info(conn, &u)?
1653            .expect("should have the page")
1654            .page)
1655    }
1656
1657    fn get_observed_page(conn: &mut PlacesDb, url: &str) -> Result<PageInfo> {
1658        get_custom_observed_page(conn, url, |o| o)
1659    }
1660
1661    fn get_tombstone_count(conn: &PlacesDb) -> u32 {
1662        let result: Result<Option<u32>> = conn.try_query_row(
1663            "SELECT COUNT(*) from moz_places_tombstones;",
1664            [],
1665            |row| Ok(row.get::<_, u32>(0)?),
1666            true,
1667        );
1668        result
1669            .expect("should have worked")
1670            .expect("should have got a value")
1671    }
1672
1673    #[test]
1674    fn test_visit_counts() -> Result<()> {
1675        error_support::init_for_tests();
1676        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1677        let url = Url::parse("https://www.example.com").expect("it's a valid url");
1678        let early_time = SystemTime::now() - Duration::new(60, 0);
1679        let late_time = SystemTime::now();
1680
1681        // add 2 local visits - add latest first
1682        let rid1 = apply_observation(
1683            &conn,
1684            VisitObservation::new(url.clone())
1685                .with_visit_type(VisitType::Link)
1686                .with_at(Some(late_time.into())),
1687        )?
1688        .expect("should get a rowid");
1689
1690        let rid2 = apply_observation(
1691            &conn,
1692            VisitObservation::new(url.clone())
1693                .with_visit_type(VisitType::Link)
1694                .with_at(Some(early_time.into())),
1695        )?
1696        .expect("should get a rowid");
1697
1698        let mut pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1699        assert_eq!(pi.page.visit_count_local, 2);
1700        assert_eq!(pi.page.last_visit_date_local, late_time.into());
1701        assert_eq!(pi.page.visit_count_remote, 0);
1702        assert_eq!(pi.page.last_visit_date_remote.0, 0);
1703
1704        // 2 remote visits, earliest first.
1705        let rid3 = apply_observation(
1706            &conn,
1707            VisitObservation::new(url.clone())
1708                .with_visit_type(VisitType::Link)
1709                .with_at(Some(early_time.into()))
1710                .with_is_remote(true),
1711        )?
1712        .expect("should get a rowid");
1713
1714        let rid4 = apply_observation(
1715            &conn,
1716            VisitObservation::new(url.clone())
1717                .with_visit_type(VisitType::Link)
1718                .with_at(Some(late_time.into()))
1719                .with_is_remote(true),
1720        )?
1721        .expect("should get a rowid");
1722
1723        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1724        assert_eq!(pi.page.visit_count_local, 2);
1725        assert_eq!(pi.page.last_visit_date_local, late_time.into());
1726        assert_eq!(pi.page.visit_count_remote, 2);
1727        assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1728
1729        // Delete some and make sure things update.
1730        // XXX - we should add a trigger to update frecency on delete, but at
1731        // this stage we don't "officially" support deletes, so this is TODO.
1732        let sql = "DELETE FROM moz_historyvisits WHERE id = :row_id";
1733        // Delete the latest local visit.
1734        conn.execute_cached(sql, &[(":row_id", &rid1)])?;
1735        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1736        assert_eq!(pi.page.visit_count_local, 1);
1737        assert_eq!(pi.page.last_visit_date_local, early_time.into());
1738        assert_eq!(pi.page.visit_count_remote, 2);
1739        assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1740
1741        // Delete the earliest remote  visit.
1742        conn.execute_cached(sql, &[(":row_id", &rid3)])?;
1743        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1744        assert_eq!(pi.page.visit_count_local, 1);
1745        assert_eq!(pi.page.last_visit_date_local, early_time.into());
1746        assert_eq!(pi.page.visit_count_remote, 1);
1747        assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1748
1749        // Delete all visits.
1750        conn.execute_cached(sql, &[(":row_id", &rid2)])?;
1751        conn.execute_cached(sql, &[(":row_id", &rid4)])?;
1752        // It may turn out that we also delete the place after deleting all
1753        // visits, but for now we don't - check the values are sane though.
1754        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1755        assert_eq!(pi.page.visit_count_local, 0);
1756        assert_eq!(pi.page.last_visit_date_local, Timestamp(0));
1757        assert_eq!(pi.page.visit_count_remote, 0);
1758        assert_eq!(pi.page.last_visit_date_remote, Timestamp(0));
1759        Ok(())
1760    }
1761
1762    #[test]
1763    fn test_get_visited() -> Result<()> {
1764        error_support::init_for_tests();
1765        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1766
1767        let unicode_in_path = "http://www.example.com/tëst😀abc";
1768        let escaped_unicode_in_path = "http://www.example.com/t%C3%ABst%F0%9F%98%80abc";
1769
1770        let unicode_in_domain = "http://www.exämple😀123.com";
1771        let escaped_unicode_in_domain = "http://www.xn--exmple123-w2a24222l.com";
1772
1773        let to_add = [
1774            "https://www.example.com/1".to_string(),
1775            "https://www.example.com/12".to_string(),
1776            "https://www.example.com/123".to_string(),
1777            "https://www.example.com/1234".to_string(),
1778            "https://www.mozilla.com".to_string(),
1779            "https://www.firefox.com".to_string(),
1780            unicode_in_path.to_string() + "/1",
1781            escaped_unicode_in_path.to_string() + "/2",
1782            unicode_in_domain.to_string() + "/1",
1783            escaped_unicode_in_domain.to_string() + "/2",
1784        ];
1785
1786        for item in &to_add {
1787            apply_observation(
1788                &conn,
1789                VisitObservation::new(Url::parse(item).unwrap()).with_visit_type(VisitType::Link),
1790            )?;
1791        }
1792
1793        let to_search = [
1794            ("https://www.example.com".to_string(), false),
1795            ("https://www.example.com/1".to_string(), true),
1796            ("https://www.example.com/12".to_string(), true),
1797            ("https://www.example.com/123".to_string(), true),
1798            ("https://www.example.com/1234".to_string(), true),
1799            ("https://www.example.com/12345".to_string(), false),
1800            ("https://www.mozilla.com".to_string(), true),
1801            ("https://www.firefox.com".to_string(), true),
1802            ("https://www.mozilla.org".to_string(), false),
1803            // dupes should still work!
1804            ("https://www.example.com/1234".to_string(), true),
1805            ("https://www.example.com/12345".to_string(), false),
1806            // The unicode URLs should work when escaped the way we
1807            // encountered them
1808            (unicode_in_path.to_string() + "/1", true),
1809            (escaped_unicode_in_path.to_string() + "/2", true),
1810            (unicode_in_domain.to_string() + "/1", true),
1811            (escaped_unicode_in_domain.to_string() + "/2", true),
1812            // But also the other way.
1813            (unicode_in_path.to_string() + "/2", true),
1814            (escaped_unicode_in_path.to_string() + "/1", true),
1815            (unicode_in_domain.to_string() + "/2", true),
1816            (escaped_unicode_in_domain.to_string() + "/1", true),
1817        ];
1818
1819        let urls = to_search
1820            .iter()
1821            .map(|(url, _expect)| Url::parse(url).unwrap())
1822            .collect::<Vec<_>>();
1823
1824        let visited = get_visited(&conn, urls).unwrap();
1825
1826        assert_eq!(visited.len(), to_search.len());
1827
1828        for (i, &did_see) in visited.iter().enumerate() {
1829            assert_eq!(
1830                did_see,
1831                to_search[i].1,
1832                "Wrong value in get_visited for '{}' (idx {}), want {}, have {}",
1833                to_search[i].0,
1834                i, // idx is logged because some things are repeated
1835                to_search[i].1,
1836                did_see
1837            );
1838        }
1839        Ok(())
1840    }
1841
1842    #[test]
1843    fn test_get_visited_into() {
1844        error_support::init_for_tests();
1845        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
1846
1847        let u0 = Url::parse("https://www.example.com/1").unwrap();
1848        let u1 = Url::parse("https://www.example.com/12").unwrap();
1849        let u2 = Url::parse("https://www.example.com/123").unwrap();
1850        let u3 = Url::parse("https://www.example.com/1234").unwrap();
1851        let u4 = Url::parse("https://www.example.com/12345").unwrap();
1852
1853        let to_add = [(&u0, false), (&u1, false), (&u2, false), (&u3, true)];
1854        for (item, is_remote) in to_add {
1855            apply_observation(
1856                &conn,
1857                VisitObservation::new(item.clone())
1858                    .with_visit_type(VisitType::Link)
1859                    .with_is_remote(is_remote),
1860            )
1861            .unwrap();
1862        }
1863        // Bookmarked, so exists in `moz_places`;
1864        // but doesn't have a last visit time, so shouldn't be visited.
1865        insert_bookmark(
1866            &conn,
1867            crate::InsertableBookmark {
1868                parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
1869                position: crate::BookmarkPosition::Append,
1870                date_added: None,
1871                last_modified: None,
1872                guid: None,
1873                url: u4.clone(),
1874                title: Some("Title".to_string()),
1875            }
1876            .into(),
1877        )
1878        .unwrap();
1879
1880        let mut results = [false; 12];
1881
1882        let get_visited_request = [
1883            // 0 blank
1884            (2, u1.clone()),
1885            (1, u0),
1886            // 3 blank
1887            (4, u2),
1888            // 5 blank
1889            // Note: url for 6 is not visited.
1890            (6, Url::parse("https://www.example.com/123456").unwrap()),
1891            // 7 blank
1892            // Note: dupe is allowed
1893            (8, u1),
1894            // 9 is blank
1895            (10, u3),
1896            (11, u4),
1897        ];
1898
1899        get_visited_into(&conn, &get_visited_request, &mut results).unwrap();
1900        let expect = [
1901            false, // 0
1902            true,  // 1
1903            true,  // 2
1904            false, // 3
1905            true,  // 4
1906            false, // 5
1907            false, // 6
1908            false, // 7
1909            true,  // 8
1910            false, // 9
1911            true,  // 10
1912            false, // 11
1913        ];
1914
1915        assert_eq!(expect, results);
1916    }
1917
1918    #[test]
1919    fn test_delete_visited() {
1920        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1921        let late: Timestamp = SystemTime::now().into();
1922        let early: Timestamp = (SystemTime::now() - Duration::from_secs(30)).into();
1923        let url1 = Url::parse("https://www.example.com/1").unwrap();
1924        let url2 = Url::parse("https://www.example.com/2").unwrap();
1925        let url3 = Url::parse("https://www.example.com/3").unwrap();
1926        let url4 = Url::parse("https://www.example.com/4").unwrap();
1927        // (url, when)
1928        let to_add = [
1929            // 2 visits to "https://www.example.com/1", one early, one late.
1930            (&url1, early),
1931            (&url1, late),
1932            // One to url2, only late.
1933            (&url2, late),
1934            // One to url2, only early.
1935            (&url3, early),
1936            // One to url4, only late - this will have SyncStatus::Normal
1937            (&url4, late),
1938        ];
1939
1940        for &(url, when) in &to_add {
1941            apply_observation(
1942                &conn,
1943                VisitObservation::new(url.clone())
1944                    .with_at(when)
1945                    .with_visit_type(VisitType::Link),
1946            )
1947            .expect("Should apply visit");
1948        }
1949        // Check we added what we think we did.
1950        let pi = fetch_page_info(&conn, &url1)
1951            .expect("should work")
1952            .expect("should get the page");
1953        assert_eq!(pi.page.visit_count_local, 2);
1954
1955        let pi2 = fetch_page_info(&conn, &url2)
1956            .expect("should work")
1957            .expect("should get the page");
1958        assert_eq!(pi2.page.visit_count_local, 1);
1959
1960        let pi3 = fetch_page_info(&conn, &url3)
1961            .expect("should work")
1962            .expect("should get the page");
1963        assert_eq!(pi3.page.visit_count_local, 1);
1964
1965        let pi4 = fetch_page_info(&conn, &url4)
1966            .expect("should work")
1967            .expect("should get the page");
1968        assert_eq!(pi4.page.visit_count_local, 1);
1969
1970        conn.execute_cached(
1971            &format!(
1972                "UPDATE moz_places set sync_status = {}
1973                 WHERE url = 'https://www.example.com/4'",
1974                (SyncStatus::Normal as u8)
1975            ),
1976            [],
1977        )
1978        .expect("should work");
1979
1980        // Delete some.
1981        delete_visits_between(&conn, late, Timestamp::now()).expect("should work");
1982        // should have removed one of the visits to /1
1983        let pi = fetch_page_info(&conn, &url1)
1984            .expect("should work")
1985            .expect("should get the page");
1986        assert_eq!(pi.page.visit_count_local, 1);
1987
1988        // should have removed all the visits to /2
1989        assert!(fetch_page_info(&conn, &url2)
1990            .expect("should work")
1991            .is_none());
1992
1993        // Should still have the 1 visit to /3
1994        let pi3 = fetch_page_info(&conn, &url3)
1995            .expect("should work")
1996            .expect("should get the page");
1997        assert_eq!(pi3.page.visit_count_local, 1);
1998
1999        // should have removed all the visits to /4
2000        assert!(fetch_page_info(&conn, &url4)
2001            .expect("should work")
2002            .is_none());
2003        // should be a tombstone for url4 and no others.
2004        assert_eq!(get_tombstone_count(&conn), 1);
2005        // XXX - test frecency?
2006        // XXX - origins?
2007    }
2008
2009    #[test]
2010    fn test_change_counter() -> Result<()> {
2011        error_support::init_for_tests();
2012        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2013        let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2014        // A new observation with just a title (ie, no visit) should update it.
2015        apply_observation(
2016            &conn,
2017            VisitObservation::new(pi.url.clone()).with_title(Some("new title".into())),
2018        )?;
2019        pi = fetch_page_info(&conn, &pi.url)?
2020            .expect("page should exist")
2021            .page;
2022        assert_eq!(pi.title, "new title");
2023        assert_eq!(pi.preview_image_url, None);
2024        assert_eq!(pi.sync_change_counter, 2);
2025        // An observation with just a preview_image_url should not update it.
2026        apply_observation(
2027            &conn,
2028            VisitObservation::new(pi.url.clone()).with_preview_image_url(Some(
2029                Url::parse("https://www.example.com/preview.png").unwrap(),
2030            )),
2031        )?;
2032        pi = fetch_page_info(&conn, &pi.url)?
2033            .expect("page should exist")
2034            .page;
2035        assert_eq!(pi.title, "new title");
2036        assert_eq!(
2037            pi.preview_image_url,
2038            Some(Url::parse("https://www.example.com/preview.png").expect("parsed"))
2039        );
2040        assert_eq!(pi.sync_change_counter, 2);
2041        Ok(())
2042    }
2043
2044    #[test]
2045    fn test_status_columns() -> Result<()> {
2046        error_support::init_for_tests();
2047        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2048        // A page with "normal" and a change counter.
2049        let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2050        assert_eq!(pi.sync_change_counter, 1);
2051        conn.execute_cached(
2052            "UPDATE moz_places
2053                                   SET frecency = 100
2054                                   WHERE id = :id",
2055            &[(":id", &pi.row_id)],
2056        )?;
2057        // A page with "new" and no change counter.
2058        let mut pi2 = get_observed_page(&mut conn, "http://example.com/2")?;
2059        conn.execute_cached(
2060            "UPDATE moz_places
2061                SET sync_status = :status,
2062                sync_change_counter = 0,
2063                frecency = 50
2064            WHERE id = :id",
2065            &[
2066                (":status", &(SyncStatus::New as u8) as &dyn rusqlite::ToSql),
2067                (":id", &pi2.row_id),
2068            ],
2069        )?;
2070
2071        // A second page with "new", a change counter (which will be ignored
2072        // as we will limit such that this isn't sent) and a low frecency.
2073        let mut pi3 = get_observed_page(&mut conn, "http://example.com/3")?;
2074        conn.execute_cached(
2075            "UPDATE moz_places
2076                SET sync_status = :status,
2077                sync_change_counter = 1,
2078                frecency = 10
2079            WHERE id = :id",
2080            &[
2081                (":status", &(SyncStatus::New as u8) as &dyn ToSql),
2082                (":id", &pi3.row_id),
2083            ],
2084        )?;
2085
2086        let outgoing = fetch_outgoing(&conn, 2, 3)?;
2087        assert_eq!(outgoing.len(), 2, "should have restricted to the limit");
2088        // want pi or pi2 (but order is indeterminate) and this seems simpler than sorting.
2089        assert!(outgoing[0].envelope.id != outgoing[1].envelope.id);
2090        assert!(outgoing[0].envelope.id == pi.guid || outgoing[0].envelope.id == pi2.guid);
2091        assert!(outgoing[1].envelope.id == pi.guid || outgoing[1].envelope.id == pi2.guid);
2092        finish_outgoing(&conn)?;
2093
2094        pi = fetch_page_info(&conn, &pi.url)?
2095            .expect("page should exist")
2096            .page;
2097        assert_eq!(pi.sync_change_counter, 0);
2098        pi2 = fetch_page_info(&conn, &pi2.url)?
2099            .expect("page should exist")
2100            .page;
2101        assert_eq!(pi2.sync_change_counter, 0);
2102        assert_eq!(pi2.sync_status, SyncStatus::Normal);
2103
2104        // pi3 wasn't uploaded, but it should still have been changed to
2105        // Normal and had the change counter reset.
2106        pi3 = fetch_page_info(&conn, &pi3.url)?
2107            .expect("page should exist")
2108            .page;
2109        assert_eq!(pi3.sync_change_counter, 0);
2110        assert_eq!(pi3.sync_status, SyncStatus::Normal);
2111        Ok(())
2112    }
2113
2114    #[test]
2115    fn test_delete_visits_for() -> Result<()> {
2116        use crate::storage::bookmarks::{
2117            self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2118        };
2119
2120        let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2121
2122        struct TestPage {
2123            href: &'static str,
2124            synced: bool,
2125            bookmark_title: Option<&'static str>,
2126            keyword: Option<&'static str>,
2127        }
2128
2129        fn page_has_tombstone(conn: &PlacesDb, guid: &SyncGuid) -> Result<bool> {
2130            let exists = conn
2131                .try_query_one::<bool, _>(
2132                    "SELECT EXISTS(SELECT 1 FROM moz_places_tombstones
2133                                   WHERE guid = :guid)",
2134                    rusqlite::named_params! { ":guid" : guid },
2135                    false,
2136                )?
2137                .unwrap_or_default();
2138            Ok(exists)
2139        }
2140
2141        fn page_has_visit_tombstones(conn: &PlacesDb, page_id: RowId) -> Result<bool> {
2142            let exists = conn
2143                .try_query_one::<bool, _>(
2144                    "SELECT EXISTS(SELECT 1 FROM moz_historyvisit_tombstones
2145                                   WHERE place_id = :page_id)",
2146                    rusqlite::named_params! { ":page_id": page_id },
2147                    false,
2148                )?
2149                .unwrap_or_default();
2150            Ok(exists)
2151        }
2152
2153        let pages = &[
2154            // A is synced and has a bookmark, so we should insert tombstones
2155            // for all its visits.
2156            TestPage {
2157                href: "http://example.com/a",
2158                synced: true,
2159                bookmark_title: Some("A"),
2160                keyword: None,
2161            },
2162            // B is synced but only has visits, so we should insert a tombstone
2163            // for the page.
2164            TestPage {
2165                href: "http://example.com/b",
2166                synced: true,
2167                bookmark_title: None,
2168                keyword: None,
2169            },
2170            // C isn't synced but has a keyword, so we should delete all its
2171            // visits, but not the page.
2172            TestPage {
2173                href: "http://example.com/c",
2174                synced: false,
2175                bookmark_title: None,
2176                keyword: Some("one"),
2177            },
2178            // D isn't synced and only has visits, so we should delete it
2179            // entirely.
2180            TestPage {
2181                href: "http://example.com/d",
2182                synced: false,
2183                bookmark_title: None,
2184                keyword: None,
2185            },
2186        ];
2187        for page in pages {
2188            let url = Url::parse(page.href)?;
2189            let obs = VisitObservation::new(url.clone())
2190                .with_visit_type(VisitType::Link)
2191                .with_at(Some(SystemTime::now().into()));
2192            apply_observation(&db, obs)?;
2193
2194            if page.synced {
2195                db.execute_cached(
2196                    &format!(
2197                        "UPDATE moz_places
2198                             SET sync_status = {}
2199                         WHERE url_hash = hash(:url) AND
2200                               url = :url",
2201                        (SyncStatus::Normal as u8)
2202                    ),
2203                    &[(":url", &url.as_str())],
2204                )?;
2205            }
2206
2207            if let Some(title) = page.bookmark_title {
2208                bookmarks::insert_bookmark(
2209                    &db,
2210                    InsertableBookmark {
2211                        parent_guid: BookmarkRootGuid::Unfiled.into(),
2212                        position: BookmarkPosition::Append,
2213                        date_added: None,
2214                        last_modified: None,
2215                        guid: None,
2216                        url: url.clone(),
2217                        title: Some(title.to_owned()),
2218                    }
2219                    .into(),
2220                )?;
2221            }
2222
2223            if let Some(keyword) = page.keyword {
2224                // We don't have a public API for inserting keywords, so just
2225                // write to the database directly.
2226                db.execute_cached(
2227                    "INSERT INTO moz_keywords(place_id, keyword)
2228                     SELECT id, :keyword
2229                     FROM moz_places
2230                     WHERE url_hash = hash(:url) AND
2231                           url = :url",
2232                    &[(":url", &url.as_str()), (":keyword", &keyword)],
2233                )?;
2234            }
2235
2236            // Now delete all visits.
2237            let (info, _) =
2238                fetch_visits(&db, &url, 0)?.expect("Should return visits for test page");
2239            delete_visits_for(&db, &info.guid)?;
2240
2241            match (
2242                page.synced,
2243                page.bookmark_title.is_some() || page.keyword.is_some(),
2244            ) {
2245                (true, true) => {
2246                    let (_, visits) = fetch_visits(&db, &url, 0)?
2247                        .expect("Shouldn't delete synced page with foreign count");
2248                    assert!(
2249                        visits.is_empty(),
2250                        "Should delete all visits from synced page with foreign count"
2251                    );
2252                    assert!(
2253                        !page_has_tombstone(&db, &info.guid)?,
2254                        "Shouldn't insert tombstone for synced page with foreign count"
2255                    );
2256                    assert!(
2257                        page_has_visit_tombstones(&db, info.row_id)?,
2258                        "Should insert visit tombstones for synced page with foreign count"
2259                    );
2260                }
2261                (true, false) => {
2262                    assert!(
2263                        fetch_visits(&db, &url, 0)?.is_none(),
2264                        "Should delete synced page"
2265                    );
2266                    assert!(
2267                        page_has_tombstone(&db, &info.guid)?,
2268                        "Should insert tombstone for synced page"
2269                    );
2270                    assert!(
2271                        !page_has_visit_tombstones(&db, info.row_id)?,
2272                        "Shouldn't insert visit tombstones for synced page"
2273                    );
2274                }
2275                (false, true) => {
2276                    let (_, visits) = fetch_visits(&db, &url, 0)?
2277                        .expect("Shouldn't delete page with foreign count");
2278                    assert!(
2279                        visits.is_empty(),
2280                        "Should delete all visits from page with foreign count"
2281                    );
2282                    assert!(
2283                        !page_has_tombstone(&db, &info.guid)?,
2284                        "Shouldn't insert tombstone for page with foreign count"
2285                    );
2286                    assert!(
2287                        !page_has_visit_tombstones(&db, info.row_id)?,
2288                        "Shouldn't insert visit tombstones for page with foreign count"
2289                    );
2290                }
2291                (false, false) => {
2292                    assert!(fetch_visits(&db, &url, 0)?.is_none(), "Should delete page");
2293                    assert!(
2294                        !page_has_tombstone(&db, &info.guid)?,
2295                        "Shouldn't insert tombstone for page"
2296                    );
2297                    assert!(
2298                        !page_has_visit_tombstones(&db, info.row_id)?,
2299                        "Shouldn't insert visit tombstones for page"
2300                    );
2301                }
2302            }
2303        }
2304
2305        Ok(())
2306    }
2307
2308    #[test]
2309    fn test_tombstones() -> Result<()> {
2310        error_support::init_for_tests();
2311        let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2312        let url = Url::parse("https://example.com")?;
2313        let obs = VisitObservation::new(url.clone())
2314            .with_visit_type(VisitType::Link)
2315            .with_at(Some(SystemTime::now().into()));
2316        apply_observation(&db, obs)?;
2317        let guid = url_to_guid(&db, &url)?.expect("should exist");
2318
2319        delete_visits_for(&db, &guid)?;
2320
2321        // status was "New", so expect no tombstone.
2322        assert_eq!(get_tombstone_count(&db), 0);
2323
2324        let obs = VisitObservation::new(url.clone())
2325            .with_visit_type(VisitType::Link)
2326            .with_at(Some(SystemTime::now().into()));
2327        apply_observation(&db, obs)?;
2328        let new_guid = url_to_guid(&db, &url)?.expect("should exist");
2329
2330        // Set the status to normal
2331        db.execute_cached(
2332            &format!(
2333                "UPDATE moz_places
2334                    SET sync_status = {}
2335                 WHERE guid = :guid",
2336                (SyncStatus::Normal as u8)
2337            ),
2338            &[(":guid", &new_guid)],
2339        )?;
2340        delete_visits_for(&db, &new_guid)?;
2341        assert_eq!(get_tombstone_count(&db), 1);
2342        Ok(())
2343    }
2344
2345    #[test]
2346    fn test_reset() -> Result<()> {
2347        fn mark_all_as_synced(db: &PlacesDb) -> Result<()> {
2348            db.execute_cached(
2349                &format!(
2350                    "UPDATE moz_places set sync_status = {}",
2351                    (SyncStatus::Normal as u8)
2352                ),
2353                [],
2354            )?;
2355            Ok(())
2356        }
2357
2358        error_support::init_for_tests();
2359        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2360
2361        // Add Sync metadata keys, to ensure they're reset.
2362        put_meta(&conn, GLOBAL_SYNCID_META_KEY, &"syncAAAAAAAA")?;
2363        put_meta(&conn, COLLECTION_SYNCID_META_KEY, &"syncBBBBBBBB")?;
2364        put_meta(&conn, LAST_SYNC_META_KEY, &12345)?;
2365
2366        // Delete everything first, to ensure we keep the high-water mark
2367        // (see #2445 for a discussion about that).
2368        delete_everything(&conn)?;
2369
2370        let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2371        mark_all_as_synced(&conn)?;
2372        pi = fetch_page_info(&conn, &pi.url)?
2373            .expect("page should exist")
2374            .page;
2375        assert_eq!(pi.sync_change_counter, 1);
2376        assert_eq!(pi.sync_status, SyncStatus::Normal);
2377
2378        let sync_ids = CollSyncIds {
2379            global: SyncGuid::random(),
2380            coll: SyncGuid::random(),
2381        };
2382        history_sync::reset(&conn, &EngineSyncAssociation::Connected(sync_ids.clone()))?;
2383
2384        assert_eq!(
2385            get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?,
2386            Some(sync_ids.global)
2387        );
2388        assert_eq!(
2389            get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2390            Some(sync_ids.coll)
2391        );
2392        assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2393        assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2394
2395        pi = fetch_page_info(&conn, &pi.url)?
2396            .expect("page should exist")
2397            .page;
2398        assert_eq!(pi.sync_change_counter, 0);
2399        assert_eq!(pi.sync_status, SyncStatus::New);
2400        // Ensure we are going to do a full re-upload after a reset.
2401        let outgoing = fetch_outgoing(&conn, 100, 100)?;
2402        assert_eq!(outgoing.len(), 1);
2403
2404        mark_all_as_synced(&conn)?;
2405        assert!(fetch_outgoing(&conn, 100, 100)?.is_empty());
2406        // ...
2407
2408        // Now simulate a reset on disconnect, and verify we've removed all Sync
2409        // metadata again.
2410        history_sync::reset(&conn, &EngineSyncAssociation::Disconnected)?;
2411
2412        assert_eq!(get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?, None);
2413        assert_eq!(
2414            get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2415            None
2416        );
2417        assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2418        assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2419
2420        Ok(())
2421    }
2422
2423    #[test]
2424    fn test_fetch_visits() -> Result<()> {
2425        error_support::init_for_tests();
2426        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2427        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2428        assert_eq!(fetch_visits(&conn, &pi.url, 0).unwrap().unwrap().1.len(), 0);
2429        assert_eq!(fetch_visits(&conn, &pi.url, 1).unwrap().unwrap().1.len(), 1);
2430        Ok(())
2431    }
2432
2433    #[test]
2434    fn test_apply_synced_reconciliation() -> Result<()> {
2435        error_support::init_for_tests();
2436        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2437        let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2438        assert_eq!(pi.sync_status, SyncStatus::New);
2439        assert_eq!(pi.sync_change_counter, 1);
2440        apply_synced_reconciliation(&conn, &pi.guid)?;
2441        pi = fetch_page_info(&conn, &pi.url)?
2442            .expect("page should exist")
2443            .page;
2444        assert_eq!(pi.sync_status, SyncStatus::Normal);
2445        assert_eq!(pi.sync_change_counter, 0);
2446        Ok(())
2447    }
2448
2449    #[test]
2450    fn test_apply_synced_deletion_new() -> Result<()> {
2451        error_support::init_for_tests();
2452        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2453        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2454        assert_eq!(pi.sync_status, SyncStatus::New);
2455        apply_synced_deletion(&conn, &pi.guid)?;
2456        assert!(
2457            fetch_page_info(&conn, &pi.url)?.is_none(),
2458            "should have been deleted"
2459        );
2460        assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2461        Ok(())
2462    }
2463
2464    #[test]
2465    fn test_apply_synced_deletion_normal() -> Result<()> {
2466        error_support::init_for_tests();
2467        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2468        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2469        assert_eq!(pi.sync_status, SyncStatus::New);
2470        conn.execute_cached(
2471            &format!(
2472                "UPDATE moz_places set sync_status = {}",
2473                (SyncStatus::Normal as u8)
2474            ),
2475            [],
2476        )?;
2477
2478        apply_synced_deletion(&conn, &pi.guid)?;
2479        assert!(
2480            fetch_page_info(&conn, &pi.url)?.is_none(),
2481            "should have been deleted"
2482        );
2483        assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2484        Ok(())
2485    }
2486
2487    #[test]
2488    fn test_apply_synced_deletions_deletes_visits_but_not_page_if_bookmark_exists() -> Result<()> {
2489        error_support::init_for_tests();
2490        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2491        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2492        let item = InsertableItem::Bookmark {
2493            b: crate::InsertableBookmark {
2494                parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
2495                position: crate::BookmarkPosition::Append,
2496                date_added: None,
2497                last_modified: None,
2498                guid: None,
2499                url: pi.url.clone(),
2500                title: Some("Title".to_string()),
2501            },
2502        };
2503        insert_bookmark(&conn, item).unwrap();
2504        apply_synced_deletion(&conn, &pi.guid)?;
2505        let page_info =
2506            fetch_page_info(&conn, &pi.url)?.expect("The places entry should have remained");
2507        assert!(
2508            page_info.last_visit_id.is_none(),
2509            "Should have no more visits"
2510        );
2511        Ok(())
2512    }
2513
2514    fn assert_tombstones(c: &PlacesDb, expected: &[(RowId, Timestamp)]) {
2515        let mut expected: Vec<(RowId, Timestamp)> = expected.into();
2516        expected.sort();
2517        let mut tombstones = c
2518            .query_rows_and_then(
2519                "SELECT place_id, visit_date FROM moz_historyvisit_tombstones",
2520                [],
2521                |row| -> Result<_> { Ok((row.get::<_, RowId>(0)?, row.get::<_, Timestamp>(1)?)) },
2522            )
2523            .unwrap();
2524        tombstones.sort();
2525        assert_eq!(expected, tombstones);
2526    }
2527
2528    #[test]
2529    fn test_visit_tombstones() {
2530        use url::Url;
2531        error_support::init_for_tests();
2532        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2533        let now = Timestamp::now();
2534
2535        let urls = &[
2536            Url::parse("http://example.com/1").unwrap(),
2537            Url::parse("http://example.com/2").unwrap(),
2538        ];
2539
2540        let dates = &[
2541            Timestamp(now.0 - 10000),
2542            Timestamp(now.0 - 5000),
2543            Timestamp(now.0),
2544        ];
2545        for url in urls {
2546            for &date in dates {
2547                get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2548            }
2549        }
2550        delete_place_visit_at_time(&conn, &urls[0], dates[1]).unwrap();
2551        // Delete the most recent visit.
2552        delete_visits_between(&conn, Timestamp(now.0 - 4000), Timestamp::now()).unwrap();
2553
2554        let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2555        assert_eq!(
2556            visits0,
2557            &[FetchedVisit {
2558                is_local: true,
2559                visit_date: dates[0],
2560                visit_type: Some(VisitType::Link)
2561            },]
2562        );
2563
2564        assert!(
2565            !visits0.iter().any(|v| v.visit_date == dates[1]),
2566            "Shouldn't have deleted visit"
2567        );
2568
2569        let (info1, mut visits1) = fetch_visits(&conn, &urls[1], 100).unwrap().unwrap();
2570        visits1.sort_by_key(|v| v.visit_date);
2571        // Shouldn't have most recent visit, but should still have the dates[1]
2572        // visit, which should be uneffected.
2573        assert_eq!(
2574            visits1,
2575            &[
2576                FetchedVisit {
2577                    is_local: true,
2578                    visit_date: dates[0],
2579                    visit_type: Some(VisitType::Link)
2580                },
2581                FetchedVisit {
2582                    is_local: true,
2583                    visit_date: dates[1],
2584                    visit_type: Some(VisitType::Link)
2585                },
2586            ]
2587        );
2588
2589        // Make sure syncing doesn't resurrect them.
2590        apply_synced_visits(
2591            &conn,
2592            &info0.guid,
2593            &info0.url,
2594            &Some(info0.title.clone()),
2595            // Ignore dates[0] since we know it's present.
2596            &dates
2597                .iter()
2598                .map(|&d| HistoryRecordVisit {
2599                    date: d.into(),
2600                    transition: VisitType::Link as u8,
2601                    unknown_fields: UnknownFields::new(),
2602                })
2603                .collect::<Vec<_>>(),
2604            &UnknownFields::new(),
2605        )
2606        .unwrap();
2607
2608        let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2609        assert_eq!(
2610            visits0,
2611            &[FetchedVisit {
2612                is_local: true,
2613                visit_date: dates[0],
2614                visit_type: Some(VisitType::Link)
2615            }]
2616        );
2617
2618        assert_tombstones(
2619            &conn,
2620            &[
2621                (info0.row_id, dates[1]),
2622                (info0.row_id, dates[2]),
2623                (info1.row_id, dates[2]),
2624            ],
2625        );
2626
2627        // Delete the last visit from info0. This should delete the page entirely,
2628        // as well as it's tomebstones.
2629        delete_place_visit_at_time(&conn, &urls[0], dates[0]).unwrap();
2630
2631        assert!(fetch_visits(&conn, &urls[0], 100).unwrap().is_none());
2632
2633        assert_tombstones(&conn, &[(info1.row_id, dates[2])]);
2634    }
2635
2636    #[test]
2637    fn test_delete_local() {
2638        use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
2639        use crate::storage::bookmarks::{
2640            self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark, InsertableItem,
2641        };
2642        use url::Url;
2643        error_support::init_for_tests();
2644        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2645        let ts = Timestamp::now().0 - 5_000_000;
2646        // Add a number of visits across a handful of origins.
2647        for o in 0..10 {
2648            for i in 0..11 {
2649                for t in 0..3 {
2650                    get_custom_observed_page(
2651                        &mut conn,
2652                        &format!("http://www.example{}.com/{}", o, i),
2653                        |obs| obs.with_at(Timestamp(ts + t * 1000 + i * 10_000 + o * 100_000)),
2654                    )
2655                    .unwrap();
2656                }
2657            }
2658        }
2659        // Add some bookmarks.
2660        let b0 = (
2661            SyncGuid::from("aaaaaaaaaaaa"),
2662            Url::parse("http://www.example3.com/5").unwrap(),
2663        );
2664        let b1 = (
2665            SyncGuid::from("bbbbbbbbbbbb"),
2666            Url::parse("http://www.example6.com/10").unwrap(),
2667        );
2668        let b2 = (
2669            SyncGuid::from("cccccccccccc"),
2670            Url::parse("http://www.example9.com/4").unwrap(),
2671        );
2672        for (guid, url) in &[&b0, &b1, &b2] {
2673            bookmarks::insert_bookmark(
2674                &conn,
2675                InsertableItem::Bookmark {
2676                    b: InsertableBookmark {
2677                        parent_guid: BookmarkRootGuid::Unfiled.into(),
2678                        position: BookmarkPosition::Append,
2679                        date_added: None,
2680                        last_modified: None,
2681                        guid: Some(guid.clone()),
2682                        url: url.clone(),
2683                        title: None,
2684                    },
2685                },
2686            )
2687            .unwrap();
2688        }
2689
2690        // Make sure tombstone insertions stick.
2691        conn.execute_all(&[
2692            &format!(
2693                "UPDATE moz_places set sync_status = {}",
2694                (SyncStatus::Normal as u8)
2695            ),
2696            &format!(
2697                "UPDATE moz_bookmarks set syncStatus = {}",
2698                (SyncStatus::Normal as u8)
2699            ),
2700        ])
2701        .unwrap();
2702
2703        // Ensure some various tombstones exist
2704        delete_visits_for(
2705            &conn,
2706            &url_to_guid(&conn, &Url::parse("http://www.example8.com/5").unwrap())
2707                .unwrap()
2708                .unwrap(),
2709        )
2710        .unwrap();
2711
2712        delete_place_visit_at_time(
2713            &conn,
2714            &Url::parse("http://www.example10.com/5").unwrap(),
2715            Timestamp(ts + 5 * 10_000 + 10 * 100_000),
2716        )
2717        .unwrap();
2718
2719        assert!(bookmarks::delete_bookmark(&conn, &b0.0).unwrap());
2720
2721        delete_everything(&conn).unwrap();
2722
2723        let places = conn
2724            .query_rows_and_then(
2725                "SELECT * FROM moz_places ORDER BY url ASC",
2726                [],
2727                PageInfo::from_row,
2728            )
2729            .unwrap();
2730        assert_eq!(places.len(), 2);
2731        assert_eq!(places[0].url, b1.1);
2732        assert_eq!(places[1].url, b2.1);
2733        for p in &places {
2734            assert_eq!(
2735                p.frecency,
2736                DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
2737            );
2738            assert_eq!(p.visit_count_local, 0);
2739            assert_eq!(p.visit_count_remote, 0);
2740            assert_eq!(p.last_visit_date_local, Timestamp(0));
2741            assert_eq!(p.last_visit_date_remote, Timestamp(0));
2742        }
2743
2744        let counts_sql = [
2745            (0i64, "SELECT COUNT(*) FROM moz_historyvisits"),
2746            (2, "SELECT COUNT(*) FROM moz_origins"),
2747            (7, "SELECT COUNT(*) FROM moz_bookmarks"), // the two we added + 5 roots
2748            (1, "SELECT COUNT(*) FROM moz_bookmarks_deleted"),
2749            (0, "SELECT COUNT(*) FROM moz_historyvisit_tombstones"),
2750            (0, "SELECT COUNT(*) FROM moz_places_tombstones"),
2751        ];
2752        for (want, query) in &counts_sql {
2753            assert_eq!(
2754                *want,
2755                conn.query_one::<i64>(query).unwrap(),
2756                "Unexpected value for {}",
2757                query
2758            );
2759        }
2760    }
2761
2762    #[test]
2763    fn test_delete_everything() {
2764        use crate::storage::bookmarks::{
2765            self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2766        };
2767        use url::Url;
2768        error_support::init_for_tests();
2769        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2770        let start = Timestamp::now();
2771
2772        let urls = &[
2773            Url::parse("http://example.com/1").unwrap(),
2774            Url::parse("http://example.com/2").unwrap(),
2775            Url::parse("http://example.com/3").unwrap(),
2776        ];
2777
2778        let dates = &[
2779            Timestamp(start.0 - 10000),
2780            Timestamp(start.0 - 5000),
2781            Timestamp(start.0),
2782        ];
2783
2784        for url in urls {
2785            for &date in dates {
2786                get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2787            }
2788        }
2789
2790        bookmarks::insert_bookmark(
2791            &conn,
2792            InsertableBookmark {
2793                parent_guid: BookmarkRootGuid::Unfiled.into(),
2794                position: BookmarkPosition::Append,
2795                date_added: None,
2796                last_modified: None,
2797                guid: Some("bookmarkAAAA".into()),
2798                url: urls[2].clone(),
2799                title: Some("A".into()),
2800            }
2801            .into(),
2802        )
2803        .expect("Should insert bookmark with URL 3");
2804
2805        conn.execute(
2806            "WITH entries(url, input) AS (
2807               VALUES(:url1, 'hi'), (:url3, 'bye')
2808             )
2809             INSERT INTO moz_inputhistory(place_id, input, use_count)
2810             SELECT h.id, e.input, 1
2811             FROM entries e
2812             JOIN moz_places h ON h.url_hash = hash(e.url) AND
2813                                  h.url = e.url",
2814            &[(":url1", &urls[1].as_str()), (":url3", &urls[2].as_str())],
2815        )
2816        .expect("Should insert autocomplete history entries");
2817
2818        delete_everything(&conn).expect("Should delete everything except URL 3");
2819
2820        std::thread::sleep(std::time::Duration::from_millis(50));
2821
2822        // Should leave bookmarked URLs alone, and keep autocomplete history for
2823        // those URLs.
2824        let mut places_stmt = conn.prepare("SELECT url FROM moz_places").unwrap();
2825        let remaining_urls: Vec<String> = places_stmt
2826            .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2827            .expect("Should fetch remaining URLs")
2828            .map(std::result::Result::unwrap)
2829            .collect();
2830        assert_eq!(remaining_urls, &["http://example.com/3"]);
2831
2832        let mut input_stmt = conn.prepare("SELECT input FROM moz_inputhistory").unwrap();
2833        let remaining_inputs: Vec<String> = input_stmt
2834            .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2835            .expect("Should fetch remaining autocomplete history entries")
2836            .map(std::result::Result::unwrap)
2837            .collect();
2838        assert_eq!(remaining_inputs, &["bye"]);
2839
2840        bookmarks::delete_bookmark(&conn, &"bookmarkAAAA".into())
2841            .expect("Should delete bookmark with URL 3");
2842
2843        delete_everything(&conn).expect("Should delete all URLs");
2844
2845        assert_eq!(
2846            0,
2847            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2848                .unwrap(),
2849        );
2850
2851        apply_synced_visits(
2852            &conn,
2853            &SyncGuid::random(),
2854            &url::Url::parse("http://www.example.com/123").unwrap(),
2855            &None,
2856            &[
2857                HistoryRecordVisit {
2858                    // This should make it in
2859                    date: Timestamp::now().into(),
2860                    transition: VisitType::Link as u8,
2861                    unknown_fields: UnknownFields::new(),
2862                },
2863                HistoryRecordVisit {
2864                    // This should not.
2865                    date: start.into(),
2866                    transition: VisitType::Link as u8,
2867                    unknown_fields: UnknownFields::new(),
2868                },
2869            ],
2870            &UnknownFields::new(),
2871        )
2872        .unwrap();
2873        assert_eq!(
2874            1,
2875            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2876                .unwrap(),
2877        );
2878        // Only one visit should be applied.
2879        assert_eq!(
2880            1,
2881            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2882                .unwrap(),
2883        );
2884
2885        // Check that we don't insert a place if all visits are too old.
2886        apply_synced_visits(
2887            &conn,
2888            &SyncGuid::random(),
2889            &url::Url::parse("http://www.example.com/1234").unwrap(),
2890            &None,
2891            &[HistoryRecordVisit {
2892                date: start.into(),
2893                transition: VisitType::Link as u8,
2894                unknown_fields: UnknownFields::new(),
2895            }],
2896            &UnknownFields::new(),
2897        )
2898        .unwrap();
2899        // unchanged.
2900        assert_eq!(
2901            1,
2902            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2903                .unwrap(),
2904        );
2905        assert_eq!(
2906            1,
2907            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2908                .unwrap(),
2909        );
2910    }
2911
2912    // See https://github.com/mozilla-mobile/fenix/issues/8531#issuecomment-590498878.
2913    #[test]
2914    fn test_delete_everything_deletes_origins() {
2915        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2916
2917        let u = Url::parse("https://www.reddit.com/r/climbing").expect("Should parse URL");
2918        let ts = Timestamp::now().0 - 5_000_000;
2919        let obs = VisitObservation::new(u)
2920            .with_visit_type(VisitType::Link)
2921            .with_at(Timestamp(ts));
2922        apply_observation(&conn, obs).expect("Should apply observation");
2923
2924        delete_everything(&conn).expect("Should delete everything");
2925
2926        // We should clear all origins after deleting everything.
2927        let origin_count = conn
2928            .query_one::<i64>("SELECT COUNT(*) FROM moz_origins")
2929            .expect("Should fetch origin count");
2930        assert_eq!(0, origin_count);
2931    }
2932
2933    #[test]
2934    fn test_apply_observation_updates_origins() {
2935        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2936
2937        let obs_for_a = VisitObservation::new(
2938            Url::parse("https://example1.com/a").expect("Should parse URL A"),
2939        )
2940        .with_visit_type(VisitType::Link)
2941        .with_at(Timestamp(Timestamp::now().0 - 5_000_000));
2942        apply_observation(&conn, obs_for_a).expect("Should apply observation for A");
2943
2944        let obs_for_b = VisitObservation::new(
2945            Url::parse("https://example2.com/b").expect("Should parse URL B"),
2946        )
2947        .with_visit_type(VisitType::Link)
2948        .with_at(Timestamp(Timestamp::now().0 - 2_500_000));
2949        apply_observation(&conn, obs_for_b).expect("Should apply observation for B");
2950
2951        let mut origins = conn
2952            .prepare("SELECT host FROM moz_origins")
2953            .expect("Should prepare origins statement")
2954            .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2955            .expect("Should fetch all origins")
2956            .map(|r| r.expect("Should get origin from row"))
2957            .collect::<Vec<_>>();
2958        origins.sort();
2959        assert_eq!(origins, &["example1.com", "example2.com",]);
2960    }
2961
2962    #[test]
2963    fn test_preview_url() {
2964        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2965
2966        let url1 = Url::parse("https://www.example.com/").unwrap();
2967        // Can observe preview url without an associated visit.
2968        assert!(apply_observation(
2969            &conn,
2970            VisitObservation::new(url1.clone()).with_preview_image_url(Some(
2971                Url::parse("https://www.example.com/image.png").unwrap()
2972            ))
2973        )
2974        .unwrap()
2975        .is_none());
2976
2977        // We don't get a visit id back above, so just assume an id of the corresponding moz_places entry.
2978        let mut db_preview_url = conn
2979            .query_row_and_then_cachable(
2980                "SELECT preview_image_url FROM moz_places WHERE id = 1",
2981                [],
2982                |row| row.get(0),
2983                false,
2984            )
2985            .unwrap();
2986        assert_eq!(
2987            Some("https://www.example.com/image.png".to_string()),
2988            db_preview_url
2989        );
2990
2991        // Observing a visit afterwards doesn't erase a preview url.
2992        let visit_id = apply_observation(
2993            &conn,
2994            VisitObservation::new(url1).with_visit_type(VisitType::Link),
2995        )
2996        .unwrap();
2997        assert!(visit_id.is_some());
2998
2999        db_preview_url = conn
3000            .query_row_and_then_cachable(
3001                "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3002                &[(":id", &visit_id.unwrap() as &dyn ToSql)],
3003                |row| row.get(0),
3004                false,
3005            )
3006            .unwrap();
3007        assert_eq!(
3008            Some("https://www.example.com/image.png".to_string()),
3009            db_preview_url
3010        );
3011
3012        // Can observe a preview image url as part of a visit observation.
3013        let another_visit_id = apply_observation(
3014            &conn,
3015            VisitObservation::new(Url::parse("https://www.example.com/another/").unwrap())
3016                .with_preview_image_url(Some(
3017                    Url::parse("https://www.example.com/funky/image.png").unwrap(),
3018                ))
3019                .with_visit_type(VisitType::Link),
3020        )
3021        .unwrap();
3022        assert!(another_visit_id.is_some());
3023
3024        db_preview_url = conn
3025            .query_row_and_then_cachable(
3026                "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3027                &[(":id", &another_visit_id.unwrap())],
3028                |row| row.get(0),
3029                false,
3030            )
3031            .unwrap();
3032        assert_eq!(
3033            Some("https://www.example.com/funky/image.png".to_string()),
3034            db_preview_url
3035        );
3036    }
3037
3038    #[test]
3039    fn test_long_strings() {
3040        error_support::init_for_tests();
3041        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3042        let mut url = "http://www.example.com".to_string();
3043        while url.len() < crate::storage::URL_LENGTH_MAX {
3044            url += "/garbage";
3045        }
3046        let maybe_row = apply_observation(
3047            &conn,
3048            VisitObservation::new(Url::parse(&url).unwrap())
3049                .with_visit_type(VisitType::Link)
3050                .with_at(Timestamp::now()),
3051        )
3052        .unwrap();
3053        assert!(maybe_row.is_none(), "Shouldn't insert overlong URL");
3054
3055        let maybe_row_preview = apply_observation(
3056            &conn,
3057            VisitObservation::new(Url::parse("https://www.example.com/").unwrap())
3058                .with_visit_type(VisitType::Link)
3059                .with_preview_image_url(Url::parse(&url).unwrap()),
3060        )
3061        .unwrap();
3062        assert!(
3063            maybe_row_preview.is_some(),
3064            "Shouldn't avoid a visit observation due to an overly long preview url"
3065        );
3066
3067        let mut title = "example 1 2 3".to_string();
3068        // Make sure whatever we use here surpasses the length.
3069        while title.len() < crate::storage::TITLE_LENGTH_MAX + 10 {
3070            title += " test test";
3071        }
3072        let maybe_visit_row = apply_observation(
3073            &conn,
3074            VisitObservation::new(Url::parse("http://www.example.com/123").unwrap())
3075                .with_title(title.clone())
3076                .with_visit_type(VisitType::Link)
3077                .with_at(Timestamp::now()),
3078        )
3079        .unwrap();
3080
3081        assert!(maybe_visit_row.is_some());
3082        let db_title: String = conn
3083            .query_row_and_then_cachable(
3084                "SELECT h.title FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3085                &[(":id", &maybe_visit_row.unwrap())],
3086                |row| row.get(0),
3087                false,
3088            )
3089            .unwrap();
3090        // Ensure what we get back the trimmed title.
3091        assert_eq!(db_title.len(), crate::storage::TITLE_LENGTH_MAX);
3092        assert!(title.starts_with(&db_title));
3093    }
3094
3095    #[test]
3096    fn test_get_visit_page_with_bound() {
3097        use std::time::SystemTime;
3098        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3099        let now: Timestamp = SystemTime::now().into();
3100        let now_u64 = now.0;
3101        let now_i64 = now.0 as i64;
3102        // (url, title, when, is_remote, (expected_always, expected_only_local)
3103        let to_add = [
3104            (
3105                "https://www.example.com/0",
3106                "older 2",
3107                now_u64 - 200_200,
3108                false,
3109                (true, false),
3110            ),
3111            (
3112                "https://www.example.com/1",
3113                "older 1",
3114                now_u64 - 200_100,
3115                true,
3116                (true, false),
3117            ),
3118            (
3119                "https://www.example.com/2",
3120                "same time",
3121                now_u64 - 200_000,
3122                false,
3123                (true, false),
3124            ),
3125            (
3126                "https://www.example.com/3",
3127                "same time",
3128                now_u64 - 200_000,
3129                false,
3130                (true, false),
3131            ),
3132            (
3133                "https://www.example.com/4",
3134                "same time",
3135                now_u64 - 200_000,
3136                false,
3137                (true, false),
3138            ),
3139            (
3140                "https://www.example.com/5",
3141                "same time",
3142                now_u64 - 200_000,
3143                false,
3144                (true, false),
3145            ),
3146            (
3147                "https://www.example.com/6",
3148                "same time",
3149                now_u64 - 200_000,
3150                false,
3151                (true, false),
3152            ),
3153            (
3154                "https://www.example.com/7",
3155                "same time",
3156                now_u64 - 200_000,
3157                false,
3158                (true, false),
3159            ),
3160            (
3161                "https://www.example.com/8",
3162                "same time",
3163                now_u64 - 200_000,
3164                false,
3165                (true, false),
3166            ),
3167            (
3168                "https://www.example.com/9",
3169                "same time",
3170                now_u64 - 200_000,
3171                false,
3172                (true, false),
3173            ),
3174            (
3175                "https://www.example.com/10",
3176                "more recent 2",
3177                now_u64 - 199_000,
3178                false,
3179                (true, false),
3180            ),
3181            (
3182                "https://www.example.com/11",
3183                "more recent 1",
3184                now_u64 - 198_000,
3185                false,
3186                (true, false),
3187            ),
3188        ];
3189
3190        for &(url, title, when, remote, _) in &to_add {
3191            apply_observation(
3192                &conn,
3193                VisitObservation::new(Url::parse(url).unwrap())
3194                    .with_title(title.to_owned())
3195                    .with_at(Timestamp(when))
3196                    .with_is_remote(remote)
3197                    .with_visit_type(VisitType::Link),
3198            )
3199            .expect("Should apply visit");
3200        }
3201
3202        // test when offset fall on a point where visited_date changes
3203        let infos_with_bound =
3204            get_visit_page_with_bound(&conn, now_i64 - 200_000, 8, 2, VisitTransitionSet::empty())
3205                .unwrap();
3206        let infos = infos_with_bound.infos;
3207        assert_eq!(infos[0].title.as_ref().unwrap().as_str(), "older 1",);
3208        assert!(infos[0].is_remote); // "older 1" is remote
3209        assert_eq!(infos[1].title.as_ref().unwrap().as_str(), "older 2",);
3210        assert!(!infos[1].is_remote); // "older 2" is local
3211        assert_eq!(infos_with_bound.bound, now_i64 - 200_200,);
3212        assert_eq!(infos_with_bound.offset, 1,);
3213
3214        // test when offset fall on one item before visited_date changes
3215        let infos_with_bound =
3216            get_visit_page_with_bound(&conn, now_i64 - 200_000, 7, 1, VisitTransitionSet::empty())
3217                .unwrap();
3218        assert_eq!(
3219            infos_with_bound.infos[0].url,
3220            Url::parse("https://www.example.com/9").unwrap(),
3221        );
3222
3223        // test when offset fall on one item after visited_date changes
3224        let infos_with_bound =
3225            get_visit_page_with_bound(&conn, now_i64 - 200_000, 9, 1, VisitTransitionSet::empty())
3226                .unwrap();
3227        assert_eq!(
3228            infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3229            "older 2",
3230        );
3231
3232        // with a small page length, loop through items that have the same visited date
3233        let count = 2;
3234        let mut bound = now_i64 - 199_000;
3235        let mut offset = 1;
3236        for _i in 0..4 {
3237            let infos_with_bound =
3238                get_visit_page_with_bound(&conn, bound, offset, count, VisitTransitionSet::empty())
3239                    .unwrap();
3240            assert_eq!(
3241                infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3242                "same time",
3243            );
3244            assert_eq!(
3245                infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3246                "same time",
3247            );
3248            bound = infos_with_bound.bound;
3249            offset = infos_with_bound.offset;
3250        }
3251        // bound and offset should have skipped the 8 items that have the same visited date
3252        assert_eq!(bound, now_i64 - 200_000,);
3253        assert_eq!(offset, 8,);
3254
3255        // when bound is now and offset is zero
3256        let infos_with_bound =
3257            get_visit_page_with_bound(&conn, now_i64, 0, 2, VisitTransitionSet::empty()).unwrap();
3258        assert_eq!(
3259            infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3260            "more recent 1",
3261        );
3262        assert_eq!(
3263            infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3264            "more recent 2",
3265        );
3266        assert_eq!(infos_with_bound.bound, now_i64 - 199_000);
3267        assert_eq!(infos_with_bound.offset, 1);
3268    }
3269
3270    /// Test find_normal_visits_to_prune
3271    #[test]
3272    fn test_normal_visit_pruning() {
3273        use std::time::{Duration, SystemTime};
3274        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3275        let one_day = Duration::from_secs(60 * 60 * 24);
3276        let now: Timestamp = SystemTime::now().into();
3277        let url = Url::parse("https://mozilla.com/").unwrap();
3278
3279        // Create 1 visit per day for the last 30 days
3280        let mut visits: Vec<_> = (0..30)
3281            .map(|i| {
3282                apply_observation(
3283                    &conn,
3284                    VisitObservation::new(url.clone())
3285                        .with_at(now.checked_sub(one_day * i))
3286                        .with_visit_type(VisitType::Link),
3287                )
3288                .unwrap()
3289                .unwrap()
3290            })
3291            .collect();
3292        // Reverse visits so that they're oldest first
3293        visits.reverse();
3294
3295        check_visits_to_prune(
3296            &conn,
3297            find_normal_visits_to_prune(&conn, 4, now).unwrap(),
3298            &visits[..4],
3299        );
3300
3301        // Only visits older than 7 days should be pruned
3302        check_visits_to_prune(
3303            &conn,
3304            find_normal_visits_to_prune(&conn, 30, now).unwrap(),
3305            &visits[..22],
3306        );
3307    }
3308
3309    /// Test find_exotic_visits_to_prune
3310    #[test]
3311    fn test_exotic_visit_pruning() {
3312        use std::time::{Duration, SystemTime};
3313        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3314        let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3315        let now: Timestamp = SystemTime::now().into();
3316        let short_url = Url::parse("https://mozilla.com/").unwrap();
3317        let long_url = Url::parse(&format!(
3318            "https://mozilla.com/{}",
3319            (0..255).map(|_| "x").collect::<String>()
3320        ))
3321        .unwrap();
3322
3323        let visit_with_long_url = apply_observation(
3324            &conn,
3325            VisitObservation::new(long_url.clone())
3326                .with_at(now.checked_sub(one_month * 2))
3327                .with_visit_type(VisitType::Link),
3328        )
3329        .unwrap()
3330        .unwrap();
3331
3332        let visit_for_download = apply_observation(
3333            &conn,
3334            VisitObservation::new(short_url)
3335                .with_at(now.checked_sub(one_month * 3))
3336                .with_visit_type(VisitType::Download),
3337        )
3338        .unwrap()
3339        .unwrap();
3340
3341        // This visit should not be pruned, since it's too recent
3342        apply_observation(
3343            &conn,
3344            VisitObservation::new(long_url)
3345                .with_at(now.checked_sub(one_month))
3346                .with_visit_type(VisitType::Download),
3347        )
3348        .unwrap()
3349        .unwrap();
3350
3351        check_visits_to_prune(
3352            &conn,
3353            find_exotic_visits_to_prune(&conn, 2, now).unwrap(),
3354            &[visit_for_download, visit_with_long_url],
3355        );
3356
3357        // With limit = 1, it should pick the oldest visit
3358        check_visits_to_prune(
3359            &conn,
3360            find_exotic_visits_to_prune(&conn, 1, now).unwrap(),
3361            &[visit_for_download],
3362        );
3363
3364        // If the limit exceeds the number of candidates, it should return as many as it can find
3365        check_visits_to_prune(
3366            &conn,
3367            find_exotic_visits_to_prune(&conn, 3, now).unwrap(),
3368            &[visit_for_download, visit_with_long_url],
3369        );
3370    }
3371    /// Test that find_visits_to_prune correctly combines find_exotic_visits_to_prune and
3372    /// find_normal_visits_to_prune
3373    #[test]
3374    fn test_visit_pruning() {
3375        use std::time::{Duration, SystemTime};
3376        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3377        let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3378        let now: Timestamp = SystemTime::now().into();
3379        let short_url = Url::parse("https://mozilla.com/").unwrap();
3380        let long_url = Url::parse(&format!(
3381            "https://mozilla.com/{}",
3382            (0..255).map(|_| "x").collect::<String>()
3383        ))
3384        .unwrap();
3385
3386        // An exotic visit that should be pruned first, even if it's not the oldest
3387        let excotic_visit = apply_observation(
3388            &conn,
3389            VisitObservation::new(long_url)
3390                .with_at(now.checked_sub(one_month * 3))
3391                .with_visit_type(VisitType::Link),
3392        )
3393        .unwrap()
3394        .unwrap();
3395
3396        // Normal visits that should be pruned after excotic visits
3397        let old_visit = apply_observation(
3398            &conn,
3399            VisitObservation::new(short_url.clone())
3400                .with_at(now.checked_sub(one_month * 4))
3401                .with_visit_type(VisitType::Link),
3402        )
3403        .unwrap()
3404        .unwrap();
3405        let really_old_visit = apply_observation(
3406            &conn,
3407            VisitObservation::new(short_url.clone())
3408                .with_at(now.checked_sub(one_month * 12))
3409                .with_visit_type(VisitType::Link),
3410        )
3411        .unwrap()
3412        .unwrap();
3413
3414        // Newer visit that's too new to be pruned
3415        apply_observation(
3416            &conn,
3417            VisitObservation::new(short_url)
3418                .with_at(now.checked_sub(Duration::from_secs(100)))
3419                .with_visit_type(VisitType::Link),
3420        )
3421        .unwrap()
3422        .unwrap();
3423
3424        check_visits_to_prune(
3425            &conn,
3426            find_visits_to_prune(&conn, 2, now).unwrap(),
3427            &[excotic_visit, really_old_visit],
3428        );
3429
3430        check_visits_to_prune(
3431            &conn,
3432            find_visits_to_prune(&conn, 10, now).unwrap(),
3433            &[excotic_visit, really_old_visit, old_visit],
3434        );
3435    }
3436
3437    fn check_visits_to_prune(
3438        db: &PlacesDb,
3439        visits_to_delete: Vec<VisitToDelete>,
3440        correct_visits: &[RowId],
3441    ) {
3442        assert_eq!(
3443            correct_visits.iter().collect::<HashSet<_>>(),
3444            visits_to_delete
3445                .iter()
3446                .map(|v| &v.visit_id)
3447                .collect::<HashSet<_>>()
3448        );
3449
3450        let correct_place_ids: HashSet<RowId> = correct_visits
3451            .iter()
3452            .map(|vid| {
3453                db.query_one(&format!(
3454                    "SELECT v.place_id FROM moz_historyvisits v WHERE v.id = {}",
3455                    vid
3456                ))
3457                .unwrap()
3458            })
3459            .collect();
3460        assert_eq!(
3461            correct_place_ids,
3462            visits_to_delete
3463                .iter()
3464                .map(|v| v.page_id)
3465                .collect::<HashSet<_>>()
3466        );
3467    }
3468
3469    #[test]
3470    fn test_get_visit_count_for_host() {
3471        error_support::init_for_tests();
3472        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3473        let start_timestamp = Timestamp::now();
3474        let to_add = [
3475            (
3476                "http://example.com/0",
3477                start_timestamp.0 - 200_200,
3478                VisitType::Link,
3479            ),
3480            (
3481                "http://example.com/1",
3482                start_timestamp.0 - 200_100,
3483                VisitType::Link,
3484            ),
3485            (
3486                "https://example.com/0",
3487                start_timestamp.0 - 200_000,
3488                VisitType::Link,
3489            ),
3490            (
3491                "https://example1.com/0",
3492                start_timestamp.0 - 100_600,
3493                VisitType::Link,
3494            ),
3495            (
3496                "https://example1.com/0",
3497                start_timestamp.0 - 100_500,
3498                VisitType::Reload,
3499            ),
3500            (
3501                "https://example1.com/1",
3502                start_timestamp.0 - 100_400,
3503                VisitType::Link,
3504            ),
3505            (
3506                "https://example.com/2",
3507                start_timestamp.0 - 100_300,
3508                VisitType::Link,
3509            ),
3510            (
3511                "https://example.com/1",
3512                start_timestamp.0 - 100_200,
3513                VisitType::Link,
3514            ),
3515            (
3516                "https://example.com/0",
3517                start_timestamp.0 - 100_100,
3518                VisitType::Link,
3519            ),
3520        ];
3521
3522        for &(url, when, visit_type) in &to_add {
3523            apply_observation(
3524                &conn,
3525                VisitObservation::new(Url::parse(url).unwrap())
3526                    .with_at(Timestamp(when))
3527                    .with_visit_type(visit_type),
3528            )
3529            .unwrap()
3530            .unwrap();
3531        }
3532
3533        assert_eq!(
3534            get_visit_count_for_host(
3535                &conn,
3536                "example.com",
3537                Timestamp(start_timestamp.0 - 100_000),
3538                VisitTransitionSet::for_specific(&[])
3539            )
3540            .unwrap(),
3541            6
3542        );
3543        assert_eq!(
3544            get_visit_count_for_host(
3545                &conn,
3546                "example1.com",
3547                Timestamp(start_timestamp.0 - 100_000),
3548                VisitTransitionSet::for_specific(&[])
3549            )
3550            .unwrap(),
3551            3
3552        );
3553        assert_eq!(
3554            get_visit_count_for_host(
3555                &conn,
3556                "example.com",
3557                Timestamp(start_timestamp.0 - 200_000),
3558                VisitTransitionSet::for_specific(&[])
3559            )
3560            .unwrap(),
3561            2
3562        );
3563        assert_eq!(
3564            get_visit_count_for_host(
3565                &conn,
3566                "example1.com",
3567                Timestamp(start_timestamp.0 - 100_500),
3568                VisitTransitionSet::for_specific(&[])
3569            )
3570            .unwrap(),
3571            1
3572        );
3573        assert_eq!(
3574            get_visit_count_for_host(
3575                &conn,
3576                "example1.com",
3577                Timestamp(start_timestamp.0 - 100_000),
3578                VisitTransitionSet::for_specific(&[VisitType::Reload])
3579            )
3580            .unwrap(),
3581            2
3582        );
3583    }
3584}