places/storage/
history.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5mod actions;
6
7use super::{fetch_page_info, new_page_info, PageInfo, RowId};
8use crate::db::PlacesDb;
9use crate::error::{debug, trace, warn, Result};
10use crate::ffi::{HistoryVisitInfo, HistoryVisitInfosWithBound, TopFrecentSiteInfo};
11use crate::frecency;
12use crate::hash;
13use crate::history_sync::engine::{
14    COLLECTION_SYNCID_META_KEY, GLOBAL_SYNCID_META_KEY, LAST_SYNC_META_KEY,
15};
16use crate::observation::VisitObservation;
17use crate::storage::{
18    delete_meta, delete_pending_temp_tables, get_meta, history_metadata, put_meta,
19};
20use crate::types::{
21    serialize_unknown_fields, SyncStatus, UnknownFields, VisitTransitionSet, VisitType,
22};
23use actions::*;
24use rusqlite::types::ToSql;
25use rusqlite::Result as RusqliteResult;
26use rusqlite::Row;
27use sql_support::{self, ConnExt};
28use std::collections::HashSet;
29use std::time::Duration;
30use sync15::bso::OutgoingBso;
31use sync15::engine::EngineSyncAssociation;
32use sync_guid::Guid as SyncGuid;
33use types::Timestamp;
34use url::Url;
35
36/// When `delete_everything` is called (to perform a permanent local deletion), in
37/// addition to performing the deletion as requested, we make a note of the time
38/// when it occurred, and refuse to sync incoming visits from before this time.
39///
40/// This allows us to avoid these visits trickling back in as other devices
41/// add visits to them remotely.
42static DELETION_HIGH_WATER_MARK_META_KEY: &str = "history_deleted_hwm";
43
44/// Returns the RowId of a new visit in moz_historyvisits, or None if no new visit was added.
45pub fn apply_observation(db: &PlacesDb, visit_ob: VisitObservation) -> Result<Option<RowId>> {
46    let tx = db.begin_transaction()?;
47    let result = apply_observation_direct(db, visit_ob)?;
48    delete_pending_temp_tables(db)?;
49    tx.commit()?;
50    Ok(result)
51}
52
53/// Returns the RowId of a new visit in moz_historyvisits, or None if no new visit was added.
54pub fn apply_observation_direct(
55    db: &PlacesDb,
56    visit_ob: VisitObservation,
57) -> Result<Option<RowId>> {
58    // Don't insert urls larger than our length max.
59    if visit_ob.url.as_str().len() > super::URL_LENGTH_MAX {
60        return Ok(None);
61    }
62    // Make sure we have a valid preview URL - it should parse, and not exceed max size.
63    // In case the URL is too long, ignore it and proceed with the rest of the observation.
64    // In case the URL is entirely invalid, let the caller know by failing.
65    let preview_image_url = if let Some(ref piu) = visit_ob.preview_image_url {
66        if piu.as_str().len() > super::URL_LENGTH_MAX {
67            None
68        } else {
69            Some(piu.clone())
70        }
71    } else {
72        None
73    };
74    let mut page_info = match fetch_page_info(db, &visit_ob.url)? {
75        Some(info) => info.page,
76        None => new_page_info(db, &visit_ob.url, None)?,
77    };
78    let mut update_change_counter = false;
79    let mut update_frec = false;
80    let mut updates: Vec<(&str, &str, &dyn ToSql)> = Vec::new();
81
82    if let Some(ref title) = visit_ob.title {
83        page_info.title = crate::util::slice_up_to(title, super::TITLE_LENGTH_MAX).into();
84        updates.push(("title", ":title", &page_info.title));
85        update_change_counter = true;
86    }
87    let preview_image_url_str;
88    if let Some(ref preview_image_url) = preview_image_url {
89        preview_image_url_str = preview_image_url.as_str();
90        updates.push((
91            "preview_image_url",
92            ":preview_image_url",
93            &preview_image_url_str,
94        ));
95    }
96    // There's a new visit, so update everything that implies. To help with
97    // testing we return the rowid of the visit we added.
98    let visit_row_id = match visit_ob.visit_type {
99        Some(visit_type) => {
100            // A single non-hidden visit makes the place non-hidden.
101            if !visit_ob.get_is_hidden() {
102                updates.push(("hidden", ":hidden", &false));
103            }
104            if visit_type == VisitType::Typed {
105                page_info.typed += 1;
106                updates.push(("typed", ":typed", &page_info.typed));
107            }
108
109            let at = visit_ob.at.unwrap_or_else(Timestamp::now);
110            let is_remote = visit_ob.is_remote.unwrap_or(false);
111            let row_id = add_visit(db, page_info.row_id, None, at, visit_type, !is_remote, None)?;
112            // a new visit implies new frecency except in error cases.
113            if !visit_ob.is_error.unwrap_or(false) {
114                update_frec = true;
115            }
116            update_change_counter = true;
117            Some(row_id)
118        }
119        None => None,
120    };
121
122    if update_change_counter {
123        page_info.sync_change_counter += 1;
124        updates.push((
125            "sync_change_counter",
126            ":sync_change_counter",
127            &page_info.sync_change_counter,
128        ));
129    }
130
131    if !updates.is_empty() {
132        let mut params: Vec<(&str, &dyn ToSql)> = Vec::with_capacity(updates.len() + 1);
133        let mut sets: Vec<String> = Vec::with_capacity(updates.len());
134        for (col, name, val) in updates {
135            sets.push(format!("{} = {}", col, name));
136            params.push((name, val))
137        }
138        params.push((":row_id", &page_info.row_id.0));
139        let sql = format!(
140            "UPDATE moz_places
141                          SET {}
142                          WHERE id == :row_id",
143            sets.join(",")
144        );
145        db.execute(&sql, &params[..])?;
146    }
147    // This needs to happen after the other updates.
148    if update_frec {
149        update_frecency(
150            db,
151            page_info.row_id,
152            Some(visit_ob.get_redirect_frecency_boost()),
153        )?;
154    }
155    Ok(visit_row_id)
156}
157
158pub fn update_frecency(db: &PlacesDb, id: RowId, redirect_boost: Option<bool>) -> Result<()> {
159    let score = frecency::calculate_frecency(
160        db.conn(),
161        &frecency::DEFAULT_FRECENCY_SETTINGS,
162        id.0, // TODO: calculate_frecency should take a RowId here.
163        redirect_boost,
164    )?;
165
166    db.execute(
167        "
168        UPDATE moz_places
169            SET frecency = :frecency
170        WHERE id = :page_id",
171        &[
172            (":frecency", &score as &dyn rusqlite::ToSql),
173            (":page_id", &id.0),
174        ],
175    )?;
176
177    Ok(())
178}
179
180/// Indicates if and when a URL's frecency was marked as stale.
181pub fn frecency_stale_at(db: &PlacesDb, url: &Url) -> Result<Option<Timestamp>> {
182    let result = db.try_query_row(
183        "SELECT stale_at FROM moz_places_stale_frecencies s
184         JOIN moz_places h ON h.id = s.place_id
185         WHERE h.url_hash = hash(:url) AND
186               h.url = :url",
187        &[(":url", &url.as_str())],
188        |row| -> rusqlite::Result<_> { row.get::<_, Timestamp>(0) },
189        true,
190    )?;
191    Ok(result)
192}
193
194// Add a single visit - you must know the page rowid. Does not update the
195// page info - if you are calling this, you will also need to update the
196// parent page with an updated change counter etc.
197fn add_visit(
198    db: &PlacesDb,
199    page_id: RowId,
200    from_visit: Option<RowId>,
201    visit_date: Timestamp,
202    visit_type: VisitType,
203    is_local: bool,
204    unknown_fields: Option<String>,
205) -> Result<RowId> {
206    let sql = "INSERT INTO moz_historyvisits
207            (from_visit, place_id, visit_date, visit_type, is_local, unknown_fields)
208        VALUES (:from_visit, :page_id, :visit_date, :visit_type, :is_local, :unknown_fields)";
209    db.execute_cached(
210        sql,
211        &[
212            (":from_visit", &from_visit as &dyn rusqlite::ToSql),
213            (":page_id", &page_id),
214            (":visit_date", &visit_date),
215            (":visit_type", &visit_type),
216            (":is_local", &is_local),
217            (":unknown_fields", &unknown_fields),
218        ],
219    )?;
220    let rid = db.conn().last_insert_rowid();
221    // Delete any tombstone that exists.
222    db.execute_cached(
223        "DELETE FROM moz_historyvisit_tombstones
224         WHERE place_id = :place_id
225           AND visit_date = :visit_date",
226        &[
227            (":place_id", &page_id as &dyn rusqlite::ToSql),
228            (":visit_date", &visit_date),
229        ],
230    )?;
231    Ok(RowId(rid))
232}
233
234/// Returns the GUID for the specified Url, or None if it doesn't exist.
235pub fn url_to_guid(db: &PlacesDb, url: &Url) -> Result<Option<SyncGuid>> {
236    href_to_guid(db, url.clone().as_str())
237}
238
239/// Returns the GUID for the specified Url String, or None if it doesn't exist.
240pub fn href_to_guid(db: &PlacesDb, url: &str) -> Result<Option<SyncGuid>> {
241    let sql = "SELECT guid FROM moz_places WHERE url_hash = hash(:url) AND url = :url";
242    let result: Option<SyncGuid> = db.try_query_row(
243        sql,
244        &[(":url", &url.to_owned())],
245        // subtle: we explicitly need to specify rusqlite::Result or the compiler
246        // struggles to work out what error type to return from try_query_row.
247        |row| -> rusqlite::Result<_> { row.get::<_, SyncGuid>(0) },
248        true,
249    )?;
250    Ok(result)
251}
252
253/// Internal function for deleting a page, creating a tombstone if necessary.
254/// Assumes a transaction is already set up by the caller.
255fn delete_visits_for_in_tx(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
256    // We only create tombstones for history which exists and with sync_status
257    // == SyncStatus::Normal
258    let to_clean = db.conn().try_query_row(
259        "SELECT id,
260                (foreign_count != 0) AS has_foreign,
261                1 as has_visits,
262                sync_status
263        FROM moz_places
264        WHERE guid = :guid",
265        &[(":guid", guid)],
266        PageToClean::from_row,
267        true,
268    )?;
269    // Note that history metadata has an `ON DELETE CASCADE` for the place ID - so if we
270    // call `delete_page` here, we assume history metadata dies too. Otherwise we
271    // explicitly delete the metadata after we delete the visits themselves.
272    match to_clean {
273        Some(PageToClean {
274            id,
275            has_foreign: true,
276            sync_status: SyncStatus::Normal,
277            ..
278        }) => {
279            // If our page is syncing, and has foreign key references (like
280            // local or synced bookmarks, keywords, and tags), we can't delete
281            // its row from `moz_places` directly; that would cause a constraint
282            // violation. Instead, we must insert tombstones for all visits, and
283            // then delete just the visits, keeping the page in place (pun most
284            // definitely intended).
285            insert_tombstones_for_all_page_visits(db, id)?;
286            delete_all_visits_for_page(db, id)?;
287            history_metadata::delete_all_metadata_for_page(db, id)?;
288        }
289        Some(PageToClean {
290            id,
291            has_foreign: false,
292            sync_status: SyncStatus::Normal,
293            ..
294        }) => {
295            // However, if our page is syncing and _doesn't_ have any foreign
296            // key references, we can delete it from `moz_places` outright, and
297            // write a tombstone for the page instead of all the visits.
298            insert_tombstone_for_page(db, guid)?;
299            delete_page(db, id)?;
300        }
301        Some(PageToClean {
302            id,
303            has_foreign: true,
304            ..
305        }) => {
306            // If our page has foreign key references but _isn't_ syncing,
307            // we still can't delete it; we must delete its visits. But we
308            // don't need to write any tombstones for those deleted visits.
309            delete_all_visits_for_page(db, id)?;
310            // and we need to delete all history metadata.
311            history_metadata::delete_all_metadata_for_page(db, id)?;
312        }
313        Some(PageToClean {
314            id,
315            has_foreign: false,
316            ..
317        }) => {
318            // And, finally, the easiest case: not syncing, and no foreign
319            // key references, so just delete the page.
320            delete_page(db, id)?;
321        }
322        None => {}
323    }
324    delete_pending_temp_tables(db)?;
325    Ok(())
326}
327
328/// Inserts Sync tombstones for all of a page's visits.
329fn insert_tombstones_for_all_page_visits(db: &PlacesDb, page_id: RowId) -> Result<()> {
330    db.execute_cached(
331        "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date)
332         SELECT place_id, visit_date
333         FROM moz_historyvisits
334         WHERE place_id = :page_id",
335        &[(":page_id", &page_id)],
336    )?;
337    Ok(())
338}
339
340/// Removes all visits from a page. DOES NOT remove history_metadata - use
341/// `history_metadata::delete_all_metadata_for_page` for that.
342fn delete_all_visits_for_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
343    db.execute_cached(
344        "DELETE FROM moz_historyvisits
345         WHERE place_id = :page_id",
346        &[(":page_id", &page_id)],
347    )?;
348    Ok(())
349}
350
351/// Inserts a Sync tombstone for a page.
352fn insert_tombstone_for_page(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
353    db.execute_cached(
354        "INSERT OR IGNORE INTO moz_places_tombstones (guid)
355         VALUES(:guid)",
356        &[(":guid", guid)],
357    )?;
358    Ok(())
359}
360
361/// Deletes a page. Note that this throws a constraint violation if the page is
362/// bookmarked, or has a keyword or tags.
363fn delete_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
364    db.execute_cached(
365        "DELETE FROM moz_places
366         WHERE id = :page_id",
367        &[(":page_id", &page_id)],
368    )?;
369    Ok(())
370}
371
372/// Deletes all visits for a page given its GUID, creating tombstones if
373/// necessary.
374pub fn delete_visits_for(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
375    let tx = db.begin_transaction()?;
376    let result = delete_visits_for_in_tx(db, guid);
377    tx.commit()?;
378    result
379}
380
381/// Delete all visits in a date range.
382pub fn delete_visits_between(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
383    let tx = db.begin_transaction()?;
384    delete_visits_between_in_tx(db, start, end)?;
385    tx.commit()?;
386    Ok(())
387}
388
389pub fn delete_place_visit_at_time(db: &PlacesDb, place: &Url, visit: Timestamp) -> Result<()> {
390    delete_place_visit_at_time_by_href(db, place.as_str(), visit)
391}
392
393pub fn delete_place_visit_at_time_by_href(
394    db: &PlacesDb,
395    place: &str,
396    visit: Timestamp,
397) -> Result<()> {
398    let tx = db.begin_transaction()?;
399    delete_place_visit_at_time_in_tx(db, place, visit)?;
400    tx.commit()?;
401    Ok(())
402}
403
404pub fn prune_older_visits(db: &PlacesDb, limit: u32) -> Result<()> {
405    let tx = db.begin_transaction()?;
406
407    let result = DbAction::apply_all(
408        db,
409        db_actions_from_visits_to_delete(find_visits_to_prune(
410            db,
411            limit as usize,
412            Timestamp::now(),
413        )?),
414    );
415    tx.commit()?;
416    result
417}
418
419fn find_visits_to_prune(db: &PlacesDb, limit: usize, now: Timestamp) -> Result<Vec<VisitToDelete>> {
420    // Start with the exotic visits
421    let mut to_delete: HashSet<_> = find_exotic_visits_to_prune(db, limit, now)?
422        .into_iter()
423        .collect();
424    // If we still have more visits to prune, then add them from find_normal_visits_to_prune,
425    // leveraging the HashSet to ensure we don't add a duplicate item.
426    if to_delete.len() < limit {
427        for delete_visit in find_normal_visits_to_prune(db, limit, now)? {
428            to_delete.insert(delete_visit);
429            if to_delete.len() >= limit {
430                break;
431            }
432        }
433    }
434    Ok(Vec::from_iter(to_delete))
435}
436
437fn find_normal_visits_to_prune(
438    db: &PlacesDb,
439    limit: usize,
440    now: Timestamp,
441) -> Result<Vec<VisitToDelete>> {
442    // 7 days ago
443    let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 7));
444    db.query_rows_and_then(
445        "
446        SELECT v.id, v.place_id
447        FROM moz_places p
448        JOIN moz_historyvisits v ON v.place_id = p.id
449        WHERE v.visit_date < :visit_date_cuttoff
450        ORDER BY v.visit_date
451        LIMIT :limit
452        ",
453        rusqlite::named_params! {
454            ":visit_date_cuttoff": visit_date_cutoff,
455            ":limit": limit,
456        },
457        VisitToDelete::from_row,
458    )
459}
460
461/// Find "exotic" visits to prune.  These are visits visits that should be pruned first because
462/// they are less useful to the user because:
463///   - They're very old
464///   - They're not useful in the awesome bar because they're either a long URL or a download
465///
466/// This is based on the desktop pruning logic:
467/// https://searchfox.org/mozilla-central/search?q=QUERY_FIND_EXOTIC_VISITS_TO_EXPIRE
468fn find_exotic_visits_to_prune(
469    db: &PlacesDb,
470    limit: usize,
471    now: Timestamp,
472) -> Result<Vec<VisitToDelete>> {
473    // 60 days ago
474    let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 60));
475    db.query_rows_and_then(
476        "
477        SELECT v.id, v.place_id
478        FROM moz_places p
479        JOIN moz_historyvisits v ON v.place_id = p.id
480        WHERE v.visit_date < :visit_date_cuttoff
481        AND (LENGTH(p.url) > 255 OR v.visit_type = :download_visit_type)
482        ORDER BY v.visit_date
483        LIMIT :limit
484        ",
485        rusqlite::named_params! {
486            ":visit_date_cuttoff": visit_date_cutoff,
487            ":download_visit_type": VisitType::Download,
488            ":limit": limit,
489        },
490        VisitToDelete::from_row,
491    )
492}
493
494fn wipe_local_in_tx(db: &PlacesDb) -> Result<()> {
495    use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
496    db.execute_all(&[
497        "DELETE FROM moz_places WHERE foreign_count == 0",
498        "DELETE FROM moz_places_metadata",
499        "DELETE FROM moz_places_metadata_search_queries",
500        "DELETE FROM moz_historyvisits",
501        "DELETE FROM moz_places_tombstones",
502        "DELETE FROM moz_inputhistory AS i WHERE NOT EXISTS(
503             SELECT 1 FROM moz_places h
504             WHERE h.id = i.place_id)",
505        "DELETE FROM moz_historyvisit_tombstones",
506        "DELETE FROM moz_origins
507         WHERE id NOT IN (SELECT origin_id FROM moz_places)",
508        &format!(
509            r#"UPDATE moz_places SET
510                frecency = (CASE WHEN url_hash BETWEEN hash("place", "prefix_lo") AND
511                                                       hash("place", "prefix_hi")
512                                 THEN 0
513                                 ELSE {unvisited_bookmark_frec}
514                            END),
515                sync_change_counter = 0"#,
516            unvisited_bookmark_frec = DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
517        ),
518    ])?;
519
520    let need_frecency_update =
521        db.query_rows_and_then("SELECT id FROM moz_places", [], |r| r.get::<_, RowId>(0))?;
522    // Update the frecency for any remaining items, which basically means just
523    // for the bookmarks.
524    for row_id in need_frecency_update {
525        update_frecency(db, row_id, None)?;
526    }
527    delete_pending_temp_tables(db)?;
528    Ok(())
529}
530
531pub fn delete_everything(db: &PlacesDb) -> Result<()> {
532    let tx = db.begin_transaction()?;
533
534    // Remote visits could have a higher date than `now` if our clock is weird.
535    let most_recent_known_visit_time = db
536        .try_query_one::<Timestamp, _>("SELECT MAX(visit_date) FROM moz_historyvisits", [], false)?
537        .unwrap_or_default();
538
539    // Check the old value (if any) for the same reason
540    let previous_mark =
541        get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
542
543    let new_mark = Timestamp::now()
544        .max(previous_mark)
545        .max(most_recent_known_visit_time);
546
547    put_meta(db, DELETION_HIGH_WATER_MARK_META_KEY, &new_mark)?;
548
549    wipe_local_in_tx(db)?;
550
551    // Remove Sync metadata, too.
552    reset_in_tx(db, &EngineSyncAssociation::Disconnected)?;
553
554    tx.commit()?;
555
556    // Note: SQLite cannot VACUUM within a transaction.
557    db.execute_batch("VACUUM")?;
558    Ok(())
559}
560
561fn delete_place_visit_at_time_in_tx(db: &PlacesDb, url: &str, visit_date: Timestamp) -> Result<()> {
562    DbAction::apply_all(
563        db,
564        db_actions_from_visits_to_delete(db.query_rows_and_then(
565            "SELECT v.id, v.place_id
566                 FROM moz_places h
567                 JOIN moz_historyvisits v
568                   ON v.place_id = h.id
569                 WHERE v.visit_date = :visit_date
570                   AND h.url_hash = hash(:url)
571                   AND h.url = :url",
572            &[
573                (":url", &url as &dyn rusqlite::ToSql),
574                (":visit_date", &visit_date),
575            ],
576            VisitToDelete::from_row,
577        )?),
578    )
579}
580
581pub fn delete_visits_between_in_tx(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
582    // Like desktop's removeVisitsByFilter, we query the visit and place ids
583    // affected, then delete all visits, then delete all place ids in the set
584    // which are orphans after the delete.
585    let sql = "
586        SELECT id, place_id, visit_date
587        FROM moz_historyvisits
588        WHERE visit_date
589            BETWEEN :start AND :end
590    ";
591    let visits = db.query_rows_and_then(
592        sql,
593        &[(":start", &start), (":end", &end)],
594        |row| -> rusqlite::Result<_> {
595            Ok((
596                row.get::<_, RowId>(0)?,
597                row.get::<_, RowId>(1)?,
598                row.get::<_, Timestamp>(2)?,
599            ))
600        },
601    )?;
602
603    sql_support::each_chunk_mapped(
604        &visits,
605        |(visit_id, _, _)| visit_id,
606        |chunk, _| -> Result<()> {
607            db.conn().execute(
608                &format!(
609                    "DELETE from moz_historyvisits WHERE id IN ({})",
610                    sql_support::repeat_sql_vars(chunk.len()),
611                ),
612                rusqlite::params_from_iter(chunk),
613            )?;
614            Ok(())
615        },
616    )?;
617
618    // Insert tombstones for the deleted visits.
619    if !visits.is_empty() {
620        let sql = format!(
621            "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date) VALUES {}",
622            sql_support::repeat_display(visits.len(), ",", |i, f| {
623                let (_, place_id, visit_date) = visits[i];
624                write!(f, "({},{})", place_id.0, visit_date.0)
625            })
626        );
627        db.conn().execute(&sql, [])?;
628    }
629
630    // Find out which pages have been possibly orphaned and clean them up.
631    sql_support::each_chunk_mapped(
632        &visits,
633        |(_, place_id, _)| place_id.0,
634        |chunk, _| -> Result<()> {
635            let query = format!(
636                "SELECT id,
637                    (foreign_count != 0) AS has_foreign,
638                    ((last_visit_date_local + last_visit_date_remote) != 0) as has_visits,
639                    sync_status
640                FROM moz_places
641                WHERE id IN ({})",
642                sql_support::repeat_sql_vars(chunk.len()),
643            );
644
645            let mut stmt = db.conn().prepare(&query)?;
646            let page_results =
647                stmt.query_and_then(rusqlite::params_from_iter(chunk), PageToClean::from_row)?;
648            let pages: Vec<PageToClean> = page_results.collect::<Result<_>>()?;
649            cleanup_pages(db, &pages)
650        },
651    )?;
652
653    // Clean up history metadata between start and end
654    history_metadata::delete_between(db, start.as_millis_i64(), end.as_millis_i64())?;
655    delete_pending_temp_tables(db)?;
656    Ok(())
657}
658
659#[derive(Debug)]
660struct PageToClean {
661    id: RowId,
662    has_foreign: bool,
663    has_visits: bool,
664    sync_status: SyncStatus,
665}
666
667impl PageToClean {
668    pub fn from_row(row: &Row<'_>) -> Result<Self> {
669        Ok(Self {
670            id: row.get("id")?,
671            has_foreign: row.get("has_foreign")?,
672            has_visits: row.get("has_visits")?,
673            sync_status: row.get("sync_status")?,
674        })
675    }
676}
677
678/// Clean up pages whose history has been modified, by either
679/// removing them entirely (if they are marked for removal,
680/// typically because all visits have been removed and there
681/// are no more foreign keys such as bookmarks) or updating
682/// their frecency.
683fn cleanup_pages(db: &PlacesDb, pages: &[PageToClean]) -> Result<()> {
684    // desktop does this frecency work using a function in a single sql
685    // statement - we should see if we can do that too.
686    let frec_ids = pages
687        .iter()
688        .filter(|&p| p.has_foreign || p.has_visits)
689        .map(|p| p.id);
690
691    for id in frec_ids {
692        update_frecency(db, id, None)?;
693    }
694
695    // Like desktop, we do "AND foreign_count = 0 AND last_visit_date ISNULL"
696    // to creating orphans in case of async race conditions - in Desktop's
697    // case, it reads the pages before starting a write transaction, so that
698    // probably is possible. We don't currently do that, but might later, so
699    // we do it anyway.
700    let remove_ids: Vec<RowId> = pages
701        .iter()
702        .filter(|p| !p.has_foreign && !p.has_visits)
703        .map(|p| p.id)
704        .collect();
705    sql_support::each_chunk(&remove_ids, |chunk, _| -> Result<()> {
706        // tombstones first.
707        db.conn().execute(
708            &format!(
709                "
710                INSERT OR IGNORE INTO moz_places_tombstones (guid)
711                SELECT guid FROM moz_places
712                WHERE id in ({ids}) AND sync_status = {status}
713                    AND foreign_count = 0
714                    AND last_visit_date_local = 0
715                    AND last_visit_date_remote = 0",
716                ids = sql_support::repeat_sql_vars(chunk.len()),
717                status = SyncStatus::Normal as u8,
718            ),
719            rusqlite::params_from_iter(chunk),
720        )?;
721        db.conn().execute(
722            &format!(
723                "
724                DELETE FROM moz_places
725                WHERE id IN ({ids})
726                    AND foreign_count = 0
727                    AND last_visit_date_local = 0
728                    AND last_visit_date_remote = 0",
729                ids = sql_support::repeat_sql_vars(chunk.len())
730            ),
731            rusqlite::params_from_iter(chunk),
732        )?;
733        Ok(())
734    })?;
735
736    Ok(())
737}
738
739fn reset_in_tx(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
740    // Reset change counters and sync statuses for all URLs.
741    db.execute_cached(
742        &format!(
743            "
744            UPDATE moz_places
745                SET sync_change_counter = 0,
746                sync_status = {}",
747            (SyncStatus::New as u8)
748        ),
749        [],
750    )?;
751
752    // Reset the last sync time, so that the next sync fetches fresh records
753    // from the server.
754    put_meta(db, LAST_SYNC_META_KEY, &0)?;
755
756    // Clear the sync ID if we're signing out, or set it to whatever the
757    // server gave us if we're signing in.
758    match assoc {
759        EngineSyncAssociation::Disconnected => {
760            delete_meta(db, GLOBAL_SYNCID_META_KEY)?;
761            delete_meta(db, COLLECTION_SYNCID_META_KEY)?;
762        }
763        EngineSyncAssociation::Connected(ids) => {
764            put_meta(db, GLOBAL_SYNCID_META_KEY, &ids.global)?;
765            put_meta(db, COLLECTION_SYNCID_META_KEY, &ids.coll)?;
766        }
767    }
768
769    Ok(())
770}
771
772// Support for Sync - in its own module to try and keep a delineation
773pub mod history_sync {
774    use sync15::bso::OutgoingEnvelope;
775
776    use super::*;
777    use crate::history_sync::record::{HistoryRecord, HistoryRecordVisit};
778    use crate::history_sync::HISTORY_TTL;
779    use std::collections::HashSet;
780
781    #[derive(Debug, Clone, PartialEq, Eq)]
782    pub struct FetchedVisit {
783        pub is_local: bool,
784        pub visit_date: Timestamp,
785        pub visit_type: Option<VisitType>,
786    }
787
788    impl FetchedVisit {
789        pub fn from_row(row: &Row<'_>) -> Result<Self> {
790            Ok(Self {
791                is_local: row.get("is_local")?,
792                visit_date: row
793                    .get::<_, Option<Timestamp>>("visit_date")?
794                    .unwrap_or_default(),
795                visit_type: VisitType::from_primitive(
796                    row.get::<_, Option<u8>>("visit_type")?.unwrap_or(0),
797                ),
798            })
799        }
800    }
801
802    #[derive(Debug)]
803    pub struct FetchedVisitPage {
804        pub url: Url,
805        pub guid: SyncGuid,
806        pub row_id: RowId,
807        pub title: String,
808        pub unknown_fields: UnknownFields,
809    }
810
811    impl FetchedVisitPage {
812        pub fn from_row(row: &Row<'_>) -> Result<Self> {
813            Ok(Self {
814                url: Url::parse(&row.get::<_, String>("url")?)?,
815                guid: row.get::<_, String>("guid")?.into(),
816                row_id: row.get("id")?,
817                title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
818                unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
819                    None => UnknownFields::new(),
820                    Some(v) => serde_json::from_str(&v)?,
821                },
822            })
823        }
824    }
825
826    pub fn fetch_visits(
827        db: &PlacesDb,
828        url: &Url,
829        limit: usize,
830    ) -> Result<Option<(FetchedVisitPage, Vec<FetchedVisit>)>> {
831        // We do this in 2 steps - "do we have a page" then "get visits"
832        let page_sql = "
833          SELECT guid, url, id, title, unknown_fields
834          FROM moz_places h
835          WHERE url_hash = hash(:url) AND url = :url";
836
837        let page_info = match db.try_query_row(
838            page_sql,
839            &[(":url", &url.to_string())],
840            FetchedVisitPage::from_row,
841            true,
842        )? {
843            None => return Ok(None),
844            Some(pi) => pi,
845        };
846
847        let visits = db.query_rows_and_then(
848            "SELECT is_local, visit_type, visit_date
849            FROM moz_historyvisits
850            WHERE place_id = :place_id
851            LIMIT :limit",
852            &[
853                (":place_id", &page_info.row_id as &dyn rusqlite::ToSql),
854                (":limit", &(limit as u32)),
855            ],
856            FetchedVisit::from_row,
857        )?;
858        Ok(Some((page_info, visits)))
859    }
860
861    /// Apply history visit from sync. This assumes they have all been
862    /// validated, deduped, etc - it's just the storage we do here.
863    pub fn apply_synced_visits(
864        db: &PlacesDb,
865        incoming_guid: &SyncGuid,
866        url: &Url,
867        title: &Option<String>,
868        visits: &[HistoryRecordVisit],
869        unknown_fields: &UnknownFields,
870    ) -> Result<()> {
871        // At some point we may have done a local wipe of all visits. We skip applying
872        // incoming visits that could have been part of that deletion, to avoid them
873        // trickling back in.
874        let visit_ignored_mark =
875            get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
876
877        let visits = visits
878            .iter()
879            .filter(|v| Timestamp::from(v.date) > visit_ignored_mark)
880            .collect::<Vec<_>>();
881
882        let mut counter_incr = 0;
883        let page_info = match fetch_page_info(db, url)? {
884            Some(mut info) => {
885                // If the existing record has not yet been synced, then we will
886                // change the GUID to the incoming one. If it has been synced
887                // we keep the existing guid, but still apply the visits.
888                // See doc/history_duping.rst for more details.
889                if &info.page.guid != incoming_guid {
890                    if info.page.sync_status == SyncStatus::New {
891                        db.execute_cached(
892                            "UPDATE moz_places SET guid = :new_guid WHERE id = :row_id",
893                            &[
894                                (":new_guid", incoming_guid as &dyn rusqlite::ToSql),
895                                (":row_id", &info.page.row_id),
896                            ],
897                        )?;
898                        info.page.guid = incoming_guid.clone();
899                    }
900                    // Even if we didn't take the new guid, we are going to
901                    // take the new visits - so we want the change counter to
902                    // reflect there are changes.
903                    counter_incr = 1;
904                }
905                info.page
906            }
907            None => {
908                // Before we insert a new page_info, make sure we actually will
909                // have any visits to add.
910                if visits.is_empty() {
911                    return Ok(());
912                }
913                new_page_info(db, url, Some(incoming_guid.clone()))?
914            }
915        };
916
917        if !visits.is_empty() {
918            // Skip visits that are in tombstones, or that happen at the same time
919            // as visit that's already present. The 2nd lets us avoid inserting
920            // visits that we sent up to the server in the first place.
921            //
922            // It does cause us to ignore visits that legitimately happen
923            // at the same time, but that's probably fine and not worth
924            // worrying about.
925            let mut visits_to_skip: HashSet<Timestamp> = db.query_rows_into(
926                &format!(
927                    "SELECT t.visit_date AS visit_date
928                     FROM moz_historyvisit_tombstones t
929                     WHERE t.place_id = {place}
930                       AND t.visit_date IN ({dates})
931                     UNION ALL
932                     SELECT v.visit_date AS visit_date
933                     FROM moz_historyvisits v
934                     WHERE v.place_id = {place}
935                       AND v.visit_date IN ({dates})",
936                    place = page_info.row_id,
937                    dates = sql_support::repeat_display(visits.len(), ",", |i, f| write!(
938                        f,
939                        "{}",
940                        Timestamp::from(visits[i].date).0
941                    )),
942                ),
943                [],
944                |row| row.get::<_, Timestamp>(0),
945            )?;
946
947            visits_to_skip.reserve(visits.len());
948
949            for visit in visits {
950                let timestamp = Timestamp::from(visit.date);
951                // Don't insert visits that have been locally deleted.
952                if visits_to_skip.contains(&timestamp) {
953                    continue;
954                }
955                let transition = VisitType::from_primitive(visit.transition)
956                    .expect("these should already be validated");
957                add_visit(
958                    db,
959                    page_info.row_id,
960                    None,
961                    timestamp,
962                    transition,
963                    false,
964                    serialize_unknown_fields(&visit.unknown_fields)?,
965                )?;
966                // Make sure that even if a history entry weirdly has the same visit
967                // twice, we don't insert it twice. (This avoids us needing to
968                // recompute visits_to_skip in each step of the iteration)
969                visits_to_skip.insert(timestamp);
970            }
971        }
972        // XXX - we really need a better story for frecency-boost than
973        // Option<bool> - None vs Some(false) is confusing. We should use an enum.
974        update_frecency(db, page_info.row_id, None)?;
975
976        // and the place itself if necessary.
977        let new_title = title.as_ref().unwrap_or(&page_info.title);
978        // We set the Status to Normal, otherwise we will re-upload it as
979        // outgoing even if nothing has changed. Note that we *do not* reset
980        // the change counter - if it is non-zero now, we want it to remain
981        // as non-zero, so we do re-upload it if there were actual changes)
982        db.execute_cached(
983            "UPDATE moz_places
984             SET title = :title,
985                 unknown_fields = :unknown_fields,
986                 sync_status = :status,
987                 sync_change_counter = :sync_change_counter
988             WHERE id == :row_id",
989            &[
990                (":title", new_title as &dyn rusqlite::ToSql),
991                (":row_id", &page_info.row_id),
992                (":status", &SyncStatus::Normal),
993                (
994                    ":unknown_fields",
995                    &serialize_unknown_fields(unknown_fields)?,
996                ),
997                (
998                    ":sync_change_counter",
999                    &(page_info.sync_change_counter + counter_incr),
1000                ),
1001            ],
1002        )?;
1003
1004        Ok(())
1005    }
1006
1007    pub fn apply_synced_reconciliation(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1008        db.execute_cached(
1009            "UPDATE moz_places
1010                SET sync_status = :status,
1011                    sync_change_counter = 0
1012             WHERE guid == :guid",
1013            &[
1014                (":guid", guid as &dyn rusqlite::ToSql),
1015                (":status", &SyncStatus::Normal),
1016            ],
1017        )?;
1018        Ok(())
1019    }
1020
1021    pub fn apply_synced_deletion(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1022        // First we delete any visits for the page
1023        // because it's possible the moz_places foreign_count is not 0
1024        // and thus the moz_places entry won't be deleted.
1025        db.execute_cached(
1026            "DELETE FROM moz_historyvisits
1027              WHERE place_id IN (
1028                  SELECT id
1029                  FROM moz_places
1030                  WHERE guid = :guid
1031              )",
1032            &[(":guid", guid)],
1033        )?;
1034        db.execute_cached(
1035            "DELETE FROM moz_places WHERE guid = :guid AND foreign_count = 0",
1036            &[(":guid", guid)],
1037        )?;
1038        Ok(())
1039    }
1040
1041    pub fn fetch_outgoing(
1042        db: &PlacesDb,
1043        max_places: usize,
1044        max_visits: usize,
1045    ) -> Result<Vec<OutgoingBso>> {
1046        // Note that we want *all* "new" regardless of change counter,
1047        // so that we do the right thing after a "reset". We also
1048        // exclude hidden URLs from syncing, to match Desktop
1049        // (bug 1173359).
1050        let places_sql = format!(
1051            "
1052            SELECT guid, url, id, title, hidden, typed, frecency,
1053                visit_count_local, visit_count_remote,
1054                last_visit_date_local, last_visit_date_remote,
1055                sync_status, sync_change_counter, preview_image_url,
1056                unknown_fields
1057            FROM moz_places
1058            WHERE (sync_change_counter > 0 OR sync_status != {}) AND
1059                  NOT hidden
1060            ORDER BY frecency DESC
1061            LIMIT :max_places",
1062            (SyncStatus::Normal as u8)
1063        );
1064        let visits_sql = "
1065            SELECT visit_date as date, visit_type as transition, unknown_fields
1066            FROM moz_historyvisits
1067            WHERE place_id = :place_id
1068            ORDER BY visit_date DESC
1069            LIMIT :max_visits";
1070        // tombstones
1071        let tombstones_sql = "SELECT guid FROM moz_places_tombstones LIMIT :max_places";
1072
1073        let mut tombstone_ids = HashSet::new();
1074        let mut result = Vec::new();
1075
1076        // We want to limit to 5000 places - tombstones are arguably the
1077        // most important, so we fetch these first.
1078        let ts_rows = db.query_rows_and_then(
1079            tombstones_sql,
1080            &[(":max_places", &(max_places as u32))],
1081            |row| -> rusqlite::Result<SyncGuid> { Ok(row.get::<_, String>("guid")?.into()) },
1082        )?;
1083        // It's unfortunatee that query_rows_and_then returns a Vec instead of an iterator
1084        // (which would be very hard to do), but as long as we have it, we might as well make use
1085        // of it...
1086        result.reserve(ts_rows.len());
1087        tombstone_ids.reserve(ts_rows.len());
1088        for guid in ts_rows {
1089            trace!("outgoing tombstone {:?}", &guid);
1090            let envelope = OutgoingEnvelope {
1091                id: guid.clone(),
1092                ttl: Some(HISTORY_TTL),
1093                ..Default::default()
1094            };
1095            result.push(OutgoingBso::new_tombstone(envelope));
1096            tombstone_ids.insert(guid);
1097        }
1098
1099        // Max records is now limited by how many tombstones we found.
1100        let max_places_left = max_places - result.len();
1101
1102        // We write info about the records we are updating to a temp table.
1103        // While we could carry this around in memory, we'll need a temp table
1104        // in `finish_outgoing` anyway, because we execute a `NOT IN` query
1105        // there - which, in a worst-case scenario, is a very large `NOT IN`
1106        // set.
1107        db.execute(
1108            "CREATE TEMP TABLE IF NOT EXISTS temp_sync_updated_meta
1109                    (id INTEGER PRIMARY KEY,
1110                     change_delta INTEGER NOT NULL)",
1111            [],
1112        )?;
1113
1114        let insert_meta_sql = "
1115            INSERT INTO temp_sync_updated_meta VALUES (:row_id, :change_delta)";
1116
1117        let rows = db.query_rows_and_then(
1118            &places_sql,
1119            &[(":max_places", &(max_places_left as u32))],
1120            PageInfo::from_row,
1121        )?;
1122        result.reserve(rows.len());
1123        let mut ids_to_update = Vec::with_capacity(rows.len());
1124        for page in rows {
1125            let visits = db.query_rows_and_then_cached(
1126                visits_sql,
1127                &[
1128                    (":max_visits", &(max_visits as u32) as &dyn rusqlite::ToSql),
1129                    (":place_id", &page.row_id),
1130                ],
1131                |row| -> Result<_> {
1132                    Ok(HistoryRecordVisit {
1133                        date: row.get::<_, Timestamp>("date")?.into(),
1134                        transition: row.get::<_, u8>("transition")?,
1135                        unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
1136                            None => UnknownFields::new(),
1137                            Some(v) => serde_json::from_str(&v)?,
1138                        },
1139                    })
1140                },
1141            )?;
1142            if tombstone_ids.contains(&page.guid) {
1143                // should be impossible!
1144                warn!("Found {:?} in both tombstones and live records", &page.guid);
1145                continue;
1146            }
1147            if visits.is_empty() {
1148                // This will be true for things like bookmarks which haven't
1149                // had visits locally applied, and if we later prune old visits
1150                // we'll also hit it, so don't make much log noise.
1151                trace!(
1152                    "Page {:?} is flagged to be uploaded, but has no visits - skipping",
1153                    &page.guid
1154                );
1155                continue;
1156            }
1157            trace!("outgoing record {:?}", &page.guid);
1158            ids_to_update.push(page.row_id);
1159            db.execute_cached(
1160                insert_meta_sql,
1161                &[
1162                    (":row_id", &page.row_id as &dyn rusqlite::ToSql),
1163                    (":change_delta", &page.sync_change_counter),
1164                ],
1165            )?;
1166
1167            let content = HistoryRecord {
1168                id: page.guid.clone(),
1169                title: page.title,
1170                hist_uri: page.url.to_string(),
1171                visits,
1172                unknown_fields: page.unknown_fields,
1173            };
1174
1175            let envelope = OutgoingEnvelope {
1176                id: page.guid,
1177                sortindex: Some(page.frecency),
1178                ttl: Some(HISTORY_TTL),
1179            };
1180            let bso = OutgoingBso::from_content(envelope, content)?;
1181            result.push(bso);
1182        }
1183
1184        // We need to update the sync status of these items now rather than after
1185        // the upload, because if we are interrupted between upload and writing
1186        // we could end up with local items with state New even though we
1187        // uploaded them.
1188        sql_support::each_chunk(&ids_to_update, |chunk, _| -> Result<()> {
1189            db.conn().execute(
1190                &format!(
1191                    "UPDATE moz_places SET sync_status={status}
1192                                 WHERE id IN ({vars})",
1193                    vars = sql_support::repeat_sql_vars(chunk.len()),
1194                    status = SyncStatus::Normal as u8
1195                ),
1196                rusqlite::params_from_iter(chunk),
1197            )?;
1198            Ok(())
1199        })?;
1200
1201        Ok(result)
1202    }
1203
1204    pub fn finish_outgoing(db: &PlacesDb) -> Result<()> {
1205        // So all items *other* than those above must be set to "not dirty"
1206        // (ie, status=SyncStatus::Normal, change_counter=0). Otherwise every
1207        // subsequent sync will continue to add more and more local pages
1208        // until every page we have is uploaded. And we only want to do it
1209        // at the end of the sync because if we are interrupted, we'll end up
1210        // thinking we have nothing to upload.
1211        // BUT - this is potentially alot of rows! Because we want "NOT IN (...)"
1212        // we can't do chunking and building a literal string with the ids seems
1213        // wrong and likely to hit max sql length limits.
1214        // So we use a temp table.
1215        debug!("Updating all synced rows");
1216        // XXX - is there a better way to express this SQL? Multi-selects
1217        // doesn't seem ideal...
1218        db.conn().execute_cached(
1219            "
1220            UPDATE moz_places
1221                SET sync_change_counter = sync_change_counter -
1222                (SELECT change_delta FROM temp_sync_updated_meta m WHERE moz_places.id = m.id)
1223            WHERE id IN (SELECT id FROM temp_sync_updated_meta)
1224            ",
1225            [],
1226        )?;
1227
1228        debug!("Updating all non-synced rows");
1229        db.execute_all(&[
1230            &format!(
1231                "UPDATE moz_places
1232                    SET sync_change_counter = 0, sync_status = {}
1233                WHERE id NOT IN (SELECT id from temp_sync_updated_meta)",
1234                (SyncStatus::Normal as u8)
1235            ),
1236            "DELETE FROM temp_sync_updated_meta",
1237        ])?;
1238
1239        debug!("Removing local tombstones");
1240        db.conn()
1241            .execute_cached("DELETE from moz_places_tombstones", [])?;
1242
1243        Ok(())
1244    }
1245
1246    /// Resets all sync metadata, including change counters, sync statuses,
1247    /// the last sync time, and sync ID. This should be called when the user
1248    /// signs out of Sync.
1249    pub(crate) fn reset(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
1250        let tx = db.begin_transaction()?;
1251        reset_in_tx(db, assoc)?;
1252        tx.commit()?;
1253        Ok(())
1254    }
1255} // end of sync module.
1256
1257pub fn get_visited<I>(db: &PlacesDb, urls: I) -> Result<Vec<bool>>
1258where
1259    I: IntoIterator<Item = Url>,
1260    I::IntoIter: ExactSizeIterator,
1261{
1262    let iter = urls.into_iter();
1263    let mut result = vec![false; iter.len()];
1264    let url_idxs = iter.enumerate().collect::<Vec<_>>();
1265    get_visited_into(db, &url_idxs, &mut result)?;
1266    Ok(result)
1267}
1268
1269/// Low level api used to implement both get_visited and the FFI get_visited call.
1270/// Takes a slice where we should output the results, as well as a slice of
1271/// index/url pairs.
1272///
1273/// This is done so that the FFI can more easily support returning
1274/// false when asked if it's visited an invalid URL.
1275pub fn get_visited_into(
1276    db: &PlacesDb,
1277    urls_idxs: &[(usize, Url)],
1278    result: &mut [bool],
1279) -> Result<()> {
1280    sql_support::each_chunk_mapped(
1281        urls_idxs,
1282        |(_, url)| url.as_str(),
1283        |chunk, offset| -> Result<()> {
1284            let values_with_idx = sql_support::repeat_display(chunk.len(), ",", |i, f| {
1285                let (idx, url) = &urls_idxs[i + offset];
1286                write!(f, "({},{},?)", *idx, hash::hash_url(url.as_str()))
1287            });
1288            let sql = format!(
1289                "WITH to_fetch(fetch_url_index, url_hash, url) AS (VALUES {})
1290                 SELECT fetch_url_index
1291                 FROM moz_places h
1292                 JOIN to_fetch f ON h.url_hash = f.url_hash
1293                   AND h.url = f.url
1294                   AND (h.last_visit_date_local + h.last_visit_date_remote) != 0",
1295                values_with_idx
1296            );
1297            let mut stmt = db.prepare(&sql)?;
1298            for idx_r in stmt.query_and_then(
1299                rusqlite::params_from_iter(chunk),
1300                |row| -> rusqlite::Result<_> { Ok(row.get::<_, i64>(0)? as usize) },
1301            )? {
1302                let idx = idx_r?;
1303                result[idx] = true;
1304            }
1305            Ok(())
1306        },
1307    )?;
1308    Ok(())
1309}
1310
1311/// Get the set of urls that were visited between `start` and `end`. Only considers local visits
1312/// unless you pass in `include_remote`.
1313pub fn get_visited_urls(
1314    db: &PlacesDb,
1315    start: Timestamp,
1316    end: Timestamp,
1317    include_remote: bool,
1318) -> Result<Vec<String>> {
1319    // TODO: if `end` is >= now then we can probably just look at last_visit_date_{local,remote},
1320    // and avoid touching `moz_historyvisits` at all. That said, this query is taken more or less
1321    // from what places does so it's probably fine.
1322    let sql = format!(
1323        "SELECT h.url
1324        FROM moz_places h
1325        WHERE EXISTS (
1326            SELECT 1 FROM moz_historyvisits v
1327            WHERE place_id = h.id
1328                AND visit_date BETWEEN :start AND :end
1329                {and_is_local}
1330            LIMIT 1
1331        )",
1332        and_is_local = if include_remote { "" } else { "AND is_local" }
1333    );
1334    Ok(db.query_rows_and_then_cached(
1335        &sql,
1336        &[(":start", &start), (":end", &end)],
1337        |row| -> RusqliteResult<_> { row.get::<_, String>(0) },
1338    )?)
1339}
1340
1341pub fn get_top_frecent_site_infos(
1342    db: &PlacesDb,
1343    num_items: i32,
1344    frecency_threshold: i64,
1345) -> Result<Vec<TopFrecentSiteInfo>> {
1346    // Get the complement of the visit types that should be excluded.
1347    let allowed_types = VisitTransitionSet::for_specific(&[
1348        VisitType::Download,
1349        VisitType::Embed,
1350        VisitType::RedirectPermanent,
1351        VisitType::RedirectTemporary,
1352        VisitType::FramedLink,
1353        VisitType::Reload,
1354    ])
1355    .complement();
1356
1357    let infos = db.query_rows_and_then_cached(
1358        "SELECT h.frecency, h.title, h.url
1359        FROM moz_places h
1360        WHERE EXISTS (
1361            SELECT v.visit_type
1362            FROM moz_historyvisits v
1363            WHERE h.id = v.place_id
1364              AND (SUBSTR(h.url, 1, 6) == 'https:' OR SUBSTR(h.url, 1, 5) == 'http:')
1365              AND (h.last_visit_date_local + h.last_visit_date_remote) != 0
1366              AND ((1 << v.visit_type) & :allowed_types) != 0
1367              AND h.frecency >= :frecency_threshold AND
1368              NOT h.hidden
1369        )
1370        ORDER BY h.frecency DESC
1371        LIMIT :limit",
1372        rusqlite::named_params! {
1373            ":limit": num_items,
1374            ":allowed_types": allowed_types,
1375            ":frecency_threshold": frecency_threshold,
1376        },
1377        TopFrecentSiteInfo::from_row,
1378    )?;
1379    Ok(infos)
1380}
1381
1382pub fn get_visit_infos(
1383    db: &PlacesDb,
1384    start: Timestamp,
1385    end: Timestamp,
1386    exclude_types: VisitTransitionSet,
1387) -> Result<Vec<HistoryVisitInfo>> {
1388    let allowed_types = exclude_types.complement();
1389    let infos = db.query_rows_and_then_cached(
1390        "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1391                v.is_local
1392         FROM moz_places h
1393         JOIN moz_historyvisits v
1394           ON h.id = v.place_id
1395         WHERE v.visit_date BETWEEN :start AND :end
1396           AND ((1 << visit_type) & :allowed_types) != 0 AND
1397           NOT h.hidden
1398         ORDER BY v.visit_date",
1399        rusqlite::named_params! {
1400            ":start": start,
1401            ":end": end,
1402            ":allowed_types": allowed_types,
1403        },
1404        HistoryVisitInfo::from_row,
1405    )?;
1406    Ok(infos)
1407}
1408
1409pub fn get_visit_count(db: &PlacesDb, exclude_types: VisitTransitionSet) -> Result<i64> {
1410    let count = if exclude_types.is_empty() {
1411        db.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")?
1412    } else {
1413        let allowed_types = exclude_types.complement();
1414        db.query_row_and_then_cachable(
1415            "SELECT COUNT(*)
1416             FROM moz_historyvisits
1417             WHERE ((1 << visit_type) & :allowed_types) != 0",
1418            rusqlite::named_params! {
1419                ":allowed_types": allowed_types,
1420            },
1421            |r| r.get(0),
1422            true,
1423        )?
1424    };
1425    Ok(count)
1426}
1427
1428pub fn get_visit_count_for_host(
1429    db: &PlacesDb,
1430    host: &str,
1431    before: Timestamp,
1432    exclude_types: VisitTransitionSet,
1433) -> Result<i64> {
1434    let allowed_types = exclude_types.complement();
1435    let count = db.query_row_and_then_cachable(
1436        "SELECT COUNT(*)
1437        FROM moz_historyvisits
1438        JOIN moz_places ON moz_places.id = moz_historyvisits.place_id
1439        JOIN moz_origins ON moz_origins.id = moz_places.origin_id
1440        WHERE moz_origins.host = :host
1441          AND visit_date < :before
1442          AND ((1 << visit_type) & :allowed_types) != 0",
1443        rusqlite::named_params! {
1444            ":host": host,
1445            ":before": before,
1446            ":allowed_types": allowed_types,
1447        },
1448        |r| r.get(0),
1449        true,
1450    )?;
1451    Ok(count)
1452}
1453
1454pub fn get_visit_page(
1455    db: &PlacesDb,
1456    offset: i64,
1457    count: i64,
1458    exclude_types: VisitTransitionSet,
1459) -> Result<Vec<HistoryVisitInfo>> {
1460    let allowed_types = exclude_types.complement();
1461    let infos = db.query_rows_and_then_cached(
1462        "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1463                v.is_local
1464         FROM moz_places h
1465         JOIN moz_historyvisits v
1466           ON h.id = v.place_id
1467         WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1468               NOT h.hidden
1469         ORDER BY v.visit_date DESC, v.id
1470         LIMIT :count
1471         OFFSET :offset",
1472        rusqlite::named_params! {
1473            ":count": count,
1474            ":offset": offset,
1475            ":allowed_types": allowed_types,
1476        },
1477        HistoryVisitInfo::from_row,
1478    )?;
1479    Ok(infos)
1480}
1481
1482pub fn get_visit_page_with_bound(
1483    db: &PlacesDb,
1484    bound: i64,
1485    offset: i64,
1486    count: i64,
1487    exclude_types: VisitTransitionSet,
1488) -> Result<HistoryVisitInfosWithBound> {
1489    let allowed_types = exclude_types.complement();
1490    let infos = db.query_rows_and_then_cached(
1491        "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1492                v.is_local
1493         FROM moz_places h
1494         JOIN moz_historyvisits v
1495           ON h.id = v.place_id
1496         WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1497               NOT h.hidden
1498               AND v.visit_date <= :bound
1499         ORDER BY v.visit_date DESC, v.id
1500         LIMIT :count
1501         OFFSET :offset",
1502        rusqlite::named_params! {
1503            ":allowed_types": allowed_types,
1504            ":bound": bound,
1505            ":count": count,
1506            ":offset": offset,
1507        },
1508        HistoryVisitInfo::from_row,
1509    )?;
1510
1511    if let Some(l) = infos.last() {
1512        if l.timestamp.as_millis_i64() == bound {
1513            // all items' timestamp are equal to the previous bound
1514            let offset = offset + infos.len() as i64;
1515            Ok(HistoryVisitInfosWithBound {
1516                infos,
1517                bound,
1518                offset,
1519            })
1520        } else {
1521            let bound = l.timestamp;
1522            let offset = infos
1523                .iter()
1524                .rev()
1525                .take_while(|i| i.timestamp == bound)
1526                .count() as i64;
1527            Ok(HistoryVisitInfosWithBound {
1528                infos,
1529                bound: bound.as_millis_i64(),
1530                offset,
1531            })
1532        }
1533    } else {
1534        // infos is Empty
1535        Ok(HistoryVisitInfosWithBound {
1536            infos,
1537            bound: 0,
1538            offset: 0,
1539        })
1540    }
1541}
1542
1543#[cfg(test)]
1544mod tests {
1545    use super::history_sync::*;
1546    use super::*;
1547    use crate::history_sync::record::HistoryRecordVisit;
1548    use crate::storage::bookmarks::{insert_bookmark, InsertableItem};
1549    use crate::types::VisitTransitionSet;
1550    use crate::{api::places_api::ConnectionType, storage::bookmarks::BookmarkRootGuid};
1551    use pretty_assertions::assert_eq;
1552    use std::time::{Duration, SystemTime};
1553    use sync15::engine::CollSyncIds;
1554    use types::Timestamp;
1555
1556    #[test]
1557    fn test_get_visited_urls() {
1558        use std::collections::HashSet;
1559        use std::time::SystemTime;
1560        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1561        let now: Timestamp = SystemTime::now().into();
1562        let now_u64 = now.0;
1563        // (url, when, is_remote, (expected_always, expected_only_local)
1564        let to_add = [
1565            (
1566                "https://www.example.com/1",
1567                now_u64 - 200_100,
1568                false,
1569                (false, false),
1570            ),
1571            (
1572                "https://www.example.com/12",
1573                now_u64 - 200_000,
1574                false,
1575                (true, true),
1576            ),
1577            (
1578                "https://www.example.com/123",
1579                now_u64 - 10_000,
1580                true,
1581                (true, false),
1582            ),
1583            (
1584                "https://www.example.com/1234",
1585                now_u64 - 1000,
1586                false,
1587                (true, true),
1588            ),
1589            (
1590                "https://www.mozilla.com",
1591                now_u64 - 500,
1592                false,
1593                (false, false),
1594            ),
1595        ];
1596
1597        for &(url, when, remote, _) in &to_add {
1598            apply_observation(
1599                &conn,
1600                VisitObservation::new(Url::parse(url).unwrap())
1601                    .with_at(Timestamp(when))
1602                    .with_is_remote(remote)
1603                    .with_visit_type(VisitType::Link),
1604            )
1605            .expect("Should apply visit");
1606        }
1607
1608        let visited_all = get_visited_urls(
1609            &conn,
1610            Timestamp(now_u64 - 200_000),
1611            Timestamp(now_u64 - 1000),
1612            true,
1613        )
1614        .unwrap()
1615        .into_iter()
1616        .collect::<HashSet<_>>();
1617
1618        let visited_local = get_visited_urls(
1619            &conn,
1620            Timestamp(now_u64 - 200_000),
1621            Timestamp(now_u64 - 1000),
1622            false,
1623        )
1624        .unwrap()
1625        .into_iter()
1626        .collect::<HashSet<_>>();
1627
1628        for &(url, ts, is_remote, (expected_in_all, expected_in_local)) in &to_add {
1629            // Make sure we format stuff the same way (in practice, just trailing slashes)
1630            let url = Url::parse(url).unwrap().to_string();
1631            assert_eq!(
1632                expected_in_local,
1633                visited_local.contains(&url),
1634                "Failed in local for {:?}",
1635                (url, ts, is_remote)
1636            );
1637            assert_eq!(
1638                expected_in_all,
1639                visited_all.contains(&url),
1640                "Failed in all for {:?}",
1641                (url, ts, is_remote)
1642            );
1643        }
1644    }
1645
1646    fn get_custom_observed_page<F>(conn: &mut PlacesDb, url: &str, custom: F) -> Result<PageInfo>
1647    where
1648        F: Fn(VisitObservation) -> VisitObservation,
1649    {
1650        let u = Url::parse(url)?;
1651        let obs = VisitObservation::new(u.clone()).with_visit_type(VisitType::Link);
1652        apply_observation(conn, custom(obs))?;
1653        Ok(fetch_page_info(conn, &u)?
1654            .expect("should have the page")
1655            .page)
1656    }
1657
1658    fn get_observed_page(conn: &mut PlacesDb, url: &str) -> Result<PageInfo> {
1659        get_custom_observed_page(conn, url, |o| o)
1660    }
1661
1662    fn get_tombstone_count(conn: &PlacesDb) -> u32 {
1663        let result: Result<Option<u32>> = conn.try_query_row(
1664            "SELECT COUNT(*) from moz_places_tombstones;",
1665            [],
1666            |row| Ok(row.get::<_, u32>(0)?),
1667            true,
1668        );
1669        result
1670            .expect("should have worked")
1671            .expect("should have got a value")
1672    }
1673
1674    #[test]
1675    fn test_visit_counts() -> Result<()> {
1676        error_support::init_for_tests();
1677        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1678        let url = Url::parse("https://www.example.com").expect("it's a valid url");
1679        let early_time = SystemTime::now() - Duration::new(60, 0);
1680        let late_time = SystemTime::now();
1681
1682        // add 2 local visits - add latest first
1683        let rid1 = apply_observation(
1684            &conn,
1685            VisitObservation::new(url.clone())
1686                .with_visit_type(VisitType::Link)
1687                .with_at(Some(late_time.into())),
1688        )?
1689        .expect("should get a rowid");
1690
1691        let rid2 = apply_observation(
1692            &conn,
1693            VisitObservation::new(url.clone())
1694                .with_visit_type(VisitType::Link)
1695                .with_at(Some(early_time.into())),
1696        )?
1697        .expect("should get a rowid");
1698
1699        let mut pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1700        assert_eq!(pi.page.visit_count_local, 2);
1701        assert_eq!(pi.page.last_visit_date_local, late_time.into());
1702        assert_eq!(pi.page.visit_count_remote, 0);
1703        assert_eq!(pi.page.last_visit_date_remote.0, 0);
1704
1705        // 2 remote visits, earliest first.
1706        let rid3 = apply_observation(
1707            &conn,
1708            VisitObservation::new(url.clone())
1709                .with_visit_type(VisitType::Link)
1710                .with_at(Some(early_time.into()))
1711                .with_is_remote(true),
1712        )?
1713        .expect("should get a rowid");
1714
1715        let rid4 = apply_observation(
1716            &conn,
1717            VisitObservation::new(url.clone())
1718                .with_visit_type(VisitType::Link)
1719                .with_at(Some(late_time.into()))
1720                .with_is_remote(true),
1721        )?
1722        .expect("should get a rowid");
1723
1724        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1725        assert_eq!(pi.page.visit_count_local, 2);
1726        assert_eq!(pi.page.last_visit_date_local, late_time.into());
1727        assert_eq!(pi.page.visit_count_remote, 2);
1728        assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1729
1730        // Delete some and make sure things update.
1731        // XXX - we should add a trigger to update frecency on delete, but at
1732        // this stage we don't "officially" support deletes, so this is TODO.
1733        let sql = "DELETE FROM moz_historyvisits WHERE id = :row_id";
1734        // Delete the latest local visit.
1735        conn.execute_cached(sql, &[(":row_id", &rid1)])?;
1736        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1737        assert_eq!(pi.page.visit_count_local, 1);
1738        assert_eq!(pi.page.last_visit_date_local, early_time.into());
1739        assert_eq!(pi.page.visit_count_remote, 2);
1740        assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1741
1742        // Delete the earliest remote  visit.
1743        conn.execute_cached(sql, &[(":row_id", &rid3)])?;
1744        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1745        assert_eq!(pi.page.visit_count_local, 1);
1746        assert_eq!(pi.page.last_visit_date_local, early_time.into());
1747        assert_eq!(pi.page.visit_count_remote, 1);
1748        assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1749
1750        // Delete all visits.
1751        conn.execute_cached(sql, &[(":row_id", &rid2)])?;
1752        conn.execute_cached(sql, &[(":row_id", &rid4)])?;
1753        // It may turn out that we also delete the place after deleting all
1754        // visits, but for now we don't - check the values are sane though.
1755        pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1756        assert_eq!(pi.page.visit_count_local, 0);
1757        assert_eq!(pi.page.last_visit_date_local, Timestamp(0));
1758        assert_eq!(pi.page.visit_count_remote, 0);
1759        assert_eq!(pi.page.last_visit_date_remote, Timestamp(0));
1760        Ok(())
1761    }
1762
1763    #[test]
1764    fn test_get_visited() -> Result<()> {
1765        error_support::init_for_tests();
1766        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1767
1768        let unicode_in_path = "http://www.example.com/tëst😀abc";
1769        let escaped_unicode_in_path = "http://www.example.com/t%C3%ABst%F0%9F%98%80abc";
1770
1771        let unicode_in_domain = "http://www.exämple😀123.com";
1772        let escaped_unicode_in_domain = "http://www.xn--exmple123-w2a24222l.com";
1773
1774        let to_add = [
1775            "https://www.example.com/1".to_string(),
1776            "https://www.example.com/12".to_string(),
1777            "https://www.example.com/123".to_string(),
1778            "https://www.example.com/1234".to_string(),
1779            "https://www.mozilla.com".to_string(),
1780            "https://www.firefox.com".to_string(),
1781            unicode_in_path.to_string() + "/1",
1782            escaped_unicode_in_path.to_string() + "/2",
1783            unicode_in_domain.to_string() + "/1",
1784            escaped_unicode_in_domain.to_string() + "/2",
1785        ];
1786
1787        for item in &to_add {
1788            apply_observation(
1789                &conn,
1790                VisitObservation::new(Url::parse(item).unwrap()).with_visit_type(VisitType::Link),
1791            )?;
1792        }
1793
1794        let to_search = [
1795            ("https://www.example.com".to_string(), false),
1796            ("https://www.example.com/1".to_string(), true),
1797            ("https://www.example.com/12".to_string(), true),
1798            ("https://www.example.com/123".to_string(), true),
1799            ("https://www.example.com/1234".to_string(), true),
1800            ("https://www.example.com/12345".to_string(), false),
1801            ("https://www.mozilla.com".to_string(), true),
1802            ("https://www.firefox.com".to_string(), true),
1803            ("https://www.mozilla.org".to_string(), false),
1804            // dupes should still work!
1805            ("https://www.example.com/1234".to_string(), true),
1806            ("https://www.example.com/12345".to_string(), false),
1807            // The unicode URLs should work when escaped the way we
1808            // encountered them
1809            (unicode_in_path.to_string() + "/1", true),
1810            (escaped_unicode_in_path.to_string() + "/2", true),
1811            (unicode_in_domain.to_string() + "/1", true),
1812            (escaped_unicode_in_domain.to_string() + "/2", true),
1813            // But also the other way.
1814            (unicode_in_path.to_string() + "/2", true),
1815            (escaped_unicode_in_path.to_string() + "/1", true),
1816            (unicode_in_domain.to_string() + "/2", true),
1817            (escaped_unicode_in_domain.to_string() + "/1", true),
1818        ];
1819
1820        let urls = to_search
1821            .iter()
1822            .map(|(url, _expect)| Url::parse(url).unwrap())
1823            .collect::<Vec<_>>();
1824
1825        let visited = get_visited(&conn, urls).unwrap();
1826
1827        assert_eq!(visited.len(), to_search.len());
1828
1829        for (i, &did_see) in visited.iter().enumerate() {
1830            assert_eq!(
1831                did_see,
1832                to_search[i].1,
1833                "Wrong value in get_visited for '{}' (idx {}), want {}, have {}",
1834                to_search[i].0,
1835                i, // idx is logged because some things are repeated
1836                to_search[i].1,
1837                did_see
1838            );
1839        }
1840        Ok(())
1841    }
1842
1843    #[test]
1844    fn test_get_visited_into() {
1845        error_support::init_for_tests();
1846        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
1847
1848        let u0 = Url::parse("https://www.example.com/1").unwrap();
1849        let u1 = Url::parse("https://www.example.com/12").unwrap();
1850        let u2 = Url::parse("https://www.example.com/123").unwrap();
1851        let u3 = Url::parse("https://www.example.com/1234").unwrap();
1852        let u4 = Url::parse("https://www.example.com/12345").unwrap();
1853
1854        let to_add = [(&u0, false), (&u1, false), (&u2, false), (&u3, true)];
1855        for (item, is_remote) in to_add {
1856            apply_observation(
1857                &conn,
1858                VisitObservation::new(item.clone())
1859                    .with_visit_type(VisitType::Link)
1860                    .with_is_remote(is_remote),
1861            )
1862            .unwrap();
1863        }
1864        // Bookmarked, so exists in `moz_places`;
1865        // but doesn't have a last visit time, so shouldn't be visited.
1866        insert_bookmark(
1867            &conn,
1868            crate::InsertableBookmark {
1869                parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
1870                position: crate::BookmarkPosition::Append,
1871                date_added: None,
1872                last_modified: None,
1873                guid: None,
1874                url: u4.clone(),
1875                title: Some("Title".to_string()),
1876            }
1877            .into(),
1878        )
1879        .unwrap();
1880
1881        let mut results = [false; 12];
1882
1883        let get_visited_request = [
1884            // 0 blank
1885            (2, u1.clone()),
1886            (1, u0),
1887            // 3 blank
1888            (4, u2),
1889            // 5 blank
1890            // Note: url for 6 is not visited.
1891            (6, Url::parse("https://www.example.com/123456").unwrap()),
1892            // 7 blank
1893            // Note: dupe is allowed
1894            (8, u1),
1895            // 9 is blank
1896            (10, u3),
1897            (11, u4),
1898        ];
1899
1900        get_visited_into(&conn, &get_visited_request, &mut results).unwrap();
1901        let expect = [
1902            false, // 0
1903            true,  // 1
1904            true,  // 2
1905            false, // 3
1906            true,  // 4
1907            false, // 5
1908            false, // 6
1909            false, // 7
1910            true,  // 8
1911            false, // 9
1912            true,  // 10
1913            false, // 11
1914        ];
1915
1916        assert_eq!(expect, results);
1917    }
1918
1919    #[test]
1920    fn test_delete_visited() {
1921        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1922        let late: Timestamp = SystemTime::now().into();
1923        let early: Timestamp = (SystemTime::now() - Duration::from_secs(30)).into();
1924        let url1 = Url::parse("https://www.example.com/1").unwrap();
1925        let url2 = Url::parse("https://www.example.com/2").unwrap();
1926        let url3 = Url::parse("https://www.example.com/3").unwrap();
1927        let url4 = Url::parse("https://www.example.com/4").unwrap();
1928        // (url, when)
1929        let to_add = [
1930            // 2 visits to "https://www.example.com/1", one early, one late.
1931            (&url1, early),
1932            (&url1, late),
1933            // One to url2, only late.
1934            (&url2, late),
1935            // One to url2, only early.
1936            (&url3, early),
1937            // One to url4, only late - this will have SyncStatus::Normal
1938            (&url4, late),
1939        ];
1940
1941        for &(url, when) in &to_add {
1942            apply_observation(
1943                &conn,
1944                VisitObservation::new(url.clone())
1945                    .with_at(when)
1946                    .with_visit_type(VisitType::Link),
1947            )
1948            .expect("Should apply visit");
1949        }
1950        // Check we added what we think we did.
1951        let pi = fetch_page_info(&conn, &url1)
1952            .expect("should work")
1953            .expect("should get the page");
1954        assert_eq!(pi.page.visit_count_local, 2);
1955
1956        let pi2 = fetch_page_info(&conn, &url2)
1957            .expect("should work")
1958            .expect("should get the page");
1959        assert_eq!(pi2.page.visit_count_local, 1);
1960
1961        let pi3 = fetch_page_info(&conn, &url3)
1962            .expect("should work")
1963            .expect("should get the page");
1964        assert_eq!(pi3.page.visit_count_local, 1);
1965
1966        let pi4 = fetch_page_info(&conn, &url4)
1967            .expect("should work")
1968            .expect("should get the page");
1969        assert_eq!(pi4.page.visit_count_local, 1);
1970
1971        conn.execute_cached(
1972            &format!(
1973                "UPDATE moz_places set sync_status = {}
1974                 WHERE url = 'https://www.example.com/4'",
1975                (SyncStatus::Normal as u8)
1976            ),
1977            [],
1978        )
1979        .expect("should work");
1980
1981        // Delete some.
1982        delete_visits_between(&conn, late, Timestamp::now()).expect("should work");
1983        // should have removed one of the visits to /1
1984        let pi = fetch_page_info(&conn, &url1)
1985            .expect("should work")
1986            .expect("should get the page");
1987        assert_eq!(pi.page.visit_count_local, 1);
1988
1989        // should have removed all the visits to /2
1990        assert!(fetch_page_info(&conn, &url2)
1991            .expect("should work")
1992            .is_none());
1993
1994        // Should still have the 1 visit to /3
1995        let pi3 = fetch_page_info(&conn, &url3)
1996            .expect("should work")
1997            .expect("should get the page");
1998        assert_eq!(pi3.page.visit_count_local, 1);
1999
2000        // should have removed all the visits to /4
2001        assert!(fetch_page_info(&conn, &url4)
2002            .expect("should work")
2003            .is_none());
2004        // should be a tombstone for url4 and no others.
2005        assert_eq!(get_tombstone_count(&conn), 1);
2006        // XXX - test frecency?
2007        // XXX - origins?
2008    }
2009
2010    #[test]
2011    fn test_change_counter() -> Result<()> {
2012        error_support::init_for_tests();
2013        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2014        let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2015        // A new observation with just a title (ie, no visit) should update it.
2016        apply_observation(
2017            &conn,
2018            VisitObservation::new(pi.url.clone()).with_title(Some("new title".into())),
2019        )?;
2020        pi = fetch_page_info(&conn, &pi.url)?
2021            .expect("page should exist")
2022            .page;
2023        assert_eq!(pi.title, "new title");
2024        assert_eq!(pi.preview_image_url, None);
2025        assert_eq!(pi.sync_change_counter, 2);
2026        // An observation with just a preview_image_url should not update it.
2027        apply_observation(
2028            &conn,
2029            VisitObservation::new(pi.url.clone()).with_preview_image_url(Some(
2030                Url::parse("https://www.example.com/preview.png").unwrap(),
2031            )),
2032        )?;
2033        pi = fetch_page_info(&conn, &pi.url)?
2034            .expect("page should exist")
2035            .page;
2036        assert_eq!(pi.title, "new title");
2037        assert_eq!(
2038            pi.preview_image_url,
2039            Some(Url::parse("https://www.example.com/preview.png").expect("parsed"))
2040        );
2041        assert_eq!(pi.sync_change_counter, 2);
2042        Ok(())
2043    }
2044
2045    #[test]
2046    fn test_status_columns() -> Result<()> {
2047        error_support::init_for_tests();
2048        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2049        // A page with "normal" and a change counter.
2050        let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2051        assert_eq!(pi.sync_change_counter, 1);
2052        conn.execute_cached(
2053            "UPDATE moz_places
2054                                   SET frecency = 100
2055                                   WHERE id = :id",
2056            &[(":id", &pi.row_id)],
2057        )?;
2058        // A page with "new" and no change counter.
2059        let mut pi2 = get_observed_page(&mut conn, "http://example.com/2")?;
2060        conn.execute_cached(
2061            "UPDATE moz_places
2062                SET sync_status = :status,
2063                sync_change_counter = 0,
2064                frecency = 50
2065            WHERE id = :id",
2066            &[
2067                (":status", &(SyncStatus::New as u8) as &dyn rusqlite::ToSql),
2068                (":id", &pi2.row_id),
2069            ],
2070        )?;
2071
2072        // A second page with "new", a change counter (which will be ignored
2073        // as we will limit such that this isn't sent) and a low frecency.
2074        let mut pi3 = get_observed_page(&mut conn, "http://example.com/3")?;
2075        conn.execute_cached(
2076            "UPDATE moz_places
2077                SET sync_status = :status,
2078                sync_change_counter = 1,
2079                frecency = 10
2080            WHERE id = :id",
2081            &[
2082                (":status", &(SyncStatus::New as u8) as &dyn ToSql),
2083                (":id", &pi3.row_id),
2084            ],
2085        )?;
2086
2087        let outgoing = fetch_outgoing(&conn, 2, 3)?;
2088        assert_eq!(outgoing.len(), 2, "should have restricted to the limit");
2089        // want pi or pi2 (but order is indeterminate) and this seems simpler than sorting.
2090        assert!(outgoing[0].envelope.id != outgoing[1].envelope.id);
2091        assert!(outgoing[0].envelope.id == pi.guid || outgoing[0].envelope.id == pi2.guid);
2092        assert!(outgoing[1].envelope.id == pi.guid || outgoing[1].envelope.id == pi2.guid);
2093        finish_outgoing(&conn)?;
2094
2095        pi = fetch_page_info(&conn, &pi.url)?
2096            .expect("page should exist")
2097            .page;
2098        assert_eq!(pi.sync_change_counter, 0);
2099        pi2 = fetch_page_info(&conn, &pi2.url)?
2100            .expect("page should exist")
2101            .page;
2102        assert_eq!(pi2.sync_change_counter, 0);
2103        assert_eq!(pi2.sync_status, SyncStatus::Normal);
2104
2105        // pi3 wasn't uploaded, but it should still have been changed to
2106        // Normal and had the change counter reset.
2107        pi3 = fetch_page_info(&conn, &pi3.url)?
2108            .expect("page should exist")
2109            .page;
2110        assert_eq!(pi3.sync_change_counter, 0);
2111        assert_eq!(pi3.sync_status, SyncStatus::Normal);
2112        Ok(())
2113    }
2114
2115    #[test]
2116    fn test_delete_visits_for() -> Result<()> {
2117        use crate::storage::bookmarks::{
2118            self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2119        };
2120
2121        let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2122
2123        struct TestPage {
2124            href: &'static str,
2125            synced: bool,
2126            bookmark_title: Option<&'static str>,
2127            keyword: Option<&'static str>,
2128        }
2129
2130        fn page_has_tombstone(conn: &PlacesDb, guid: &SyncGuid) -> Result<bool> {
2131            let exists = conn
2132                .try_query_one::<bool, _>(
2133                    "SELECT EXISTS(SELECT 1 FROM moz_places_tombstones
2134                                   WHERE guid = :guid)",
2135                    rusqlite::named_params! { ":guid" : guid },
2136                    false,
2137                )?
2138                .unwrap_or_default();
2139            Ok(exists)
2140        }
2141
2142        fn page_has_visit_tombstones(conn: &PlacesDb, page_id: RowId) -> Result<bool> {
2143            let exists = conn
2144                .try_query_one::<bool, _>(
2145                    "SELECT EXISTS(SELECT 1 FROM moz_historyvisit_tombstones
2146                                   WHERE place_id = :page_id)",
2147                    rusqlite::named_params! { ":page_id": page_id },
2148                    false,
2149                )?
2150                .unwrap_or_default();
2151            Ok(exists)
2152        }
2153
2154        let pages = &[
2155            // A is synced and has a bookmark, so we should insert tombstones
2156            // for all its visits.
2157            TestPage {
2158                href: "http://example.com/a",
2159                synced: true,
2160                bookmark_title: Some("A"),
2161                keyword: None,
2162            },
2163            // B is synced but only has visits, so we should insert a tombstone
2164            // for the page.
2165            TestPage {
2166                href: "http://example.com/b",
2167                synced: true,
2168                bookmark_title: None,
2169                keyword: None,
2170            },
2171            // C isn't synced but has a keyword, so we should delete all its
2172            // visits, but not the page.
2173            TestPage {
2174                href: "http://example.com/c",
2175                synced: false,
2176                bookmark_title: None,
2177                keyword: Some("one"),
2178            },
2179            // D isn't synced and only has visits, so we should delete it
2180            // entirely.
2181            TestPage {
2182                href: "http://example.com/d",
2183                synced: false,
2184                bookmark_title: None,
2185                keyword: None,
2186            },
2187        ];
2188        for page in pages {
2189            let url = Url::parse(page.href)?;
2190            let obs = VisitObservation::new(url.clone())
2191                .with_visit_type(VisitType::Link)
2192                .with_at(Some(SystemTime::now().into()));
2193            apply_observation(&db, obs)?;
2194
2195            if page.synced {
2196                db.execute_cached(
2197                    &format!(
2198                        "UPDATE moz_places
2199                             SET sync_status = {}
2200                         WHERE url_hash = hash(:url) AND
2201                               url = :url",
2202                        (SyncStatus::Normal as u8)
2203                    ),
2204                    &[(":url", &url.as_str())],
2205                )?;
2206            }
2207
2208            if let Some(title) = page.bookmark_title {
2209                bookmarks::insert_bookmark(
2210                    &db,
2211                    InsertableBookmark {
2212                        parent_guid: BookmarkRootGuid::Unfiled.into(),
2213                        position: BookmarkPosition::Append,
2214                        date_added: None,
2215                        last_modified: None,
2216                        guid: None,
2217                        url: url.clone(),
2218                        title: Some(title.to_owned()),
2219                    }
2220                    .into(),
2221                )?;
2222            }
2223
2224            if let Some(keyword) = page.keyword {
2225                // We don't have a public API for inserting keywords, so just
2226                // write to the database directly.
2227                db.execute_cached(
2228                    "INSERT INTO moz_keywords(place_id, keyword)
2229                     SELECT id, :keyword
2230                     FROM moz_places
2231                     WHERE url_hash = hash(:url) AND
2232                           url = :url",
2233                    &[(":url", &url.as_str()), (":keyword", &keyword)],
2234                )?;
2235            }
2236
2237            // Now delete all visits.
2238            let (info, _) =
2239                fetch_visits(&db, &url, 0)?.expect("Should return visits for test page");
2240            delete_visits_for(&db, &info.guid)?;
2241
2242            match (
2243                page.synced,
2244                page.bookmark_title.is_some() || page.keyword.is_some(),
2245            ) {
2246                (true, true) => {
2247                    let (_, visits) = fetch_visits(&db, &url, 0)?
2248                        .expect("Shouldn't delete synced page with foreign count");
2249                    assert!(
2250                        visits.is_empty(),
2251                        "Should delete all visits from synced page with foreign count"
2252                    );
2253                    assert!(
2254                        !page_has_tombstone(&db, &info.guid)?,
2255                        "Shouldn't insert tombstone for synced page with foreign count"
2256                    );
2257                    assert!(
2258                        page_has_visit_tombstones(&db, info.row_id)?,
2259                        "Should insert visit tombstones for synced page with foreign count"
2260                    );
2261                }
2262                (true, false) => {
2263                    assert!(
2264                        fetch_visits(&db, &url, 0)?.is_none(),
2265                        "Should delete synced page"
2266                    );
2267                    assert!(
2268                        page_has_tombstone(&db, &info.guid)?,
2269                        "Should insert tombstone for synced page"
2270                    );
2271                    assert!(
2272                        !page_has_visit_tombstones(&db, info.row_id)?,
2273                        "Shouldn't insert visit tombstones for synced page"
2274                    );
2275                }
2276                (false, true) => {
2277                    let (_, visits) = fetch_visits(&db, &url, 0)?
2278                        .expect("Shouldn't delete page with foreign count");
2279                    assert!(
2280                        visits.is_empty(),
2281                        "Should delete all visits from page with foreign count"
2282                    );
2283                    assert!(
2284                        !page_has_tombstone(&db, &info.guid)?,
2285                        "Shouldn't insert tombstone for page with foreign count"
2286                    );
2287                    assert!(
2288                        !page_has_visit_tombstones(&db, info.row_id)?,
2289                        "Shouldn't insert visit tombstones for page with foreign count"
2290                    );
2291                }
2292                (false, false) => {
2293                    assert!(fetch_visits(&db, &url, 0)?.is_none(), "Should delete page");
2294                    assert!(
2295                        !page_has_tombstone(&db, &info.guid)?,
2296                        "Shouldn't insert tombstone for page"
2297                    );
2298                    assert!(
2299                        !page_has_visit_tombstones(&db, info.row_id)?,
2300                        "Shouldn't insert visit tombstones for page"
2301                    );
2302                }
2303            }
2304        }
2305
2306        Ok(())
2307    }
2308
2309    #[test]
2310    fn test_tombstones() -> Result<()> {
2311        error_support::init_for_tests();
2312        let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2313        let url = Url::parse("https://example.com")?;
2314        let obs = VisitObservation::new(url.clone())
2315            .with_visit_type(VisitType::Link)
2316            .with_at(Some(SystemTime::now().into()));
2317        apply_observation(&db, obs)?;
2318        let guid = url_to_guid(&db, &url)?.expect("should exist");
2319
2320        delete_visits_for(&db, &guid)?;
2321
2322        // status was "New", so expect no tombstone.
2323        assert_eq!(get_tombstone_count(&db), 0);
2324
2325        let obs = VisitObservation::new(url.clone())
2326            .with_visit_type(VisitType::Link)
2327            .with_at(Some(SystemTime::now().into()));
2328        apply_observation(&db, obs)?;
2329        let new_guid = url_to_guid(&db, &url)?.expect("should exist");
2330
2331        // Set the status to normal
2332        db.execute_cached(
2333            &format!(
2334                "UPDATE moz_places
2335                    SET sync_status = {}
2336                 WHERE guid = :guid",
2337                (SyncStatus::Normal as u8)
2338            ),
2339            &[(":guid", &new_guid)],
2340        )?;
2341        delete_visits_for(&db, &new_guid)?;
2342        assert_eq!(get_tombstone_count(&db), 1);
2343        Ok(())
2344    }
2345
2346    #[test]
2347    fn test_reset() -> Result<()> {
2348        fn mark_all_as_synced(db: &PlacesDb) -> Result<()> {
2349            db.execute_cached(
2350                &format!(
2351                    "UPDATE moz_places set sync_status = {}",
2352                    (SyncStatus::Normal as u8)
2353                ),
2354                [],
2355            )?;
2356            Ok(())
2357        }
2358
2359        error_support::init_for_tests();
2360        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2361
2362        // Add Sync metadata keys, to ensure they're reset.
2363        put_meta(&conn, GLOBAL_SYNCID_META_KEY, &"syncAAAAAAAA")?;
2364        put_meta(&conn, COLLECTION_SYNCID_META_KEY, &"syncBBBBBBBB")?;
2365        put_meta(&conn, LAST_SYNC_META_KEY, &12345)?;
2366
2367        // Delete everything first, to ensure we keep the high-water mark
2368        // (see #2445 for a discussion about that).
2369        delete_everything(&conn)?;
2370
2371        let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2372        mark_all_as_synced(&conn)?;
2373        pi = fetch_page_info(&conn, &pi.url)?
2374            .expect("page should exist")
2375            .page;
2376        assert_eq!(pi.sync_change_counter, 1);
2377        assert_eq!(pi.sync_status, SyncStatus::Normal);
2378
2379        let sync_ids = CollSyncIds {
2380            global: SyncGuid::random(),
2381            coll: SyncGuid::random(),
2382        };
2383        history_sync::reset(&conn, &EngineSyncAssociation::Connected(sync_ids.clone()))?;
2384
2385        assert_eq!(
2386            get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?,
2387            Some(sync_ids.global)
2388        );
2389        assert_eq!(
2390            get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2391            Some(sync_ids.coll)
2392        );
2393        assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2394        assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2395
2396        pi = fetch_page_info(&conn, &pi.url)?
2397            .expect("page should exist")
2398            .page;
2399        assert_eq!(pi.sync_change_counter, 0);
2400        assert_eq!(pi.sync_status, SyncStatus::New);
2401        // Ensure we are going to do a full re-upload after a reset.
2402        let outgoing = fetch_outgoing(&conn, 100, 100)?;
2403        assert_eq!(outgoing.len(), 1);
2404
2405        mark_all_as_synced(&conn)?;
2406        assert!(fetch_outgoing(&conn, 100, 100)?.is_empty());
2407        // ...
2408
2409        // Now simulate a reset on disconnect, and verify we've removed all Sync
2410        // metadata again.
2411        history_sync::reset(&conn, &EngineSyncAssociation::Disconnected)?;
2412
2413        assert_eq!(get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?, None);
2414        assert_eq!(
2415            get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2416            None
2417        );
2418        assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2419        assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2420
2421        Ok(())
2422    }
2423
2424    #[test]
2425    fn test_fetch_visits() -> Result<()> {
2426        error_support::init_for_tests();
2427        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2428        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2429        assert_eq!(fetch_visits(&conn, &pi.url, 0).unwrap().unwrap().1.len(), 0);
2430        assert_eq!(fetch_visits(&conn, &pi.url, 1).unwrap().unwrap().1.len(), 1);
2431        Ok(())
2432    }
2433
2434    #[test]
2435    fn test_apply_synced_reconciliation() -> Result<()> {
2436        error_support::init_for_tests();
2437        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2438        let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2439        assert_eq!(pi.sync_status, SyncStatus::New);
2440        assert_eq!(pi.sync_change_counter, 1);
2441        apply_synced_reconciliation(&conn, &pi.guid)?;
2442        pi = fetch_page_info(&conn, &pi.url)?
2443            .expect("page should exist")
2444            .page;
2445        assert_eq!(pi.sync_status, SyncStatus::Normal);
2446        assert_eq!(pi.sync_change_counter, 0);
2447        Ok(())
2448    }
2449
2450    #[test]
2451    fn test_apply_synced_deletion_new() -> Result<()> {
2452        error_support::init_for_tests();
2453        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2454        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2455        assert_eq!(pi.sync_status, SyncStatus::New);
2456        apply_synced_deletion(&conn, &pi.guid)?;
2457        assert!(
2458            fetch_page_info(&conn, &pi.url)?.is_none(),
2459            "should have been deleted"
2460        );
2461        assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2462        Ok(())
2463    }
2464
2465    #[test]
2466    fn test_apply_synced_deletion_normal() -> Result<()> {
2467        error_support::init_for_tests();
2468        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2469        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2470        assert_eq!(pi.sync_status, SyncStatus::New);
2471        conn.execute_cached(
2472            &format!(
2473                "UPDATE moz_places set sync_status = {}",
2474                (SyncStatus::Normal as u8)
2475            ),
2476            [],
2477        )?;
2478
2479        apply_synced_deletion(&conn, &pi.guid)?;
2480        assert!(
2481            fetch_page_info(&conn, &pi.url)?.is_none(),
2482            "should have been deleted"
2483        );
2484        assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2485        Ok(())
2486    }
2487
2488    #[test]
2489    fn test_apply_synced_deletions_deletes_visits_but_not_page_if_bookmark_exists() -> Result<()> {
2490        error_support::init_for_tests();
2491        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2492        let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2493        let item = InsertableItem::Bookmark {
2494            b: crate::InsertableBookmark {
2495                parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
2496                position: crate::BookmarkPosition::Append,
2497                date_added: None,
2498                last_modified: None,
2499                guid: None,
2500                url: pi.url.clone(),
2501                title: Some("Title".to_string()),
2502            },
2503        };
2504        insert_bookmark(&conn, item).unwrap();
2505        apply_synced_deletion(&conn, &pi.guid)?;
2506        let page_info =
2507            fetch_page_info(&conn, &pi.url)?.expect("The places entry should have remained");
2508        assert!(
2509            page_info.last_visit_id.is_none(),
2510            "Should have no more visits"
2511        );
2512        Ok(())
2513    }
2514
2515    fn assert_tombstones(c: &PlacesDb, expected: &[(RowId, Timestamp)]) {
2516        let mut expected: Vec<(RowId, Timestamp)> = expected.into();
2517        expected.sort();
2518        let mut tombstones = c
2519            .query_rows_and_then(
2520                "SELECT place_id, visit_date FROM moz_historyvisit_tombstones",
2521                [],
2522                |row| -> Result<_> { Ok((row.get::<_, RowId>(0)?, row.get::<_, Timestamp>(1)?)) },
2523            )
2524            .unwrap();
2525        tombstones.sort();
2526        assert_eq!(expected, tombstones);
2527    }
2528
2529    #[test]
2530    fn test_visit_tombstones() {
2531        use url::Url;
2532        error_support::init_for_tests();
2533        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2534        let now = Timestamp::now();
2535
2536        let urls = &[
2537            Url::parse("http://example.com/1").unwrap(),
2538            Url::parse("http://example.com/2").unwrap(),
2539        ];
2540
2541        let dates = &[
2542            Timestamp(now.0 - 10000),
2543            Timestamp(now.0 - 5000),
2544            Timestamp(now.0),
2545        ];
2546        for url in urls {
2547            for &date in dates {
2548                get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2549            }
2550        }
2551        delete_place_visit_at_time(&conn, &urls[0], dates[1]).unwrap();
2552        // Delete the most recent visit.
2553        delete_visits_between(&conn, Timestamp(now.0 - 4000), Timestamp::now()).unwrap();
2554
2555        let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2556        assert_eq!(
2557            visits0,
2558            &[FetchedVisit {
2559                is_local: true,
2560                visit_date: dates[0],
2561                visit_type: Some(VisitType::Link)
2562            },]
2563        );
2564
2565        assert!(
2566            !visits0.iter().any(|v| v.visit_date == dates[1]),
2567            "Shouldn't have deleted visit"
2568        );
2569
2570        let (info1, mut visits1) = fetch_visits(&conn, &urls[1], 100).unwrap().unwrap();
2571        visits1.sort_by_key(|v| v.visit_date);
2572        // Shouldn't have most recent visit, but should still have the dates[1]
2573        // visit, which should be uneffected.
2574        assert_eq!(
2575            visits1,
2576            &[
2577                FetchedVisit {
2578                    is_local: true,
2579                    visit_date: dates[0],
2580                    visit_type: Some(VisitType::Link)
2581                },
2582                FetchedVisit {
2583                    is_local: true,
2584                    visit_date: dates[1],
2585                    visit_type: Some(VisitType::Link)
2586                },
2587            ]
2588        );
2589
2590        // Make sure syncing doesn't resurrect them.
2591        apply_synced_visits(
2592            &conn,
2593            &info0.guid,
2594            &info0.url,
2595            &Some(info0.title.clone()),
2596            // Ignore dates[0] since we know it's present.
2597            &dates
2598                .iter()
2599                .map(|&d| HistoryRecordVisit {
2600                    date: d.into(),
2601                    transition: VisitType::Link as u8,
2602                    unknown_fields: UnknownFields::new(),
2603                })
2604                .collect::<Vec<_>>(),
2605            &UnknownFields::new(),
2606        )
2607        .unwrap();
2608
2609        let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2610        assert_eq!(
2611            visits0,
2612            &[FetchedVisit {
2613                is_local: true,
2614                visit_date: dates[0],
2615                visit_type: Some(VisitType::Link)
2616            }]
2617        );
2618
2619        assert_tombstones(
2620            &conn,
2621            &[
2622                (info0.row_id, dates[1]),
2623                (info0.row_id, dates[2]),
2624                (info1.row_id, dates[2]),
2625            ],
2626        );
2627
2628        // Delete the last visit from info0. This should delete the page entirely,
2629        // as well as it's tomebstones.
2630        delete_place_visit_at_time(&conn, &urls[0], dates[0]).unwrap();
2631
2632        assert!(fetch_visits(&conn, &urls[0], 100).unwrap().is_none());
2633
2634        assert_tombstones(&conn, &[(info1.row_id, dates[2])]);
2635    }
2636
2637    #[test]
2638    fn test_delete_local() {
2639        use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
2640        use crate::storage::bookmarks::{
2641            self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark, InsertableItem,
2642        };
2643        use url::Url;
2644        error_support::init_for_tests();
2645        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2646        let ts = Timestamp::now().0 - 5_000_000;
2647        // Add a number of visits across a handful of origins.
2648        for o in 0..10 {
2649            for i in 0..11 {
2650                for t in 0..3 {
2651                    get_custom_observed_page(
2652                        &mut conn,
2653                        &format!("http://www.example{}.com/{}", o, i),
2654                        |obs| obs.with_at(Timestamp(ts + t * 1000 + i * 10_000 + o * 100_000)),
2655                    )
2656                    .unwrap();
2657                }
2658            }
2659        }
2660        // Add some bookmarks.
2661        let b0 = (
2662            SyncGuid::from("aaaaaaaaaaaa"),
2663            Url::parse("http://www.example3.com/5").unwrap(),
2664        );
2665        let b1 = (
2666            SyncGuid::from("bbbbbbbbbbbb"),
2667            Url::parse("http://www.example6.com/10").unwrap(),
2668        );
2669        let b2 = (
2670            SyncGuid::from("cccccccccccc"),
2671            Url::parse("http://www.example9.com/4").unwrap(),
2672        );
2673        for (guid, url) in &[&b0, &b1, &b2] {
2674            bookmarks::insert_bookmark(
2675                &conn,
2676                InsertableItem::Bookmark {
2677                    b: InsertableBookmark {
2678                        parent_guid: BookmarkRootGuid::Unfiled.into(),
2679                        position: BookmarkPosition::Append,
2680                        date_added: None,
2681                        last_modified: None,
2682                        guid: Some(guid.clone()),
2683                        url: url.clone(),
2684                        title: None,
2685                    },
2686                },
2687            )
2688            .unwrap();
2689        }
2690
2691        // Make sure tombstone insertions stick.
2692        conn.execute_all(&[
2693            &format!(
2694                "UPDATE moz_places set sync_status = {}",
2695                (SyncStatus::Normal as u8)
2696            ),
2697            &format!(
2698                "UPDATE moz_bookmarks set syncStatus = {}",
2699                (SyncStatus::Normal as u8)
2700            ),
2701        ])
2702        .unwrap();
2703
2704        // Ensure some various tombstones exist
2705        delete_visits_for(
2706            &conn,
2707            &url_to_guid(&conn, &Url::parse("http://www.example8.com/5").unwrap())
2708                .unwrap()
2709                .unwrap(),
2710        )
2711        .unwrap();
2712
2713        delete_place_visit_at_time(
2714            &conn,
2715            &Url::parse("http://www.example10.com/5").unwrap(),
2716            Timestamp(ts + 5 * 10_000 + 10 * 100_000),
2717        )
2718        .unwrap();
2719
2720        assert!(bookmarks::delete_bookmark(&conn, &b0.0).unwrap());
2721
2722        delete_everything(&conn).unwrap();
2723
2724        let places = conn
2725            .query_rows_and_then(
2726                "SELECT * FROM moz_places ORDER BY url ASC",
2727                [],
2728                PageInfo::from_row,
2729            )
2730            .unwrap();
2731        assert_eq!(places.len(), 2);
2732        assert_eq!(places[0].url, b1.1);
2733        assert_eq!(places[1].url, b2.1);
2734        for p in &places {
2735            assert_eq!(
2736                p.frecency,
2737                DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
2738            );
2739            assert_eq!(p.visit_count_local, 0);
2740            assert_eq!(p.visit_count_remote, 0);
2741            assert_eq!(p.last_visit_date_local, Timestamp(0));
2742            assert_eq!(p.last_visit_date_remote, Timestamp(0));
2743        }
2744
2745        let counts_sql = [
2746            (0i64, "SELECT COUNT(*) FROM moz_historyvisits"),
2747            (2, "SELECT COUNT(*) FROM moz_origins"),
2748            (7, "SELECT COUNT(*) FROM moz_bookmarks"), // the two we added + 5 roots
2749            (1, "SELECT COUNT(*) FROM moz_bookmarks_deleted"),
2750            (0, "SELECT COUNT(*) FROM moz_historyvisit_tombstones"),
2751            (0, "SELECT COUNT(*) FROM moz_places_tombstones"),
2752        ];
2753        for (want, query) in &counts_sql {
2754            assert_eq!(
2755                *want,
2756                conn.query_one::<i64>(query).unwrap(),
2757                "Unexpected value for {}",
2758                query
2759            );
2760        }
2761    }
2762
2763    #[test]
2764    fn test_delete_everything() {
2765        use crate::storage::bookmarks::{
2766            self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2767        };
2768        use url::Url;
2769        error_support::init_for_tests();
2770        let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2771        let start = Timestamp::now();
2772
2773        let urls = &[
2774            Url::parse("http://example.com/1").unwrap(),
2775            Url::parse("http://example.com/2").unwrap(),
2776            Url::parse("http://example.com/3").unwrap(),
2777        ];
2778
2779        let dates = &[
2780            Timestamp(start.0 - 10000),
2781            Timestamp(start.0 - 5000),
2782            Timestamp(start.0),
2783        ];
2784
2785        for url in urls {
2786            for &date in dates {
2787                get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2788            }
2789        }
2790
2791        bookmarks::insert_bookmark(
2792            &conn,
2793            InsertableBookmark {
2794                parent_guid: BookmarkRootGuid::Unfiled.into(),
2795                position: BookmarkPosition::Append,
2796                date_added: None,
2797                last_modified: None,
2798                guid: Some("bookmarkAAAA".into()),
2799                url: urls[2].clone(),
2800                title: Some("A".into()),
2801            }
2802            .into(),
2803        )
2804        .expect("Should insert bookmark with URL 3");
2805
2806        conn.execute(
2807            "WITH entries(url, input) AS (
2808               VALUES(:url1, 'hi'), (:url3, 'bye')
2809             )
2810             INSERT INTO moz_inputhistory(place_id, input, use_count)
2811             SELECT h.id, e.input, 1
2812             FROM entries e
2813             JOIN moz_places h ON h.url_hash = hash(e.url) AND
2814                                  h.url = e.url",
2815            &[(":url1", &urls[1].as_str()), (":url3", &urls[2].as_str())],
2816        )
2817        .expect("Should insert autocomplete history entries");
2818
2819        delete_everything(&conn).expect("Should delete everything except URL 3");
2820
2821        std::thread::sleep(std::time::Duration::from_millis(50));
2822
2823        // Should leave bookmarked URLs alone, and keep autocomplete history for
2824        // those URLs.
2825        let mut places_stmt = conn.prepare("SELECT url FROM moz_places").unwrap();
2826        let remaining_urls: Vec<String> = places_stmt
2827            .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2828            .expect("Should fetch remaining URLs")
2829            .map(std::result::Result::unwrap)
2830            .collect();
2831        assert_eq!(remaining_urls, &["http://example.com/3"]);
2832
2833        let mut input_stmt = conn.prepare("SELECT input FROM moz_inputhistory").unwrap();
2834        let remaining_inputs: Vec<String> = input_stmt
2835            .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2836            .expect("Should fetch remaining autocomplete history entries")
2837            .map(std::result::Result::unwrap)
2838            .collect();
2839        assert_eq!(remaining_inputs, &["bye"]);
2840
2841        bookmarks::delete_bookmark(&conn, &"bookmarkAAAA".into())
2842            .expect("Should delete bookmark with URL 3");
2843
2844        delete_everything(&conn).expect("Should delete all URLs");
2845
2846        assert_eq!(
2847            0,
2848            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2849                .unwrap(),
2850        );
2851
2852        apply_synced_visits(
2853            &conn,
2854            &SyncGuid::random(),
2855            &url::Url::parse("http://www.example.com/123").unwrap(),
2856            &None,
2857            &[
2858                HistoryRecordVisit {
2859                    // This should make it in
2860                    date: Timestamp::now().into(),
2861                    transition: VisitType::Link as u8,
2862                    unknown_fields: UnknownFields::new(),
2863                },
2864                HistoryRecordVisit {
2865                    // This should not.
2866                    date: start.into(),
2867                    transition: VisitType::Link as u8,
2868                    unknown_fields: UnknownFields::new(),
2869                },
2870            ],
2871            &UnknownFields::new(),
2872        )
2873        .unwrap();
2874        assert_eq!(
2875            1,
2876            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2877                .unwrap(),
2878        );
2879        // Only one visit should be applied.
2880        assert_eq!(
2881            1,
2882            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2883                .unwrap(),
2884        );
2885
2886        // Check that we don't insert a place if all visits are too old.
2887        apply_synced_visits(
2888            &conn,
2889            &SyncGuid::random(),
2890            &url::Url::parse("http://www.example.com/1234").unwrap(),
2891            &None,
2892            &[HistoryRecordVisit {
2893                date: start.into(),
2894                transition: VisitType::Link as u8,
2895                unknown_fields: UnknownFields::new(),
2896            }],
2897            &UnknownFields::new(),
2898        )
2899        .unwrap();
2900        // unchanged.
2901        assert_eq!(
2902            1,
2903            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2904                .unwrap(),
2905        );
2906        assert_eq!(
2907            1,
2908            conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2909                .unwrap(),
2910        );
2911    }
2912
2913    // See https://github.com/mozilla-mobile/fenix/issues/8531#issuecomment-590498878.
2914    #[test]
2915    fn test_delete_everything_deletes_origins() {
2916        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2917
2918        let u = Url::parse("https://www.reddit.com/r/climbing").expect("Should parse URL");
2919        let ts = Timestamp::now().0 - 5_000_000;
2920        let obs = VisitObservation::new(u)
2921            .with_visit_type(VisitType::Link)
2922            .with_at(Timestamp(ts));
2923        apply_observation(&conn, obs).expect("Should apply observation");
2924
2925        delete_everything(&conn).expect("Should delete everything");
2926
2927        // We should clear all origins after deleting everything.
2928        let origin_count = conn
2929            .query_one::<i64>("SELECT COUNT(*) FROM moz_origins")
2930            .expect("Should fetch origin count");
2931        assert_eq!(0, origin_count);
2932    }
2933
2934    #[test]
2935    fn test_apply_observation_updates_origins() {
2936        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2937
2938        let obs_for_a = VisitObservation::new(
2939            Url::parse("https://example1.com/a").expect("Should parse URL A"),
2940        )
2941        .with_visit_type(VisitType::Link)
2942        .with_at(Timestamp(Timestamp::now().0 - 5_000_000));
2943        apply_observation(&conn, obs_for_a).expect("Should apply observation for A");
2944
2945        let obs_for_b = VisitObservation::new(
2946            Url::parse("https://example2.com/b").expect("Should parse URL B"),
2947        )
2948        .with_visit_type(VisitType::Link)
2949        .with_at(Timestamp(Timestamp::now().0 - 2_500_000));
2950        apply_observation(&conn, obs_for_b).expect("Should apply observation for B");
2951
2952        let mut origins = conn
2953            .prepare("SELECT host FROM moz_origins")
2954            .expect("Should prepare origins statement")
2955            .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2956            .expect("Should fetch all origins")
2957            .map(|r| r.expect("Should get origin from row"))
2958            .collect::<Vec<_>>();
2959        origins.sort();
2960        assert_eq!(origins, &["example1.com", "example2.com",]);
2961    }
2962
2963    #[test]
2964    fn test_preview_url() {
2965        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2966
2967        let url1 = Url::parse("https://www.example.com/").unwrap();
2968        // Can observe preview url without an associated visit.
2969        assert!(apply_observation(
2970            &conn,
2971            VisitObservation::new(url1.clone()).with_preview_image_url(Some(
2972                Url::parse("https://www.example.com/image.png").unwrap()
2973            ))
2974        )
2975        .unwrap()
2976        .is_none());
2977
2978        // We don't get a visit id back above, so just assume an id of the corresponding moz_places entry.
2979        let mut db_preview_url = conn
2980            .query_row_and_then_cachable(
2981                "SELECT preview_image_url FROM moz_places WHERE id = 1",
2982                [],
2983                |row| row.get(0),
2984                false,
2985            )
2986            .unwrap();
2987        assert_eq!(
2988            Some("https://www.example.com/image.png".to_string()),
2989            db_preview_url
2990        );
2991
2992        // Observing a visit afterwards doesn't erase a preview url.
2993        let visit_id = apply_observation(
2994            &conn,
2995            VisitObservation::new(url1).with_visit_type(VisitType::Link),
2996        )
2997        .unwrap();
2998        assert!(visit_id.is_some());
2999
3000        db_preview_url = conn
3001            .query_row_and_then_cachable(
3002                "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3003                &[(":id", &visit_id.unwrap() as &dyn ToSql)],
3004                |row| row.get(0),
3005                false,
3006            )
3007            .unwrap();
3008        assert_eq!(
3009            Some("https://www.example.com/image.png".to_string()),
3010            db_preview_url
3011        );
3012
3013        // Can observe a preview image url as part of a visit observation.
3014        let another_visit_id = apply_observation(
3015            &conn,
3016            VisitObservation::new(Url::parse("https://www.example.com/another/").unwrap())
3017                .with_preview_image_url(Some(
3018                    Url::parse("https://www.example.com/funky/image.png").unwrap(),
3019                ))
3020                .with_visit_type(VisitType::Link),
3021        )
3022        .unwrap();
3023        assert!(another_visit_id.is_some());
3024
3025        db_preview_url = conn
3026            .query_row_and_then_cachable(
3027                "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3028                &[(":id", &another_visit_id.unwrap())],
3029                |row| row.get(0),
3030                false,
3031            )
3032            .unwrap();
3033        assert_eq!(
3034            Some("https://www.example.com/funky/image.png".to_string()),
3035            db_preview_url
3036        );
3037    }
3038
3039    #[test]
3040    fn test_long_strings() {
3041        error_support::init_for_tests();
3042        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3043        let mut url = "http://www.example.com".to_string();
3044        while url.len() < crate::storage::URL_LENGTH_MAX {
3045            url += "/garbage";
3046        }
3047        let maybe_row = apply_observation(
3048            &conn,
3049            VisitObservation::new(Url::parse(&url).unwrap())
3050                .with_visit_type(VisitType::Link)
3051                .with_at(Timestamp::now()),
3052        )
3053        .unwrap();
3054        assert!(maybe_row.is_none(), "Shouldn't insert overlong URL");
3055
3056        let maybe_row_preview = apply_observation(
3057            &conn,
3058            VisitObservation::new(Url::parse("https://www.example.com/").unwrap())
3059                .with_visit_type(VisitType::Link)
3060                .with_preview_image_url(Url::parse(&url).unwrap()),
3061        )
3062        .unwrap();
3063        assert!(
3064            maybe_row_preview.is_some(),
3065            "Shouldn't avoid a visit observation due to an overly long preview url"
3066        );
3067
3068        let mut title = "example 1 2 3".to_string();
3069        // Make sure whatever we use here surpasses the length.
3070        while title.len() < crate::storage::TITLE_LENGTH_MAX + 10 {
3071            title += " test test";
3072        }
3073        let maybe_visit_row = apply_observation(
3074            &conn,
3075            VisitObservation::new(Url::parse("http://www.example.com/123").unwrap())
3076                .with_title(title.clone())
3077                .with_visit_type(VisitType::Link)
3078                .with_at(Timestamp::now()),
3079        )
3080        .unwrap();
3081
3082        assert!(maybe_visit_row.is_some());
3083        let db_title: String = conn
3084            .query_row_and_then_cachable(
3085                "SELECT h.title FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3086                &[(":id", &maybe_visit_row.unwrap())],
3087                |row| row.get(0),
3088                false,
3089            )
3090            .unwrap();
3091        // Ensure what we get back the trimmed title.
3092        assert_eq!(db_title.len(), crate::storage::TITLE_LENGTH_MAX);
3093        assert!(title.starts_with(&db_title));
3094    }
3095
3096    #[test]
3097    fn test_get_visit_page_with_bound() {
3098        use std::time::SystemTime;
3099        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3100        let now: Timestamp = SystemTime::now().into();
3101        let now_u64 = now.0;
3102        let now_i64 = now.0 as i64;
3103        // (url, title, when, is_remote, (expected_always, expected_only_local)
3104        let to_add = [
3105            (
3106                "https://www.example.com/0",
3107                "older 2",
3108                now_u64 - 200_200,
3109                false,
3110                (true, false),
3111            ),
3112            (
3113                "https://www.example.com/1",
3114                "older 1",
3115                now_u64 - 200_100,
3116                true,
3117                (true, false),
3118            ),
3119            (
3120                "https://www.example.com/2",
3121                "same time",
3122                now_u64 - 200_000,
3123                false,
3124                (true, false),
3125            ),
3126            (
3127                "https://www.example.com/3",
3128                "same time",
3129                now_u64 - 200_000,
3130                false,
3131                (true, false),
3132            ),
3133            (
3134                "https://www.example.com/4",
3135                "same time",
3136                now_u64 - 200_000,
3137                false,
3138                (true, false),
3139            ),
3140            (
3141                "https://www.example.com/5",
3142                "same time",
3143                now_u64 - 200_000,
3144                false,
3145                (true, false),
3146            ),
3147            (
3148                "https://www.example.com/6",
3149                "same time",
3150                now_u64 - 200_000,
3151                false,
3152                (true, false),
3153            ),
3154            (
3155                "https://www.example.com/7",
3156                "same time",
3157                now_u64 - 200_000,
3158                false,
3159                (true, false),
3160            ),
3161            (
3162                "https://www.example.com/8",
3163                "same time",
3164                now_u64 - 200_000,
3165                false,
3166                (true, false),
3167            ),
3168            (
3169                "https://www.example.com/9",
3170                "same time",
3171                now_u64 - 200_000,
3172                false,
3173                (true, false),
3174            ),
3175            (
3176                "https://www.example.com/10",
3177                "more recent 2",
3178                now_u64 - 199_000,
3179                false,
3180                (true, false),
3181            ),
3182            (
3183                "https://www.example.com/11",
3184                "more recent 1",
3185                now_u64 - 198_000,
3186                false,
3187                (true, false),
3188            ),
3189        ];
3190
3191        for &(url, title, when, remote, _) in &to_add {
3192            apply_observation(
3193                &conn,
3194                VisitObservation::new(Url::parse(url).unwrap())
3195                    .with_title(title.to_owned())
3196                    .with_at(Timestamp(when))
3197                    .with_is_remote(remote)
3198                    .with_visit_type(VisitType::Link),
3199            )
3200            .expect("Should apply visit");
3201        }
3202
3203        // test when offset fall on a point where visited_date changes
3204        let infos_with_bound =
3205            get_visit_page_with_bound(&conn, now_i64 - 200_000, 8, 2, VisitTransitionSet::empty())
3206                .unwrap();
3207        let infos = infos_with_bound.infos;
3208        assert_eq!(infos[0].title.as_ref().unwrap().as_str(), "older 1",);
3209        assert!(infos[0].is_remote); // "older 1" is remote
3210        assert_eq!(infos[1].title.as_ref().unwrap().as_str(), "older 2",);
3211        assert!(!infos[1].is_remote); // "older 2" is local
3212        assert_eq!(infos_with_bound.bound, now_i64 - 200_200,);
3213        assert_eq!(infos_with_bound.offset, 1,);
3214
3215        // test when offset fall on one item before visited_date changes
3216        let infos_with_bound =
3217            get_visit_page_with_bound(&conn, now_i64 - 200_000, 7, 1, VisitTransitionSet::empty())
3218                .unwrap();
3219        assert_eq!(
3220            infos_with_bound.infos[0].url,
3221            Url::parse("https://www.example.com/9").unwrap(),
3222        );
3223
3224        // test when offset fall on one item after visited_date changes
3225        let infos_with_bound =
3226            get_visit_page_with_bound(&conn, now_i64 - 200_000, 9, 1, VisitTransitionSet::empty())
3227                .unwrap();
3228        assert_eq!(
3229            infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3230            "older 2",
3231        );
3232
3233        // with a small page length, loop through items that have the same visited date
3234        let count = 2;
3235        let mut bound = now_i64 - 199_000;
3236        let mut offset = 1;
3237        for _i in 0..4 {
3238            let infos_with_bound =
3239                get_visit_page_with_bound(&conn, bound, offset, count, VisitTransitionSet::empty())
3240                    .unwrap();
3241            assert_eq!(
3242                infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3243                "same time",
3244            );
3245            assert_eq!(
3246                infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3247                "same time",
3248            );
3249            bound = infos_with_bound.bound;
3250            offset = infos_with_bound.offset;
3251        }
3252        // bound and offset should have skipped the 8 items that have the same visited date
3253        assert_eq!(bound, now_i64 - 200_000,);
3254        assert_eq!(offset, 8,);
3255
3256        // when bound is now and offset is zero
3257        let infos_with_bound =
3258            get_visit_page_with_bound(&conn, now_i64, 0, 2, VisitTransitionSet::empty()).unwrap();
3259        assert_eq!(
3260            infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3261            "more recent 1",
3262        );
3263        assert_eq!(
3264            infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3265            "more recent 2",
3266        );
3267        assert_eq!(infos_with_bound.bound, now_i64 - 199_000);
3268        assert_eq!(infos_with_bound.offset, 1);
3269    }
3270
3271    /// Test find_normal_visits_to_prune
3272    #[test]
3273    fn test_normal_visit_pruning() {
3274        use std::time::{Duration, SystemTime};
3275        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3276        let one_day = Duration::from_secs(60 * 60 * 24);
3277        let now: Timestamp = SystemTime::now().into();
3278        let url = Url::parse("https://mozilla.com/").unwrap();
3279
3280        // Create 1 visit per day for the last 30 days
3281        let mut visits: Vec<_> = (0..30)
3282            .map(|i| {
3283                apply_observation(
3284                    &conn,
3285                    VisitObservation::new(url.clone())
3286                        .with_at(now.checked_sub(one_day * i))
3287                        .with_visit_type(VisitType::Link),
3288                )
3289                .unwrap()
3290                .unwrap()
3291            })
3292            .collect();
3293        // Reverse visits so that they're oldest first
3294        visits.reverse();
3295
3296        check_visits_to_prune(
3297            &conn,
3298            find_normal_visits_to_prune(&conn, 4, now).unwrap(),
3299            &visits[..4],
3300        );
3301
3302        // Only visits older than 7 days should be pruned
3303        check_visits_to_prune(
3304            &conn,
3305            find_normal_visits_to_prune(&conn, 30, now).unwrap(),
3306            &visits[..22],
3307        );
3308    }
3309
3310    /// Test find_exotic_visits_to_prune
3311    #[test]
3312    fn test_exotic_visit_pruning() {
3313        use std::time::{Duration, SystemTime};
3314        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3315        let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3316        let now: Timestamp = SystemTime::now().into();
3317        let short_url = Url::parse("https://mozilla.com/").unwrap();
3318        let long_url = Url::parse(&format!(
3319            "https://mozilla.com/{}",
3320            (0..255).map(|_| "x").collect::<String>()
3321        ))
3322        .unwrap();
3323
3324        let visit_with_long_url = apply_observation(
3325            &conn,
3326            VisitObservation::new(long_url.clone())
3327                .with_at(now.checked_sub(one_month * 2))
3328                .with_visit_type(VisitType::Link),
3329        )
3330        .unwrap()
3331        .unwrap();
3332
3333        let visit_for_download = apply_observation(
3334            &conn,
3335            VisitObservation::new(short_url)
3336                .with_at(now.checked_sub(one_month * 3))
3337                .with_visit_type(VisitType::Download),
3338        )
3339        .unwrap()
3340        .unwrap();
3341
3342        // This visit should not be pruned, since it's too recent
3343        apply_observation(
3344            &conn,
3345            VisitObservation::new(long_url)
3346                .with_at(now.checked_sub(one_month))
3347                .with_visit_type(VisitType::Download),
3348        )
3349        .unwrap()
3350        .unwrap();
3351
3352        check_visits_to_prune(
3353            &conn,
3354            find_exotic_visits_to_prune(&conn, 2, now).unwrap(),
3355            &[visit_for_download, visit_with_long_url],
3356        );
3357
3358        // With limit = 1, it should pick the oldest visit
3359        check_visits_to_prune(
3360            &conn,
3361            find_exotic_visits_to_prune(&conn, 1, now).unwrap(),
3362            &[visit_for_download],
3363        );
3364
3365        // If the limit exceeds the number of candidates, it should return as many as it can find
3366        check_visits_to_prune(
3367            &conn,
3368            find_exotic_visits_to_prune(&conn, 3, now).unwrap(),
3369            &[visit_for_download, visit_with_long_url],
3370        );
3371    }
3372    /// Test that find_visits_to_prune correctly combines find_exotic_visits_to_prune and
3373    /// find_normal_visits_to_prune
3374    #[test]
3375    fn test_visit_pruning() {
3376        use std::time::{Duration, SystemTime};
3377        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3378        let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3379        let now: Timestamp = SystemTime::now().into();
3380        let short_url = Url::parse("https://mozilla.com/").unwrap();
3381        let long_url = Url::parse(&format!(
3382            "https://mozilla.com/{}",
3383            (0..255).map(|_| "x").collect::<String>()
3384        ))
3385        .unwrap();
3386
3387        // An exotic visit that should be pruned first, even if it's not the oldest
3388        let excotic_visit = apply_observation(
3389            &conn,
3390            VisitObservation::new(long_url)
3391                .with_at(now.checked_sub(one_month * 3))
3392                .with_visit_type(VisitType::Link),
3393        )
3394        .unwrap()
3395        .unwrap();
3396
3397        // Normal visits that should be pruned after excotic visits
3398        let old_visit = apply_observation(
3399            &conn,
3400            VisitObservation::new(short_url.clone())
3401                .with_at(now.checked_sub(one_month * 4))
3402                .with_visit_type(VisitType::Link),
3403        )
3404        .unwrap()
3405        .unwrap();
3406        let really_old_visit = apply_observation(
3407            &conn,
3408            VisitObservation::new(short_url.clone())
3409                .with_at(now.checked_sub(one_month * 12))
3410                .with_visit_type(VisitType::Link),
3411        )
3412        .unwrap()
3413        .unwrap();
3414
3415        // Newer visit that's too new to be pruned
3416        apply_observation(
3417            &conn,
3418            VisitObservation::new(short_url)
3419                .with_at(now.checked_sub(Duration::from_secs(100)))
3420                .with_visit_type(VisitType::Link),
3421        )
3422        .unwrap()
3423        .unwrap();
3424
3425        check_visits_to_prune(
3426            &conn,
3427            find_visits_to_prune(&conn, 2, now).unwrap(),
3428            &[excotic_visit, really_old_visit],
3429        );
3430
3431        check_visits_to_prune(
3432            &conn,
3433            find_visits_to_prune(&conn, 10, now).unwrap(),
3434            &[excotic_visit, really_old_visit, old_visit],
3435        );
3436    }
3437
3438    fn check_visits_to_prune(
3439        db: &PlacesDb,
3440        visits_to_delete: Vec<VisitToDelete>,
3441        correct_visits: &[RowId],
3442    ) {
3443        assert_eq!(
3444            correct_visits.iter().collect::<HashSet<_>>(),
3445            visits_to_delete
3446                .iter()
3447                .map(|v| &v.visit_id)
3448                .collect::<HashSet<_>>()
3449        );
3450
3451        let correct_place_ids: HashSet<RowId> = correct_visits
3452            .iter()
3453            .map(|vid| {
3454                db.query_one(&format!(
3455                    "SELECT v.place_id FROM moz_historyvisits v WHERE v.id = {}",
3456                    vid
3457                ))
3458                .unwrap()
3459            })
3460            .collect();
3461        assert_eq!(
3462            correct_place_ids,
3463            visits_to_delete
3464                .iter()
3465                .map(|v| v.page_id)
3466                .collect::<HashSet<_>>()
3467        );
3468    }
3469
3470    #[test]
3471    fn test_get_visit_count_for_host() {
3472        error_support::init_for_tests();
3473        let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3474        let start_timestamp = Timestamp::now();
3475        let to_add = [
3476            (
3477                "http://example.com/0",
3478                start_timestamp.0 - 200_200,
3479                VisitType::Link,
3480            ),
3481            (
3482                "http://example.com/1",
3483                start_timestamp.0 - 200_100,
3484                VisitType::Link,
3485            ),
3486            (
3487                "https://example.com/0",
3488                start_timestamp.0 - 200_000,
3489                VisitType::Link,
3490            ),
3491            (
3492                "https://example1.com/0",
3493                start_timestamp.0 - 100_600,
3494                VisitType::Link,
3495            ),
3496            (
3497                "https://example1.com/0",
3498                start_timestamp.0 - 100_500,
3499                VisitType::Reload,
3500            ),
3501            (
3502                "https://example1.com/1",
3503                start_timestamp.0 - 100_400,
3504                VisitType::Link,
3505            ),
3506            (
3507                "https://example.com/2",
3508                start_timestamp.0 - 100_300,
3509                VisitType::Link,
3510            ),
3511            (
3512                "https://example.com/1",
3513                start_timestamp.0 - 100_200,
3514                VisitType::Link,
3515            ),
3516            (
3517                "https://example.com/0",
3518                start_timestamp.0 - 100_100,
3519                VisitType::Link,
3520            ),
3521        ];
3522
3523        for &(url, when, visit_type) in &to_add {
3524            apply_observation(
3525                &conn,
3526                VisitObservation::new(Url::parse(url).unwrap())
3527                    .with_at(Timestamp(when))
3528                    .with_visit_type(visit_type),
3529            )
3530            .unwrap()
3531            .unwrap();
3532        }
3533
3534        assert_eq!(
3535            get_visit_count_for_host(
3536                &conn,
3537                "example.com",
3538                Timestamp(start_timestamp.0 - 100_000),
3539                VisitTransitionSet::for_specific(&[])
3540            )
3541            .unwrap(),
3542            6
3543        );
3544        assert_eq!(
3545            get_visit_count_for_host(
3546                &conn,
3547                "example1.com",
3548                Timestamp(start_timestamp.0 - 100_000),
3549                VisitTransitionSet::for_specific(&[])
3550            )
3551            .unwrap(),
3552            3
3553        );
3554        assert_eq!(
3555            get_visit_count_for_host(
3556                &conn,
3557                "example.com",
3558                Timestamp(start_timestamp.0 - 200_000),
3559                VisitTransitionSet::for_specific(&[])
3560            )
3561            .unwrap(),
3562            2
3563        );
3564        assert_eq!(
3565            get_visit_count_for_host(
3566                &conn,
3567                "example1.com",
3568                Timestamp(start_timestamp.0 - 100_500),
3569                VisitTransitionSet::for_specific(&[])
3570            )
3571            .unwrap(),
3572            1
3573        );
3574        assert_eq!(
3575            get_visit_count_for_host(
3576                &conn,
3577                "example1.com",
3578                Timestamp(start_timestamp.0 - 100_000),
3579                VisitTransitionSet::for_specific(&[VisitType::Reload])
3580            )
3581            .unwrap(),
3582            2
3583        );
3584    }
3585}