1mod actions;
6
7use super::{fetch_page_info, new_page_info, PageInfo, RowId};
8use crate::db::PlacesDb;
9use crate::error::{debug, trace, warn, Result};
10use crate::ffi::{HistoryVisitInfo, HistoryVisitInfosWithBound, TopFrecentSiteInfo};
11use crate::frecency;
12use crate::hash;
13use crate::history_sync::engine::{
14 COLLECTION_SYNCID_META_KEY, GLOBAL_SYNCID_META_KEY, LAST_SYNC_META_KEY,
15};
16use crate::observation::VisitObservation;
17use crate::storage::{
18 delete_meta, delete_pending_temp_tables, get_meta, history_metadata, put_meta,
19};
20use crate::types::{
21 serialize_unknown_fields, SyncStatus, UnknownFields, VisitTransitionSet, VisitType,
22};
23use actions::*;
24use rusqlite::types::ToSql;
25use rusqlite::Result as RusqliteResult;
26use rusqlite::Row;
27use sql_support::{self, ConnExt};
28use std::collections::HashSet;
29use std::time::Duration;
30use sync15::bso::OutgoingBso;
31use sync15::engine::EngineSyncAssociation;
32use sync_guid::Guid as SyncGuid;
33use types::Timestamp;
34use url::Url;
35
36static DELETION_HIGH_WATER_MARK_META_KEY: &str = "history_deleted_hwm";
43
44pub fn apply_observation(db: &PlacesDb, visit_ob: VisitObservation) -> Result<Option<RowId>> {
46 let tx = db.begin_transaction()?;
47 let result = apply_observation_direct(db, visit_ob)?;
48 delete_pending_temp_tables(db)?;
49 tx.commit()?;
50 Ok(result)
51}
52
53pub fn apply_observation_direct(
55 db: &PlacesDb,
56 visit_ob: VisitObservation,
57) -> Result<Option<RowId>> {
58 if visit_ob.url.as_str().len() > super::URL_LENGTH_MAX {
60 return Ok(None);
61 }
62 let preview_image_url = if let Some(ref piu) = visit_ob.preview_image_url {
66 if piu.as_str().len() > super::URL_LENGTH_MAX {
67 None
68 } else {
69 Some(piu.clone())
70 }
71 } else {
72 None
73 };
74 let mut page_info = match fetch_page_info(db, &visit_ob.url)? {
75 Some(info) => info.page,
76 None => new_page_info(db, &visit_ob.url, None)?,
77 };
78 let mut update_change_counter = false;
79 let mut update_frec = false;
80 let mut updates: Vec<(&str, &str, &dyn ToSql)> = Vec::new();
81
82 if let Some(ref title) = visit_ob.title {
83 page_info.title = crate::util::slice_up_to(title, super::TITLE_LENGTH_MAX).into();
84 updates.push(("title", ":title", &page_info.title));
85 update_change_counter = true;
86 }
87 let preview_image_url_str;
88 if let Some(ref preview_image_url) = preview_image_url {
89 preview_image_url_str = preview_image_url.as_str();
90 updates.push((
91 "preview_image_url",
92 ":preview_image_url",
93 &preview_image_url_str,
94 ));
95 }
96 let visit_row_id = match visit_ob.visit_type {
99 Some(visit_type) => {
100 if !visit_ob.get_is_hidden() {
102 updates.push(("hidden", ":hidden", &false));
103 }
104 if visit_type == VisitType::Typed {
105 page_info.typed += 1;
106 updates.push(("typed", ":typed", &page_info.typed));
107 }
108
109 let at = visit_ob.at.unwrap_or_else(Timestamp::now);
110 let is_remote = visit_ob.is_remote.unwrap_or(false);
111 let row_id = add_visit(db, page_info.row_id, None, at, visit_type, !is_remote, None)?;
112 if !visit_ob.is_error.unwrap_or(false) {
114 update_frec = true;
115 }
116 update_change_counter = true;
117 Some(row_id)
118 }
119 None => None,
120 };
121
122 if update_change_counter {
123 page_info.sync_change_counter += 1;
124 updates.push((
125 "sync_change_counter",
126 ":sync_change_counter",
127 &page_info.sync_change_counter,
128 ));
129 }
130
131 if !updates.is_empty() {
132 let mut params: Vec<(&str, &dyn ToSql)> = Vec::with_capacity(updates.len() + 1);
133 let mut sets: Vec<String> = Vec::with_capacity(updates.len());
134 for (col, name, val) in updates {
135 sets.push(format!("{} = {}", col, name));
136 params.push((name, val))
137 }
138 params.push((":row_id", &page_info.row_id.0));
139 let sql = format!(
140 "UPDATE moz_places
141 SET {}
142 WHERE id == :row_id",
143 sets.join(",")
144 );
145 db.execute(&sql, ¶ms[..])?;
146 }
147 if update_frec {
149 update_frecency(
150 db,
151 page_info.row_id,
152 Some(visit_ob.get_redirect_frecency_boost()),
153 )?;
154 }
155 Ok(visit_row_id)
156}
157
158pub fn update_frecency(db: &PlacesDb, id: RowId, redirect_boost: Option<bool>) -> Result<()> {
159 let score = frecency::calculate_frecency(
160 db.conn(),
161 &frecency::DEFAULT_FRECENCY_SETTINGS,
162 id.0, redirect_boost,
164 )?;
165
166 db.execute(
167 "
168 UPDATE moz_places
169 SET frecency = :frecency
170 WHERE id = :page_id",
171 &[
172 (":frecency", &score as &dyn rusqlite::ToSql),
173 (":page_id", &id.0),
174 ],
175 )?;
176
177 Ok(())
178}
179
180pub fn frecency_stale_at(db: &PlacesDb, url: &Url) -> Result<Option<Timestamp>> {
182 let result = db.try_query_row(
183 "SELECT stale_at FROM moz_places_stale_frecencies s
184 JOIN moz_places h ON h.id = s.place_id
185 WHERE h.url_hash = hash(:url) AND
186 h.url = :url",
187 &[(":url", &url.as_str())],
188 |row| -> rusqlite::Result<_> { row.get::<_, Timestamp>(0) },
189 true,
190 )?;
191 Ok(result)
192}
193
194fn add_visit(
198 db: &PlacesDb,
199 page_id: RowId,
200 from_visit: Option<RowId>,
201 visit_date: Timestamp,
202 visit_type: VisitType,
203 is_local: bool,
204 unknown_fields: Option<String>,
205) -> Result<RowId> {
206 let sql = "INSERT INTO moz_historyvisits
207 (from_visit, place_id, visit_date, visit_type, is_local, unknown_fields)
208 VALUES (:from_visit, :page_id, :visit_date, :visit_type, :is_local, :unknown_fields)";
209 db.execute_cached(
210 sql,
211 &[
212 (":from_visit", &from_visit as &dyn rusqlite::ToSql),
213 (":page_id", &page_id),
214 (":visit_date", &visit_date),
215 (":visit_type", &visit_type),
216 (":is_local", &is_local),
217 (":unknown_fields", &unknown_fields),
218 ],
219 )?;
220 let rid = db.conn().last_insert_rowid();
221 db.execute_cached(
223 "DELETE FROM moz_historyvisit_tombstones
224 WHERE place_id = :place_id
225 AND visit_date = :visit_date",
226 &[
227 (":place_id", &page_id as &dyn rusqlite::ToSql),
228 (":visit_date", &visit_date),
229 ],
230 )?;
231 Ok(RowId(rid))
232}
233
234pub fn url_to_guid(db: &PlacesDb, url: &Url) -> Result<Option<SyncGuid>> {
236 href_to_guid(db, url.clone().as_str())
237}
238
239pub fn href_to_guid(db: &PlacesDb, url: &str) -> Result<Option<SyncGuid>> {
241 let sql = "SELECT guid FROM moz_places WHERE url_hash = hash(:url) AND url = :url";
242 let result: Option<SyncGuid> = db.try_query_row(
243 sql,
244 &[(":url", &url.to_owned())],
245 |row| -> rusqlite::Result<_> { row.get::<_, SyncGuid>(0) },
248 true,
249 )?;
250 Ok(result)
251}
252
253fn delete_visits_for_in_tx(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
256 let to_clean = db.conn().try_query_row(
259 "SELECT id,
260 (foreign_count != 0) AS has_foreign,
261 1 as has_visits,
262 sync_status
263 FROM moz_places
264 WHERE guid = :guid",
265 &[(":guid", guid)],
266 PageToClean::from_row,
267 true,
268 )?;
269 match to_clean {
273 Some(PageToClean {
274 id,
275 has_foreign: true,
276 sync_status: SyncStatus::Normal,
277 ..
278 }) => {
279 insert_tombstones_for_all_page_visits(db, id)?;
286 delete_all_visits_for_page(db, id)?;
287 history_metadata::delete_all_metadata_for_page(db, id)?;
288 }
289 Some(PageToClean {
290 id,
291 has_foreign: false,
292 sync_status: SyncStatus::Normal,
293 ..
294 }) => {
295 insert_tombstone_for_page(db, guid)?;
299 delete_page(db, id)?;
300 }
301 Some(PageToClean {
302 id,
303 has_foreign: true,
304 ..
305 }) => {
306 delete_all_visits_for_page(db, id)?;
310 history_metadata::delete_all_metadata_for_page(db, id)?;
312 }
313 Some(PageToClean {
314 id,
315 has_foreign: false,
316 ..
317 }) => {
318 delete_page(db, id)?;
321 }
322 None => {}
323 }
324 delete_pending_temp_tables(db)?;
325 Ok(())
326}
327
328fn insert_tombstones_for_all_page_visits(db: &PlacesDb, page_id: RowId) -> Result<()> {
330 db.execute_cached(
331 "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date)
332 SELECT place_id, visit_date
333 FROM moz_historyvisits
334 WHERE place_id = :page_id",
335 &[(":page_id", &page_id)],
336 )?;
337 Ok(())
338}
339
340fn delete_all_visits_for_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
343 db.execute_cached(
344 "DELETE FROM moz_historyvisits
345 WHERE place_id = :page_id",
346 &[(":page_id", &page_id)],
347 )?;
348 Ok(())
349}
350
351fn insert_tombstone_for_page(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
353 db.execute_cached(
354 "INSERT OR IGNORE INTO moz_places_tombstones (guid)
355 VALUES(:guid)",
356 &[(":guid", guid)],
357 )?;
358 Ok(())
359}
360
361fn delete_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
364 db.execute_cached(
365 "DELETE FROM moz_places
366 WHERE id = :page_id",
367 &[(":page_id", &page_id)],
368 )?;
369 Ok(())
370}
371
372pub fn delete_visits_for(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
375 let tx = db.begin_transaction()?;
376 let result = delete_visits_for_in_tx(db, guid);
377 tx.commit()?;
378 result
379}
380
381pub fn delete_visits_between(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
383 let tx = db.begin_transaction()?;
384 delete_visits_between_in_tx(db, start, end)?;
385 tx.commit()?;
386 Ok(())
387}
388
389pub fn delete_place_visit_at_time(db: &PlacesDb, place: &Url, visit: Timestamp) -> Result<()> {
390 delete_place_visit_at_time_by_href(db, place.as_str(), visit)
391}
392
393pub fn delete_place_visit_at_time_by_href(
394 db: &PlacesDb,
395 place: &str,
396 visit: Timestamp,
397) -> Result<()> {
398 let tx = db.begin_transaction()?;
399 delete_place_visit_at_time_in_tx(db, place, visit)?;
400 tx.commit()?;
401 Ok(())
402}
403
404pub fn prune_older_visits(db: &PlacesDb, limit: u32) -> Result<()> {
405 let tx = db.begin_transaction()?;
406
407 let result = DbAction::apply_all(
408 db,
409 db_actions_from_visits_to_delete(find_visits_to_prune(
410 db,
411 limit as usize,
412 Timestamp::now(),
413 )?),
414 );
415 tx.commit()?;
416 result
417}
418
419fn find_visits_to_prune(db: &PlacesDb, limit: usize, now: Timestamp) -> Result<Vec<VisitToDelete>> {
420 let mut to_delete: HashSet<_> = find_exotic_visits_to_prune(db, limit, now)?
422 .into_iter()
423 .collect();
424 if to_delete.len() < limit {
427 for delete_visit in find_normal_visits_to_prune(db, limit, now)? {
428 to_delete.insert(delete_visit);
429 if to_delete.len() >= limit {
430 break;
431 }
432 }
433 }
434 Ok(Vec::from_iter(to_delete))
435}
436
437fn find_normal_visits_to_prune(
438 db: &PlacesDb,
439 limit: usize,
440 now: Timestamp,
441) -> Result<Vec<VisitToDelete>> {
442 let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 7));
444 db.query_rows_and_then(
445 "
446 SELECT v.id, v.place_id
447 FROM moz_places p
448 JOIN moz_historyvisits v ON v.place_id = p.id
449 WHERE v.visit_date < :visit_date_cuttoff
450 ORDER BY v.visit_date
451 LIMIT :limit
452 ",
453 rusqlite::named_params! {
454 ":visit_date_cuttoff": visit_date_cutoff,
455 ":limit": limit,
456 },
457 VisitToDelete::from_row,
458 )
459}
460
461fn find_exotic_visits_to_prune(
469 db: &PlacesDb,
470 limit: usize,
471 now: Timestamp,
472) -> Result<Vec<VisitToDelete>> {
473 let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 60));
475 db.query_rows_and_then(
476 "
477 SELECT v.id, v.place_id
478 FROM moz_places p
479 JOIN moz_historyvisits v ON v.place_id = p.id
480 WHERE v.visit_date < :visit_date_cuttoff
481 AND (LENGTH(p.url) > 255 OR v.visit_type = :download_visit_type)
482 ORDER BY v.visit_date
483 LIMIT :limit
484 ",
485 rusqlite::named_params! {
486 ":visit_date_cuttoff": visit_date_cutoff,
487 ":download_visit_type": VisitType::Download,
488 ":limit": limit,
489 },
490 VisitToDelete::from_row,
491 )
492}
493
494fn wipe_local_in_tx(db: &PlacesDb) -> Result<()> {
495 use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
496 db.execute_all(&[
497 "DELETE FROM moz_places WHERE foreign_count == 0",
498 "DELETE FROM moz_places_metadata",
499 "DELETE FROM moz_places_metadata_search_queries",
500 "DELETE FROM moz_historyvisits",
501 "DELETE FROM moz_places_tombstones",
502 "DELETE FROM moz_inputhistory AS i WHERE NOT EXISTS(
503 SELECT 1 FROM moz_places h
504 WHERE h.id = i.place_id)",
505 "DELETE FROM moz_historyvisit_tombstones",
506 "DELETE FROM moz_origins
507 WHERE id NOT IN (SELECT origin_id FROM moz_places)",
508 &format!(
509 r#"UPDATE moz_places SET
510 frecency = (CASE WHEN url_hash BETWEEN hash("place", "prefix_lo") AND
511 hash("place", "prefix_hi")
512 THEN 0
513 ELSE {unvisited_bookmark_frec}
514 END),
515 sync_change_counter = 0"#,
516 unvisited_bookmark_frec = DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
517 ),
518 ])?;
519
520 let need_frecency_update =
521 db.query_rows_and_then("SELECT id FROM moz_places", [], |r| r.get::<_, RowId>(0))?;
522 for row_id in need_frecency_update {
525 update_frecency(db, row_id, None)?;
526 }
527 delete_pending_temp_tables(db)?;
528 Ok(())
529}
530
531pub fn delete_everything(db: &PlacesDb) -> Result<()> {
532 let tx = db.begin_transaction()?;
533
534 let most_recent_known_visit_time = db
536 .try_query_one::<Timestamp, _>("SELECT MAX(visit_date) FROM moz_historyvisits", [], false)?
537 .unwrap_or_default();
538
539 let previous_mark =
541 get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
542
543 let new_mark = Timestamp::now()
544 .max(previous_mark)
545 .max(most_recent_known_visit_time);
546
547 put_meta(db, DELETION_HIGH_WATER_MARK_META_KEY, &new_mark)?;
548
549 wipe_local_in_tx(db)?;
550
551 reset_in_tx(db, &EngineSyncAssociation::Disconnected)?;
553
554 tx.commit()?;
555
556 db.execute_batch("VACUUM")?;
558 Ok(())
559}
560
561fn delete_place_visit_at_time_in_tx(db: &PlacesDb, url: &str, visit_date: Timestamp) -> Result<()> {
562 DbAction::apply_all(
563 db,
564 db_actions_from_visits_to_delete(db.query_rows_and_then(
565 "SELECT v.id, v.place_id
566 FROM moz_places h
567 JOIN moz_historyvisits v
568 ON v.place_id = h.id
569 WHERE v.visit_date = :visit_date
570 AND h.url_hash = hash(:url)
571 AND h.url = :url",
572 &[
573 (":url", &url as &dyn rusqlite::ToSql),
574 (":visit_date", &visit_date),
575 ],
576 VisitToDelete::from_row,
577 )?),
578 )
579}
580
581pub fn delete_visits_between_in_tx(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
582 let sql = "
586 SELECT id, place_id, visit_date
587 FROM moz_historyvisits
588 WHERE visit_date
589 BETWEEN :start AND :end
590 ";
591 let visits = db.query_rows_and_then(
592 sql,
593 &[(":start", &start), (":end", &end)],
594 |row| -> rusqlite::Result<_> {
595 Ok((
596 row.get::<_, RowId>(0)?,
597 row.get::<_, RowId>(1)?,
598 row.get::<_, Timestamp>(2)?,
599 ))
600 },
601 )?;
602
603 sql_support::each_chunk_mapped(
604 &visits,
605 |(visit_id, _, _)| visit_id,
606 |chunk, _| -> Result<()> {
607 db.conn().execute(
608 &format!(
609 "DELETE from moz_historyvisits WHERE id IN ({})",
610 sql_support::repeat_sql_vars(chunk.len()),
611 ),
612 rusqlite::params_from_iter(chunk),
613 )?;
614 Ok(())
615 },
616 )?;
617
618 if !visits.is_empty() {
620 let sql = format!(
621 "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date) VALUES {}",
622 sql_support::repeat_display(visits.len(), ",", |i, f| {
623 let (_, place_id, visit_date) = visits[i];
624 write!(f, "({},{})", place_id.0, visit_date.0)
625 })
626 );
627 db.conn().execute(&sql, [])?;
628 }
629
630 sql_support::each_chunk_mapped(
632 &visits,
633 |(_, place_id, _)| place_id.0,
634 |chunk, _| -> Result<()> {
635 let query = format!(
636 "SELECT id,
637 (foreign_count != 0) AS has_foreign,
638 ((last_visit_date_local + last_visit_date_remote) != 0) as has_visits,
639 sync_status
640 FROM moz_places
641 WHERE id IN ({})",
642 sql_support::repeat_sql_vars(chunk.len()),
643 );
644
645 let mut stmt = db.conn().prepare(&query)?;
646 let page_results =
647 stmt.query_and_then(rusqlite::params_from_iter(chunk), PageToClean::from_row)?;
648 let pages: Vec<PageToClean> = page_results.collect::<Result<_>>()?;
649 cleanup_pages(db, &pages)
650 },
651 )?;
652
653 history_metadata::delete_between(db, start.as_millis_i64(), end.as_millis_i64())?;
655 delete_pending_temp_tables(db)?;
656 Ok(())
657}
658
659#[derive(Debug)]
660struct PageToClean {
661 id: RowId,
662 has_foreign: bool,
663 has_visits: bool,
664 sync_status: SyncStatus,
665}
666
667impl PageToClean {
668 pub fn from_row(row: &Row<'_>) -> Result<Self> {
669 Ok(Self {
670 id: row.get("id")?,
671 has_foreign: row.get("has_foreign")?,
672 has_visits: row.get("has_visits")?,
673 sync_status: row.get("sync_status")?,
674 })
675 }
676}
677
678fn cleanup_pages(db: &PlacesDb, pages: &[PageToClean]) -> Result<()> {
684 let frec_ids = pages
687 .iter()
688 .filter(|&p| p.has_foreign || p.has_visits)
689 .map(|p| p.id);
690
691 for id in frec_ids {
692 update_frecency(db, id, None)?;
693 }
694
695 let remove_ids: Vec<RowId> = pages
701 .iter()
702 .filter(|p| !p.has_foreign && !p.has_visits)
703 .map(|p| p.id)
704 .collect();
705 sql_support::each_chunk(&remove_ids, |chunk, _| -> Result<()> {
706 db.conn().execute(
708 &format!(
709 "
710 INSERT OR IGNORE INTO moz_places_tombstones (guid)
711 SELECT guid FROM moz_places
712 WHERE id in ({ids}) AND sync_status = {status}
713 AND foreign_count = 0
714 AND last_visit_date_local = 0
715 AND last_visit_date_remote = 0",
716 ids = sql_support::repeat_sql_vars(chunk.len()),
717 status = SyncStatus::Normal as u8,
718 ),
719 rusqlite::params_from_iter(chunk),
720 )?;
721 db.conn().execute(
722 &format!(
723 "
724 DELETE FROM moz_places
725 WHERE id IN ({ids})
726 AND foreign_count = 0
727 AND last_visit_date_local = 0
728 AND last_visit_date_remote = 0",
729 ids = sql_support::repeat_sql_vars(chunk.len())
730 ),
731 rusqlite::params_from_iter(chunk),
732 )?;
733 Ok(())
734 })?;
735
736 Ok(())
737}
738
739fn reset_in_tx(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
740 db.execute_cached(
742 &format!(
743 "
744 UPDATE moz_places
745 SET sync_change_counter = 0,
746 sync_status = {}",
747 (SyncStatus::New as u8)
748 ),
749 [],
750 )?;
751
752 put_meta(db, LAST_SYNC_META_KEY, &0)?;
755
756 match assoc {
759 EngineSyncAssociation::Disconnected => {
760 delete_meta(db, GLOBAL_SYNCID_META_KEY)?;
761 delete_meta(db, COLLECTION_SYNCID_META_KEY)?;
762 }
763 EngineSyncAssociation::Connected(ids) => {
764 put_meta(db, GLOBAL_SYNCID_META_KEY, &ids.global)?;
765 put_meta(db, COLLECTION_SYNCID_META_KEY, &ids.coll)?;
766 }
767 }
768
769 Ok(())
770}
771
772pub mod history_sync {
774 use sync15::bso::OutgoingEnvelope;
775
776 use super::*;
777 use crate::history_sync::record::{HistoryRecord, HistoryRecordVisit};
778 use crate::history_sync::HISTORY_TTL;
779 use std::collections::HashSet;
780
781 #[derive(Debug, Clone, PartialEq, Eq)]
782 pub struct FetchedVisit {
783 pub is_local: bool,
784 pub visit_date: Timestamp,
785 pub visit_type: Option<VisitType>,
786 }
787
788 impl FetchedVisit {
789 pub fn from_row(row: &Row<'_>) -> Result<Self> {
790 Ok(Self {
791 is_local: row.get("is_local")?,
792 visit_date: row
793 .get::<_, Option<Timestamp>>("visit_date")?
794 .unwrap_or_default(),
795 visit_type: VisitType::from_primitive(
796 row.get::<_, Option<u8>>("visit_type")?.unwrap_or(0),
797 ),
798 })
799 }
800 }
801
802 #[derive(Debug)]
803 pub struct FetchedVisitPage {
804 pub url: Url,
805 pub guid: SyncGuid,
806 pub row_id: RowId,
807 pub title: String,
808 pub unknown_fields: UnknownFields,
809 }
810
811 impl FetchedVisitPage {
812 pub fn from_row(row: &Row<'_>) -> Result<Self> {
813 Ok(Self {
814 url: Url::parse(&row.get::<_, String>("url")?)?,
815 guid: row.get::<_, String>("guid")?.into(),
816 row_id: row.get("id")?,
817 title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
818 unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
819 None => UnknownFields::new(),
820 Some(v) => serde_json::from_str(&v)?,
821 },
822 })
823 }
824 }
825
826 pub fn fetch_visits(
827 db: &PlacesDb,
828 url: &Url,
829 limit: usize,
830 ) -> Result<Option<(FetchedVisitPage, Vec<FetchedVisit>)>> {
831 let page_sql = "
833 SELECT guid, url, id, title, unknown_fields
834 FROM moz_places h
835 WHERE url_hash = hash(:url) AND url = :url";
836
837 let page_info = match db.try_query_row(
838 page_sql,
839 &[(":url", &url.to_string())],
840 FetchedVisitPage::from_row,
841 true,
842 )? {
843 None => return Ok(None),
844 Some(pi) => pi,
845 };
846
847 let visits = db.query_rows_and_then(
848 "SELECT is_local, visit_type, visit_date
849 FROM moz_historyvisits
850 WHERE place_id = :place_id
851 LIMIT :limit",
852 &[
853 (":place_id", &page_info.row_id as &dyn rusqlite::ToSql),
854 (":limit", &(limit as u32)),
855 ],
856 FetchedVisit::from_row,
857 )?;
858 Ok(Some((page_info, visits)))
859 }
860
861 pub fn apply_synced_visits(
864 db: &PlacesDb,
865 incoming_guid: &SyncGuid,
866 url: &Url,
867 title: &Option<String>,
868 visits: &[HistoryRecordVisit],
869 unknown_fields: &UnknownFields,
870 ) -> Result<()> {
871 let visit_ignored_mark =
875 get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
876
877 let visits = visits
878 .iter()
879 .filter(|v| Timestamp::from(v.date) > visit_ignored_mark)
880 .collect::<Vec<_>>();
881
882 let mut counter_incr = 0;
883 let page_info = match fetch_page_info(db, url)? {
884 Some(mut info) => {
885 if &info.page.guid != incoming_guid {
890 if info.page.sync_status == SyncStatus::New {
891 db.execute_cached(
892 "UPDATE moz_places SET guid = :new_guid WHERE id = :row_id",
893 &[
894 (":new_guid", incoming_guid as &dyn rusqlite::ToSql),
895 (":row_id", &info.page.row_id),
896 ],
897 )?;
898 info.page.guid = incoming_guid.clone();
899 }
900 counter_incr = 1;
904 }
905 info.page
906 }
907 None => {
908 if visits.is_empty() {
911 return Ok(());
912 }
913 new_page_info(db, url, Some(incoming_guid.clone()))?
914 }
915 };
916
917 if !visits.is_empty() {
918 let mut visits_to_skip: HashSet<Timestamp> = db.query_rows_into(
926 &format!(
927 "SELECT t.visit_date AS visit_date
928 FROM moz_historyvisit_tombstones t
929 WHERE t.place_id = {place}
930 AND t.visit_date IN ({dates})
931 UNION ALL
932 SELECT v.visit_date AS visit_date
933 FROM moz_historyvisits v
934 WHERE v.place_id = {place}
935 AND v.visit_date IN ({dates})",
936 place = page_info.row_id,
937 dates = sql_support::repeat_display(visits.len(), ",", |i, f| write!(
938 f,
939 "{}",
940 Timestamp::from(visits[i].date).0
941 )),
942 ),
943 [],
944 |row| row.get::<_, Timestamp>(0),
945 )?;
946
947 visits_to_skip.reserve(visits.len());
948
949 for visit in visits {
950 let timestamp = Timestamp::from(visit.date);
951 if visits_to_skip.contains(×tamp) {
953 continue;
954 }
955 let transition = VisitType::from_primitive(visit.transition)
956 .expect("these should already be validated");
957 add_visit(
958 db,
959 page_info.row_id,
960 None,
961 timestamp,
962 transition,
963 false,
964 serialize_unknown_fields(&visit.unknown_fields)?,
965 )?;
966 visits_to_skip.insert(timestamp);
970 }
971 }
972 update_frecency(db, page_info.row_id, None)?;
975
976 let new_title = title.as_ref().unwrap_or(&page_info.title);
978 db.execute_cached(
983 "UPDATE moz_places
984 SET title = :title,
985 unknown_fields = :unknown_fields,
986 sync_status = :status,
987 sync_change_counter = :sync_change_counter
988 WHERE id == :row_id",
989 &[
990 (":title", new_title as &dyn rusqlite::ToSql),
991 (":row_id", &page_info.row_id),
992 (":status", &SyncStatus::Normal),
993 (
994 ":unknown_fields",
995 &serialize_unknown_fields(unknown_fields)?,
996 ),
997 (
998 ":sync_change_counter",
999 &(page_info.sync_change_counter + counter_incr),
1000 ),
1001 ],
1002 )?;
1003
1004 Ok(())
1005 }
1006
1007 pub fn apply_synced_reconciliation(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1008 db.execute_cached(
1009 "UPDATE moz_places
1010 SET sync_status = :status,
1011 sync_change_counter = 0
1012 WHERE guid == :guid",
1013 &[
1014 (":guid", guid as &dyn rusqlite::ToSql),
1015 (":status", &SyncStatus::Normal),
1016 ],
1017 )?;
1018 Ok(())
1019 }
1020
1021 pub fn apply_synced_deletion(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1022 db.execute_cached(
1026 "DELETE FROM moz_historyvisits
1027 WHERE place_id IN (
1028 SELECT id
1029 FROM moz_places
1030 WHERE guid = :guid
1031 )",
1032 &[(":guid", guid)],
1033 )?;
1034 db.execute_cached(
1035 "DELETE FROM moz_places WHERE guid = :guid AND foreign_count = 0",
1036 &[(":guid", guid)],
1037 )?;
1038 Ok(())
1039 }
1040
1041 pub fn fetch_outgoing(
1042 db: &PlacesDb,
1043 max_places: usize,
1044 max_visits: usize,
1045 ) -> Result<Vec<OutgoingBso>> {
1046 let places_sql = format!(
1051 "
1052 SELECT guid, url, id, title, hidden, typed, frecency,
1053 visit_count_local, visit_count_remote,
1054 last_visit_date_local, last_visit_date_remote,
1055 sync_status, sync_change_counter, preview_image_url,
1056 unknown_fields
1057 FROM moz_places
1058 WHERE (sync_change_counter > 0 OR sync_status != {}) AND
1059 NOT hidden
1060 ORDER BY frecency DESC
1061 LIMIT :max_places",
1062 (SyncStatus::Normal as u8)
1063 );
1064 let visits_sql = "
1065 SELECT visit_date as date, visit_type as transition, unknown_fields
1066 FROM moz_historyvisits
1067 WHERE place_id = :place_id
1068 ORDER BY visit_date DESC
1069 LIMIT :max_visits";
1070 let tombstones_sql = "SELECT guid FROM moz_places_tombstones LIMIT :max_places";
1072
1073 let mut tombstone_ids = HashSet::new();
1074 let mut result = Vec::new();
1075
1076 let ts_rows = db.query_rows_and_then(
1079 tombstones_sql,
1080 &[(":max_places", &(max_places as u32))],
1081 |row| -> rusqlite::Result<SyncGuid> { Ok(row.get::<_, String>("guid")?.into()) },
1082 )?;
1083 result.reserve(ts_rows.len());
1087 tombstone_ids.reserve(ts_rows.len());
1088 for guid in ts_rows {
1089 trace!("outgoing tombstone {:?}", &guid);
1090 let envelope = OutgoingEnvelope {
1091 id: guid.clone(),
1092 ttl: Some(HISTORY_TTL),
1093 ..Default::default()
1094 };
1095 result.push(OutgoingBso::new_tombstone(envelope));
1096 tombstone_ids.insert(guid);
1097 }
1098
1099 let max_places_left = max_places - result.len();
1101
1102 db.execute(
1108 "CREATE TEMP TABLE IF NOT EXISTS temp_sync_updated_meta
1109 (id INTEGER PRIMARY KEY,
1110 change_delta INTEGER NOT NULL)",
1111 [],
1112 )?;
1113
1114 let insert_meta_sql = "
1115 INSERT INTO temp_sync_updated_meta VALUES (:row_id, :change_delta)";
1116
1117 let rows = db.query_rows_and_then(
1118 &places_sql,
1119 &[(":max_places", &(max_places_left as u32))],
1120 PageInfo::from_row,
1121 )?;
1122 result.reserve(rows.len());
1123 let mut ids_to_update = Vec::with_capacity(rows.len());
1124 for page in rows {
1125 let visits = db.query_rows_and_then_cached(
1126 visits_sql,
1127 &[
1128 (":max_visits", &(max_visits as u32) as &dyn rusqlite::ToSql),
1129 (":place_id", &page.row_id),
1130 ],
1131 |row| -> Result<_> {
1132 Ok(HistoryRecordVisit {
1133 date: row.get::<_, Timestamp>("date")?.into(),
1134 transition: row.get::<_, u8>("transition")?,
1135 unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
1136 None => UnknownFields::new(),
1137 Some(v) => serde_json::from_str(&v)?,
1138 },
1139 })
1140 },
1141 )?;
1142 if tombstone_ids.contains(&page.guid) {
1143 warn!("Found {:?} in both tombstones and live records", &page.guid);
1145 continue;
1146 }
1147 if visits.is_empty() {
1148 trace!(
1152 "Page {:?} is flagged to be uploaded, but has no visits - skipping",
1153 &page.guid
1154 );
1155 continue;
1156 }
1157 trace!("outgoing record {:?}", &page.guid);
1158 ids_to_update.push(page.row_id);
1159 db.execute_cached(
1160 insert_meta_sql,
1161 &[
1162 (":row_id", &page.row_id as &dyn rusqlite::ToSql),
1163 (":change_delta", &page.sync_change_counter),
1164 ],
1165 )?;
1166
1167 let content = HistoryRecord {
1168 id: page.guid.clone(),
1169 title: page.title,
1170 hist_uri: page.url.to_string(),
1171 visits,
1172 unknown_fields: page.unknown_fields,
1173 };
1174
1175 let envelope = OutgoingEnvelope {
1176 id: page.guid,
1177 sortindex: Some(page.frecency),
1178 ttl: Some(HISTORY_TTL),
1179 };
1180 let bso = OutgoingBso::from_content(envelope, content)?;
1181 result.push(bso);
1182 }
1183
1184 sql_support::each_chunk(&ids_to_update, |chunk, _| -> Result<()> {
1189 db.conn().execute(
1190 &format!(
1191 "UPDATE moz_places SET sync_status={status}
1192 WHERE id IN ({vars})",
1193 vars = sql_support::repeat_sql_vars(chunk.len()),
1194 status = SyncStatus::Normal as u8
1195 ),
1196 rusqlite::params_from_iter(chunk),
1197 )?;
1198 Ok(())
1199 })?;
1200
1201 Ok(result)
1202 }
1203
1204 pub fn finish_outgoing(db: &PlacesDb) -> Result<()> {
1205 debug!("Updating all synced rows");
1216 db.conn().execute_cached(
1219 "
1220 UPDATE moz_places
1221 SET sync_change_counter = sync_change_counter -
1222 (SELECT change_delta FROM temp_sync_updated_meta m WHERE moz_places.id = m.id)
1223 WHERE id IN (SELECT id FROM temp_sync_updated_meta)
1224 ",
1225 [],
1226 )?;
1227
1228 debug!("Updating all non-synced rows");
1229 db.execute_all(&[
1230 &format!(
1231 "UPDATE moz_places
1232 SET sync_change_counter = 0, sync_status = {}
1233 WHERE id NOT IN (SELECT id from temp_sync_updated_meta)",
1234 (SyncStatus::Normal as u8)
1235 ),
1236 "DELETE FROM temp_sync_updated_meta",
1237 ])?;
1238
1239 debug!("Removing local tombstones");
1240 db.conn()
1241 .execute_cached("DELETE from moz_places_tombstones", [])?;
1242
1243 Ok(())
1244 }
1245
1246 pub(crate) fn reset(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
1250 let tx = db.begin_transaction()?;
1251 reset_in_tx(db, assoc)?;
1252 tx.commit()?;
1253 Ok(())
1254 }
1255} pub fn get_visited<I>(db: &PlacesDb, urls: I) -> Result<Vec<bool>>
1258where
1259 I: IntoIterator<Item = Url>,
1260 I::IntoIter: ExactSizeIterator,
1261{
1262 let iter = urls.into_iter();
1263 let mut result = vec![false; iter.len()];
1264 let url_idxs = iter.enumerate().collect::<Vec<_>>();
1265 get_visited_into(db, &url_idxs, &mut result)?;
1266 Ok(result)
1267}
1268
1269pub fn get_visited_into(
1276 db: &PlacesDb,
1277 urls_idxs: &[(usize, Url)],
1278 result: &mut [bool],
1279) -> Result<()> {
1280 sql_support::each_chunk_mapped(
1281 urls_idxs,
1282 |(_, url)| url.as_str(),
1283 |chunk, offset| -> Result<()> {
1284 let values_with_idx = sql_support::repeat_display(chunk.len(), ",", |i, f| {
1285 let (idx, url) = &urls_idxs[i + offset];
1286 write!(f, "({},{},?)", *idx, hash::hash_url(url.as_str()))
1287 });
1288 let sql = format!(
1289 "WITH to_fetch(fetch_url_index, url_hash, url) AS (VALUES {})
1290 SELECT fetch_url_index
1291 FROM moz_places h
1292 JOIN to_fetch f ON h.url_hash = f.url_hash
1293 AND h.url = f.url
1294 AND (h.last_visit_date_local + h.last_visit_date_remote) != 0",
1295 values_with_idx
1296 );
1297 let mut stmt = db.prepare(&sql)?;
1298 for idx_r in stmt.query_and_then(
1299 rusqlite::params_from_iter(chunk),
1300 |row| -> rusqlite::Result<_> { Ok(row.get::<_, i64>(0)? as usize) },
1301 )? {
1302 let idx = idx_r?;
1303 result[idx] = true;
1304 }
1305 Ok(())
1306 },
1307 )?;
1308 Ok(())
1309}
1310
1311pub fn get_visited_urls(
1314 db: &PlacesDb,
1315 start: Timestamp,
1316 end: Timestamp,
1317 include_remote: bool,
1318) -> Result<Vec<String>> {
1319 let sql = format!(
1323 "SELECT h.url
1324 FROM moz_places h
1325 WHERE EXISTS (
1326 SELECT 1 FROM moz_historyvisits v
1327 WHERE place_id = h.id
1328 AND visit_date BETWEEN :start AND :end
1329 {and_is_local}
1330 LIMIT 1
1331 )",
1332 and_is_local = if include_remote { "" } else { "AND is_local" }
1333 );
1334 Ok(db.query_rows_and_then_cached(
1335 &sql,
1336 &[(":start", &start), (":end", &end)],
1337 |row| -> RusqliteResult<_> { row.get::<_, String>(0) },
1338 )?)
1339}
1340
1341pub fn get_top_frecent_site_infos(
1342 db: &PlacesDb,
1343 num_items: i32,
1344 frecency_threshold: i64,
1345) -> Result<Vec<TopFrecentSiteInfo>> {
1346 let allowed_types = VisitTransitionSet::for_specific(&[
1348 VisitType::Download,
1349 VisitType::Embed,
1350 VisitType::RedirectPermanent,
1351 VisitType::RedirectTemporary,
1352 VisitType::FramedLink,
1353 VisitType::Reload,
1354 ])
1355 .complement();
1356
1357 let infos = db.query_rows_and_then_cached(
1358 "SELECT h.frecency, h.title, h.url
1359 FROM moz_places h
1360 WHERE EXISTS (
1361 SELECT v.visit_type
1362 FROM moz_historyvisits v
1363 WHERE h.id = v.place_id
1364 AND (SUBSTR(h.url, 1, 6) == 'https:' OR SUBSTR(h.url, 1, 5) == 'http:')
1365 AND (h.last_visit_date_local + h.last_visit_date_remote) != 0
1366 AND ((1 << v.visit_type) & :allowed_types) != 0
1367 AND h.frecency >= :frecency_threshold AND
1368 NOT h.hidden
1369 )
1370 ORDER BY h.frecency DESC
1371 LIMIT :limit",
1372 rusqlite::named_params! {
1373 ":limit": num_items,
1374 ":allowed_types": allowed_types,
1375 ":frecency_threshold": frecency_threshold,
1376 },
1377 TopFrecentSiteInfo::from_row,
1378 )?;
1379 Ok(infos)
1380}
1381
1382pub fn get_visit_infos(
1383 db: &PlacesDb,
1384 start: Timestamp,
1385 end: Timestamp,
1386 exclude_types: VisitTransitionSet,
1387) -> Result<Vec<HistoryVisitInfo>> {
1388 let allowed_types = exclude_types.complement();
1389 let infos = db.query_rows_and_then_cached(
1390 "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1391 v.is_local
1392 FROM moz_places h
1393 JOIN moz_historyvisits v
1394 ON h.id = v.place_id
1395 WHERE v.visit_date BETWEEN :start AND :end
1396 AND ((1 << visit_type) & :allowed_types) != 0 AND
1397 NOT h.hidden
1398 ORDER BY v.visit_date",
1399 rusqlite::named_params! {
1400 ":start": start,
1401 ":end": end,
1402 ":allowed_types": allowed_types,
1403 },
1404 HistoryVisitInfo::from_row,
1405 )?;
1406 Ok(infos)
1407}
1408
1409pub fn get_visit_count(db: &PlacesDb, exclude_types: VisitTransitionSet) -> Result<i64> {
1410 let count = if exclude_types.is_empty() {
1411 db.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")?
1412 } else {
1413 let allowed_types = exclude_types.complement();
1414 db.query_row_and_then_cachable(
1415 "SELECT COUNT(*)
1416 FROM moz_historyvisits
1417 WHERE ((1 << visit_type) & :allowed_types) != 0",
1418 rusqlite::named_params! {
1419 ":allowed_types": allowed_types,
1420 },
1421 |r| r.get(0),
1422 true,
1423 )?
1424 };
1425 Ok(count)
1426}
1427
1428pub fn get_visit_count_for_host(
1429 db: &PlacesDb,
1430 host: &str,
1431 before: Timestamp,
1432 exclude_types: VisitTransitionSet,
1433) -> Result<i64> {
1434 let allowed_types = exclude_types.complement();
1435 let count = db.query_row_and_then_cachable(
1436 "SELECT COUNT(*)
1437 FROM moz_historyvisits
1438 JOIN moz_places ON moz_places.id = moz_historyvisits.place_id
1439 JOIN moz_origins ON moz_origins.id = moz_places.origin_id
1440 WHERE moz_origins.host = :host
1441 AND visit_date < :before
1442 AND ((1 << visit_type) & :allowed_types) != 0",
1443 rusqlite::named_params! {
1444 ":host": host,
1445 ":before": before,
1446 ":allowed_types": allowed_types,
1447 },
1448 |r| r.get(0),
1449 true,
1450 )?;
1451 Ok(count)
1452}
1453
1454pub fn get_visit_page(
1455 db: &PlacesDb,
1456 offset: i64,
1457 count: i64,
1458 exclude_types: VisitTransitionSet,
1459) -> Result<Vec<HistoryVisitInfo>> {
1460 let allowed_types = exclude_types.complement();
1461 let infos = db.query_rows_and_then_cached(
1462 "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1463 v.is_local
1464 FROM moz_places h
1465 JOIN moz_historyvisits v
1466 ON h.id = v.place_id
1467 WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1468 NOT h.hidden
1469 ORDER BY v.visit_date DESC, v.id
1470 LIMIT :count
1471 OFFSET :offset",
1472 rusqlite::named_params! {
1473 ":count": count,
1474 ":offset": offset,
1475 ":allowed_types": allowed_types,
1476 },
1477 HistoryVisitInfo::from_row,
1478 )?;
1479 Ok(infos)
1480}
1481
1482pub fn get_visit_page_with_bound(
1483 db: &PlacesDb,
1484 bound: i64,
1485 offset: i64,
1486 count: i64,
1487 exclude_types: VisitTransitionSet,
1488) -> Result<HistoryVisitInfosWithBound> {
1489 let allowed_types = exclude_types.complement();
1490 let infos = db.query_rows_and_then_cached(
1491 "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1492 v.is_local
1493 FROM moz_places h
1494 JOIN moz_historyvisits v
1495 ON h.id = v.place_id
1496 WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1497 NOT h.hidden
1498 AND v.visit_date <= :bound
1499 ORDER BY v.visit_date DESC, v.id
1500 LIMIT :count
1501 OFFSET :offset",
1502 rusqlite::named_params! {
1503 ":allowed_types": allowed_types,
1504 ":bound": bound,
1505 ":count": count,
1506 ":offset": offset,
1507 },
1508 HistoryVisitInfo::from_row,
1509 )?;
1510
1511 if let Some(l) = infos.last() {
1512 if l.timestamp.as_millis_i64() == bound {
1513 let offset = offset + infos.len() as i64;
1515 Ok(HistoryVisitInfosWithBound {
1516 infos,
1517 bound,
1518 offset,
1519 })
1520 } else {
1521 let bound = l.timestamp;
1522 let offset = infos
1523 .iter()
1524 .rev()
1525 .take_while(|i| i.timestamp == bound)
1526 .count() as i64;
1527 Ok(HistoryVisitInfosWithBound {
1528 infos,
1529 bound: bound.as_millis_i64(),
1530 offset,
1531 })
1532 }
1533 } else {
1534 Ok(HistoryVisitInfosWithBound {
1536 infos,
1537 bound: 0,
1538 offset: 0,
1539 })
1540 }
1541}
1542
1543#[cfg(test)]
1544mod tests {
1545 use super::history_sync::*;
1546 use super::*;
1547 use crate::history_sync::record::HistoryRecordVisit;
1548 use crate::storage::bookmarks::{insert_bookmark, InsertableItem};
1549 use crate::types::VisitTransitionSet;
1550 use crate::{api::places_api::ConnectionType, storage::bookmarks::BookmarkRootGuid};
1551 use std::time::{Duration, SystemTime};
1552 use sync15::engine::CollSyncIds;
1553 use types::Timestamp;
1554
1555 #[test]
1556 fn test_get_visited_urls() {
1557 use std::collections::HashSet;
1558 use std::time::SystemTime;
1559 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1560 let now: Timestamp = SystemTime::now().into();
1561 let now_u64 = now.0;
1562 let to_add = [
1564 (
1565 "https://www.example.com/1",
1566 now_u64 - 200_100,
1567 false,
1568 (false, false),
1569 ),
1570 (
1571 "https://www.example.com/12",
1572 now_u64 - 200_000,
1573 false,
1574 (true, true),
1575 ),
1576 (
1577 "https://www.example.com/123",
1578 now_u64 - 10_000,
1579 true,
1580 (true, false),
1581 ),
1582 (
1583 "https://www.example.com/1234",
1584 now_u64 - 1000,
1585 false,
1586 (true, true),
1587 ),
1588 (
1589 "https://www.mozilla.com",
1590 now_u64 - 500,
1591 false,
1592 (false, false),
1593 ),
1594 ];
1595
1596 for &(url, when, remote, _) in &to_add {
1597 apply_observation(
1598 &conn,
1599 VisitObservation::new(Url::parse(url).unwrap())
1600 .with_at(Timestamp(when))
1601 .with_is_remote(remote)
1602 .with_visit_type(VisitType::Link),
1603 )
1604 .expect("Should apply visit");
1605 }
1606
1607 let visited_all = get_visited_urls(
1608 &conn,
1609 Timestamp(now_u64 - 200_000),
1610 Timestamp(now_u64 - 1000),
1611 true,
1612 )
1613 .unwrap()
1614 .into_iter()
1615 .collect::<HashSet<_>>();
1616
1617 let visited_local = get_visited_urls(
1618 &conn,
1619 Timestamp(now_u64 - 200_000),
1620 Timestamp(now_u64 - 1000),
1621 false,
1622 )
1623 .unwrap()
1624 .into_iter()
1625 .collect::<HashSet<_>>();
1626
1627 for &(url, ts, is_remote, (expected_in_all, expected_in_local)) in &to_add {
1628 let url = Url::parse(url).unwrap().to_string();
1630 assert_eq!(
1631 expected_in_local,
1632 visited_local.contains(&url),
1633 "Failed in local for {:?}",
1634 (url, ts, is_remote)
1635 );
1636 assert_eq!(
1637 expected_in_all,
1638 visited_all.contains(&url),
1639 "Failed in all for {:?}",
1640 (url, ts, is_remote)
1641 );
1642 }
1643 }
1644
1645 fn get_custom_observed_page<F>(conn: &mut PlacesDb, url: &str, custom: F) -> Result<PageInfo>
1646 where
1647 F: Fn(VisitObservation) -> VisitObservation,
1648 {
1649 let u = Url::parse(url)?;
1650 let obs = VisitObservation::new(u.clone()).with_visit_type(VisitType::Link);
1651 apply_observation(conn, custom(obs))?;
1652 Ok(fetch_page_info(conn, &u)?
1653 .expect("should have the page")
1654 .page)
1655 }
1656
1657 fn get_observed_page(conn: &mut PlacesDb, url: &str) -> Result<PageInfo> {
1658 get_custom_observed_page(conn, url, |o| o)
1659 }
1660
1661 fn get_tombstone_count(conn: &PlacesDb) -> u32 {
1662 let result: Result<Option<u32>> = conn.try_query_row(
1663 "SELECT COUNT(*) from moz_places_tombstones;",
1664 [],
1665 |row| Ok(row.get::<_, u32>(0)?),
1666 true,
1667 );
1668 result
1669 .expect("should have worked")
1670 .expect("should have got a value")
1671 }
1672
1673 #[test]
1674 fn test_visit_counts() -> Result<()> {
1675 error_support::init_for_tests();
1676 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1677 let url = Url::parse("https://www.example.com").expect("it's a valid url");
1678 let early_time = SystemTime::now() - Duration::new(60, 0);
1679 let late_time = SystemTime::now();
1680
1681 let rid1 = apply_observation(
1683 &conn,
1684 VisitObservation::new(url.clone())
1685 .with_visit_type(VisitType::Link)
1686 .with_at(Some(late_time.into())),
1687 )?
1688 .expect("should get a rowid");
1689
1690 let rid2 = apply_observation(
1691 &conn,
1692 VisitObservation::new(url.clone())
1693 .with_visit_type(VisitType::Link)
1694 .with_at(Some(early_time.into())),
1695 )?
1696 .expect("should get a rowid");
1697
1698 let mut pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1699 assert_eq!(pi.page.visit_count_local, 2);
1700 assert_eq!(pi.page.last_visit_date_local, late_time.into());
1701 assert_eq!(pi.page.visit_count_remote, 0);
1702 assert_eq!(pi.page.last_visit_date_remote.0, 0);
1703
1704 let rid3 = apply_observation(
1706 &conn,
1707 VisitObservation::new(url.clone())
1708 .with_visit_type(VisitType::Link)
1709 .with_at(Some(early_time.into()))
1710 .with_is_remote(true),
1711 )?
1712 .expect("should get a rowid");
1713
1714 let rid4 = apply_observation(
1715 &conn,
1716 VisitObservation::new(url.clone())
1717 .with_visit_type(VisitType::Link)
1718 .with_at(Some(late_time.into()))
1719 .with_is_remote(true),
1720 )?
1721 .expect("should get a rowid");
1722
1723 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1724 assert_eq!(pi.page.visit_count_local, 2);
1725 assert_eq!(pi.page.last_visit_date_local, late_time.into());
1726 assert_eq!(pi.page.visit_count_remote, 2);
1727 assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1728
1729 let sql = "DELETE FROM moz_historyvisits WHERE id = :row_id";
1733 conn.execute_cached(sql, &[(":row_id", &rid1)])?;
1735 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1736 assert_eq!(pi.page.visit_count_local, 1);
1737 assert_eq!(pi.page.last_visit_date_local, early_time.into());
1738 assert_eq!(pi.page.visit_count_remote, 2);
1739 assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1740
1741 conn.execute_cached(sql, &[(":row_id", &rid3)])?;
1743 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1744 assert_eq!(pi.page.visit_count_local, 1);
1745 assert_eq!(pi.page.last_visit_date_local, early_time.into());
1746 assert_eq!(pi.page.visit_count_remote, 1);
1747 assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1748
1749 conn.execute_cached(sql, &[(":row_id", &rid2)])?;
1751 conn.execute_cached(sql, &[(":row_id", &rid4)])?;
1752 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1755 assert_eq!(pi.page.visit_count_local, 0);
1756 assert_eq!(pi.page.last_visit_date_local, Timestamp(0));
1757 assert_eq!(pi.page.visit_count_remote, 0);
1758 assert_eq!(pi.page.last_visit_date_remote, Timestamp(0));
1759 Ok(())
1760 }
1761
1762 #[test]
1763 fn test_get_visited() -> Result<()> {
1764 error_support::init_for_tests();
1765 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1766
1767 let unicode_in_path = "http://www.example.com/tëst😀abc";
1768 let escaped_unicode_in_path = "http://www.example.com/t%C3%ABst%F0%9F%98%80abc";
1769
1770 let unicode_in_domain = "http://www.exämple😀123.com";
1771 let escaped_unicode_in_domain = "http://www.xn--exmple123-w2a24222l.com";
1772
1773 let to_add = [
1774 "https://www.example.com/1".to_string(),
1775 "https://www.example.com/12".to_string(),
1776 "https://www.example.com/123".to_string(),
1777 "https://www.example.com/1234".to_string(),
1778 "https://www.mozilla.com".to_string(),
1779 "https://www.firefox.com".to_string(),
1780 unicode_in_path.to_string() + "/1",
1781 escaped_unicode_in_path.to_string() + "/2",
1782 unicode_in_domain.to_string() + "/1",
1783 escaped_unicode_in_domain.to_string() + "/2",
1784 ];
1785
1786 for item in &to_add {
1787 apply_observation(
1788 &conn,
1789 VisitObservation::new(Url::parse(item).unwrap()).with_visit_type(VisitType::Link),
1790 )?;
1791 }
1792
1793 let to_search = [
1794 ("https://www.example.com".to_string(), false),
1795 ("https://www.example.com/1".to_string(), true),
1796 ("https://www.example.com/12".to_string(), true),
1797 ("https://www.example.com/123".to_string(), true),
1798 ("https://www.example.com/1234".to_string(), true),
1799 ("https://www.example.com/12345".to_string(), false),
1800 ("https://www.mozilla.com".to_string(), true),
1801 ("https://www.firefox.com".to_string(), true),
1802 ("https://www.mozilla.org".to_string(), false),
1803 ("https://www.example.com/1234".to_string(), true),
1805 ("https://www.example.com/12345".to_string(), false),
1806 (unicode_in_path.to_string() + "/1", true),
1809 (escaped_unicode_in_path.to_string() + "/2", true),
1810 (unicode_in_domain.to_string() + "/1", true),
1811 (escaped_unicode_in_domain.to_string() + "/2", true),
1812 (unicode_in_path.to_string() + "/2", true),
1814 (escaped_unicode_in_path.to_string() + "/1", true),
1815 (unicode_in_domain.to_string() + "/2", true),
1816 (escaped_unicode_in_domain.to_string() + "/1", true),
1817 ];
1818
1819 let urls = to_search
1820 .iter()
1821 .map(|(url, _expect)| Url::parse(url).unwrap())
1822 .collect::<Vec<_>>();
1823
1824 let visited = get_visited(&conn, urls).unwrap();
1825
1826 assert_eq!(visited.len(), to_search.len());
1827
1828 for (i, &did_see) in visited.iter().enumerate() {
1829 assert_eq!(
1830 did_see,
1831 to_search[i].1,
1832 "Wrong value in get_visited for '{}' (idx {}), want {}, have {}",
1833 to_search[i].0,
1834 i, to_search[i].1,
1836 did_see
1837 );
1838 }
1839 Ok(())
1840 }
1841
1842 #[test]
1843 fn test_get_visited_into() {
1844 error_support::init_for_tests();
1845 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
1846
1847 let u0 = Url::parse("https://www.example.com/1").unwrap();
1848 let u1 = Url::parse("https://www.example.com/12").unwrap();
1849 let u2 = Url::parse("https://www.example.com/123").unwrap();
1850 let u3 = Url::parse("https://www.example.com/1234").unwrap();
1851 let u4 = Url::parse("https://www.example.com/12345").unwrap();
1852
1853 let to_add = [(&u0, false), (&u1, false), (&u2, false), (&u3, true)];
1854 for (item, is_remote) in to_add {
1855 apply_observation(
1856 &conn,
1857 VisitObservation::new(item.clone())
1858 .with_visit_type(VisitType::Link)
1859 .with_is_remote(is_remote),
1860 )
1861 .unwrap();
1862 }
1863 insert_bookmark(
1866 &conn,
1867 crate::InsertableBookmark {
1868 parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
1869 position: crate::BookmarkPosition::Append,
1870 date_added: None,
1871 last_modified: None,
1872 guid: None,
1873 url: u4.clone(),
1874 title: Some("Title".to_string()),
1875 }
1876 .into(),
1877 )
1878 .unwrap();
1879
1880 let mut results = [false; 12];
1881
1882 let get_visited_request = [
1883 (2, u1.clone()),
1885 (1, u0),
1886 (4, u2),
1888 (6, Url::parse("https://www.example.com/123456").unwrap()),
1891 (8, u1),
1894 (10, u3),
1896 (11, u4),
1897 ];
1898
1899 get_visited_into(&conn, &get_visited_request, &mut results).unwrap();
1900 let expect = [
1901 false, true, true, false, true, false, false, false, true, false, true, false, ];
1914
1915 assert_eq!(expect, results);
1916 }
1917
1918 #[test]
1919 fn test_delete_visited() {
1920 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1921 let late: Timestamp = SystemTime::now().into();
1922 let early: Timestamp = (SystemTime::now() - Duration::from_secs(30)).into();
1923 let url1 = Url::parse("https://www.example.com/1").unwrap();
1924 let url2 = Url::parse("https://www.example.com/2").unwrap();
1925 let url3 = Url::parse("https://www.example.com/3").unwrap();
1926 let url4 = Url::parse("https://www.example.com/4").unwrap();
1927 let to_add = [
1929 (&url1, early),
1931 (&url1, late),
1932 (&url2, late),
1934 (&url3, early),
1936 (&url4, late),
1938 ];
1939
1940 for &(url, when) in &to_add {
1941 apply_observation(
1942 &conn,
1943 VisitObservation::new(url.clone())
1944 .with_at(when)
1945 .with_visit_type(VisitType::Link),
1946 )
1947 .expect("Should apply visit");
1948 }
1949 let pi = fetch_page_info(&conn, &url1)
1951 .expect("should work")
1952 .expect("should get the page");
1953 assert_eq!(pi.page.visit_count_local, 2);
1954
1955 let pi2 = fetch_page_info(&conn, &url2)
1956 .expect("should work")
1957 .expect("should get the page");
1958 assert_eq!(pi2.page.visit_count_local, 1);
1959
1960 let pi3 = fetch_page_info(&conn, &url3)
1961 .expect("should work")
1962 .expect("should get the page");
1963 assert_eq!(pi3.page.visit_count_local, 1);
1964
1965 let pi4 = fetch_page_info(&conn, &url4)
1966 .expect("should work")
1967 .expect("should get the page");
1968 assert_eq!(pi4.page.visit_count_local, 1);
1969
1970 conn.execute_cached(
1971 &format!(
1972 "UPDATE moz_places set sync_status = {}
1973 WHERE url = 'https://www.example.com/4'",
1974 (SyncStatus::Normal as u8)
1975 ),
1976 [],
1977 )
1978 .expect("should work");
1979
1980 delete_visits_between(&conn, late, Timestamp::now()).expect("should work");
1982 let pi = fetch_page_info(&conn, &url1)
1984 .expect("should work")
1985 .expect("should get the page");
1986 assert_eq!(pi.page.visit_count_local, 1);
1987
1988 assert!(fetch_page_info(&conn, &url2)
1990 .expect("should work")
1991 .is_none());
1992
1993 let pi3 = fetch_page_info(&conn, &url3)
1995 .expect("should work")
1996 .expect("should get the page");
1997 assert_eq!(pi3.page.visit_count_local, 1);
1998
1999 assert!(fetch_page_info(&conn, &url4)
2001 .expect("should work")
2002 .is_none());
2003 assert_eq!(get_tombstone_count(&conn), 1);
2005 }
2008
2009 #[test]
2010 fn test_change_counter() -> Result<()> {
2011 error_support::init_for_tests();
2012 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2013 let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2014 apply_observation(
2016 &conn,
2017 VisitObservation::new(pi.url.clone()).with_title(Some("new title".into())),
2018 )?;
2019 pi = fetch_page_info(&conn, &pi.url)?
2020 .expect("page should exist")
2021 .page;
2022 assert_eq!(pi.title, "new title");
2023 assert_eq!(pi.preview_image_url, None);
2024 assert_eq!(pi.sync_change_counter, 2);
2025 apply_observation(
2027 &conn,
2028 VisitObservation::new(pi.url.clone()).with_preview_image_url(Some(
2029 Url::parse("https://www.example.com/preview.png").unwrap(),
2030 )),
2031 )?;
2032 pi = fetch_page_info(&conn, &pi.url)?
2033 .expect("page should exist")
2034 .page;
2035 assert_eq!(pi.title, "new title");
2036 assert_eq!(
2037 pi.preview_image_url,
2038 Some(Url::parse("https://www.example.com/preview.png").expect("parsed"))
2039 );
2040 assert_eq!(pi.sync_change_counter, 2);
2041 Ok(())
2042 }
2043
2044 #[test]
2045 fn test_status_columns() -> Result<()> {
2046 error_support::init_for_tests();
2047 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2048 let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2050 assert_eq!(pi.sync_change_counter, 1);
2051 conn.execute_cached(
2052 "UPDATE moz_places
2053 SET frecency = 100
2054 WHERE id = :id",
2055 &[(":id", &pi.row_id)],
2056 )?;
2057 let mut pi2 = get_observed_page(&mut conn, "http://example.com/2")?;
2059 conn.execute_cached(
2060 "UPDATE moz_places
2061 SET sync_status = :status,
2062 sync_change_counter = 0,
2063 frecency = 50
2064 WHERE id = :id",
2065 &[
2066 (":status", &(SyncStatus::New as u8) as &dyn rusqlite::ToSql),
2067 (":id", &pi2.row_id),
2068 ],
2069 )?;
2070
2071 let mut pi3 = get_observed_page(&mut conn, "http://example.com/3")?;
2074 conn.execute_cached(
2075 "UPDATE moz_places
2076 SET sync_status = :status,
2077 sync_change_counter = 1,
2078 frecency = 10
2079 WHERE id = :id",
2080 &[
2081 (":status", &(SyncStatus::New as u8) as &dyn ToSql),
2082 (":id", &pi3.row_id),
2083 ],
2084 )?;
2085
2086 let outgoing = fetch_outgoing(&conn, 2, 3)?;
2087 assert_eq!(outgoing.len(), 2, "should have restricted to the limit");
2088 assert!(outgoing[0].envelope.id != outgoing[1].envelope.id);
2090 assert!(outgoing[0].envelope.id == pi.guid || outgoing[0].envelope.id == pi2.guid);
2091 assert!(outgoing[1].envelope.id == pi.guid || outgoing[1].envelope.id == pi2.guid);
2092 finish_outgoing(&conn)?;
2093
2094 pi = fetch_page_info(&conn, &pi.url)?
2095 .expect("page should exist")
2096 .page;
2097 assert_eq!(pi.sync_change_counter, 0);
2098 pi2 = fetch_page_info(&conn, &pi2.url)?
2099 .expect("page should exist")
2100 .page;
2101 assert_eq!(pi2.sync_change_counter, 0);
2102 assert_eq!(pi2.sync_status, SyncStatus::Normal);
2103
2104 pi3 = fetch_page_info(&conn, &pi3.url)?
2107 .expect("page should exist")
2108 .page;
2109 assert_eq!(pi3.sync_change_counter, 0);
2110 assert_eq!(pi3.sync_status, SyncStatus::Normal);
2111 Ok(())
2112 }
2113
2114 #[test]
2115 fn test_delete_visits_for() -> Result<()> {
2116 use crate::storage::bookmarks::{
2117 self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2118 };
2119
2120 let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2121
2122 struct TestPage {
2123 href: &'static str,
2124 synced: bool,
2125 bookmark_title: Option<&'static str>,
2126 keyword: Option<&'static str>,
2127 }
2128
2129 fn page_has_tombstone(conn: &PlacesDb, guid: &SyncGuid) -> Result<bool> {
2130 let exists = conn
2131 .try_query_one::<bool, _>(
2132 "SELECT EXISTS(SELECT 1 FROM moz_places_tombstones
2133 WHERE guid = :guid)",
2134 rusqlite::named_params! { ":guid" : guid },
2135 false,
2136 )?
2137 .unwrap_or_default();
2138 Ok(exists)
2139 }
2140
2141 fn page_has_visit_tombstones(conn: &PlacesDb, page_id: RowId) -> Result<bool> {
2142 let exists = conn
2143 .try_query_one::<bool, _>(
2144 "SELECT EXISTS(SELECT 1 FROM moz_historyvisit_tombstones
2145 WHERE place_id = :page_id)",
2146 rusqlite::named_params! { ":page_id": page_id },
2147 false,
2148 )?
2149 .unwrap_or_default();
2150 Ok(exists)
2151 }
2152
2153 let pages = &[
2154 TestPage {
2157 href: "http://example.com/a",
2158 synced: true,
2159 bookmark_title: Some("A"),
2160 keyword: None,
2161 },
2162 TestPage {
2165 href: "http://example.com/b",
2166 synced: true,
2167 bookmark_title: None,
2168 keyword: None,
2169 },
2170 TestPage {
2173 href: "http://example.com/c",
2174 synced: false,
2175 bookmark_title: None,
2176 keyword: Some("one"),
2177 },
2178 TestPage {
2181 href: "http://example.com/d",
2182 synced: false,
2183 bookmark_title: None,
2184 keyword: None,
2185 },
2186 ];
2187 for page in pages {
2188 let url = Url::parse(page.href)?;
2189 let obs = VisitObservation::new(url.clone())
2190 .with_visit_type(VisitType::Link)
2191 .with_at(Some(SystemTime::now().into()));
2192 apply_observation(&db, obs)?;
2193
2194 if page.synced {
2195 db.execute_cached(
2196 &format!(
2197 "UPDATE moz_places
2198 SET sync_status = {}
2199 WHERE url_hash = hash(:url) AND
2200 url = :url",
2201 (SyncStatus::Normal as u8)
2202 ),
2203 &[(":url", &url.as_str())],
2204 )?;
2205 }
2206
2207 if let Some(title) = page.bookmark_title {
2208 bookmarks::insert_bookmark(
2209 &db,
2210 InsertableBookmark {
2211 parent_guid: BookmarkRootGuid::Unfiled.into(),
2212 position: BookmarkPosition::Append,
2213 date_added: None,
2214 last_modified: None,
2215 guid: None,
2216 url: url.clone(),
2217 title: Some(title.to_owned()),
2218 }
2219 .into(),
2220 )?;
2221 }
2222
2223 if let Some(keyword) = page.keyword {
2224 db.execute_cached(
2227 "INSERT INTO moz_keywords(place_id, keyword)
2228 SELECT id, :keyword
2229 FROM moz_places
2230 WHERE url_hash = hash(:url) AND
2231 url = :url",
2232 &[(":url", &url.as_str()), (":keyword", &keyword)],
2233 )?;
2234 }
2235
2236 let (info, _) =
2238 fetch_visits(&db, &url, 0)?.expect("Should return visits for test page");
2239 delete_visits_for(&db, &info.guid)?;
2240
2241 match (
2242 page.synced,
2243 page.bookmark_title.is_some() || page.keyword.is_some(),
2244 ) {
2245 (true, true) => {
2246 let (_, visits) = fetch_visits(&db, &url, 0)?
2247 .expect("Shouldn't delete synced page with foreign count");
2248 assert!(
2249 visits.is_empty(),
2250 "Should delete all visits from synced page with foreign count"
2251 );
2252 assert!(
2253 !page_has_tombstone(&db, &info.guid)?,
2254 "Shouldn't insert tombstone for synced page with foreign count"
2255 );
2256 assert!(
2257 page_has_visit_tombstones(&db, info.row_id)?,
2258 "Should insert visit tombstones for synced page with foreign count"
2259 );
2260 }
2261 (true, false) => {
2262 assert!(
2263 fetch_visits(&db, &url, 0)?.is_none(),
2264 "Should delete synced page"
2265 );
2266 assert!(
2267 page_has_tombstone(&db, &info.guid)?,
2268 "Should insert tombstone for synced page"
2269 );
2270 assert!(
2271 !page_has_visit_tombstones(&db, info.row_id)?,
2272 "Shouldn't insert visit tombstones for synced page"
2273 );
2274 }
2275 (false, true) => {
2276 let (_, visits) = fetch_visits(&db, &url, 0)?
2277 .expect("Shouldn't delete page with foreign count");
2278 assert!(
2279 visits.is_empty(),
2280 "Should delete all visits from page with foreign count"
2281 );
2282 assert!(
2283 !page_has_tombstone(&db, &info.guid)?,
2284 "Shouldn't insert tombstone for page with foreign count"
2285 );
2286 assert!(
2287 !page_has_visit_tombstones(&db, info.row_id)?,
2288 "Shouldn't insert visit tombstones for page with foreign count"
2289 );
2290 }
2291 (false, false) => {
2292 assert!(fetch_visits(&db, &url, 0)?.is_none(), "Should delete page");
2293 assert!(
2294 !page_has_tombstone(&db, &info.guid)?,
2295 "Shouldn't insert tombstone for page"
2296 );
2297 assert!(
2298 !page_has_visit_tombstones(&db, info.row_id)?,
2299 "Shouldn't insert visit tombstones for page"
2300 );
2301 }
2302 }
2303 }
2304
2305 Ok(())
2306 }
2307
2308 #[test]
2309 fn test_tombstones() -> Result<()> {
2310 error_support::init_for_tests();
2311 let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2312 let url = Url::parse("https://example.com")?;
2313 let obs = VisitObservation::new(url.clone())
2314 .with_visit_type(VisitType::Link)
2315 .with_at(Some(SystemTime::now().into()));
2316 apply_observation(&db, obs)?;
2317 let guid = url_to_guid(&db, &url)?.expect("should exist");
2318
2319 delete_visits_for(&db, &guid)?;
2320
2321 assert_eq!(get_tombstone_count(&db), 0);
2323
2324 let obs = VisitObservation::new(url.clone())
2325 .with_visit_type(VisitType::Link)
2326 .with_at(Some(SystemTime::now().into()));
2327 apply_observation(&db, obs)?;
2328 let new_guid = url_to_guid(&db, &url)?.expect("should exist");
2329
2330 db.execute_cached(
2332 &format!(
2333 "UPDATE moz_places
2334 SET sync_status = {}
2335 WHERE guid = :guid",
2336 (SyncStatus::Normal as u8)
2337 ),
2338 &[(":guid", &new_guid)],
2339 )?;
2340 delete_visits_for(&db, &new_guid)?;
2341 assert_eq!(get_tombstone_count(&db), 1);
2342 Ok(())
2343 }
2344
2345 #[test]
2346 fn test_reset() -> Result<()> {
2347 fn mark_all_as_synced(db: &PlacesDb) -> Result<()> {
2348 db.execute_cached(
2349 &format!(
2350 "UPDATE moz_places set sync_status = {}",
2351 (SyncStatus::Normal as u8)
2352 ),
2353 [],
2354 )?;
2355 Ok(())
2356 }
2357
2358 error_support::init_for_tests();
2359 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2360
2361 put_meta(&conn, GLOBAL_SYNCID_META_KEY, &"syncAAAAAAAA")?;
2363 put_meta(&conn, COLLECTION_SYNCID_META_KEY, &"syncBBBBBBBB")?;
2364 put_meta(&conn, LAST_SYNC_META_KEY, &12345)?;
2365
2366 delete_everything(&conn)?;
2369
2370 let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2371 mark_all_as_synced(&conn)?;
2372 pi = fetch_page_info(&conn, &pi.url)?
2373 .expect("page should exist")
2374 .page;
2375 assert_eq!(pi.sync_change_counter, 1);
2376 assert_eq!(pi.sync_status, SyncStatus::Normal);
2377
2378 let sync_ids = CollSyncIds {
2379 global: SyncGuid::random(),
2380 coll: SyncGuid::random(),
2381 };
2382 history_sync::reset(&conn, &EngineSyncAssociation::Connected(sync_ids.clone()))?;
2383
2384 assert_eq!(
2385 get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?,
2386 Some(sync_ids.global)
2387 );
2388 assert_eq!(
2389 get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2390 Some(sync_ids.coll)
2391 );
2392 assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2393 assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2394
2395 pi = fetch_page_info(&conn, &pi.url)?
2396 .expect("page should exist")
2397 .page;
2398 assert_eq!(pi.sync_change_counter, 0);
2399 assert_eq!(pi.sync_status, SyncStatus::New);
2400 let outgoing = fetch_outgoing(&conn, 100, 100)?;
2402 assert_eq!(outgoing.len(), 1);
2403
2404 mark_all_as_synced(&conn)?;
2405 assert!(fetch_outgoing(&conn, 100, 100)?.is_empty());
2406 history_sync::reset(&conn, &EngineSyncAssociation::Disconnected)?;
2411
2412 assert_eq!(get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?, None);
2413 assert_eq!(
2414 get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2415 None
2416 );
2417 assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2418 assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2419
2420 Ok(())
2421 }
2422
2423 #[test]
2424 fn test_fetch_visits() -> Result<()> {
2425 error_support::init_for_tests();
2426 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2427 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2428 assert_eq!(fetch_visits(&conn, &pi.url, 0).unwrap().unwrap().1.len(), 0);
2429 assert_eq!(fetch_visits(&conn, &pi.url, 1).unwrap().unwrap().1.len(), 1);
2430 Ok(())
2431 }
2432
2433 #[test]
2434 fn test_apply_synced_reconciliation() -> Result<()> {
2435 error_support::init_for_tests();
2436 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2437 let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2438 assert_eq!(pi.sync_status, SyncStatus::New);
2439 assert_eq!(pi.sync_change_counter, 1);
2440 apply_synced_reconciliation(&conn, &pi.guid)?;
2441 pi = fetch_page_info(&conn, &pi.url)?
2442 .expect("page should exist")
2443 .page;
2444 assert_eq!(pi.sync_status, SyncStatus::Normal);
2445 assert_eq!(pi.sync_change_counter, 0);
2446 Ok(())
2447 }
2448
2449 #[test]
2450 fn test_apply_synced_deletion_new() -> Result<()> {
2451 error_support::init_for_tests();
2452 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2453 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2454 assert_eq!(pi.sync_status, SyncStatus::New);
2455 apply_synced_deletion(&conn, &pi.guid)?;
2456 assert!(
2457 fetch_page_info(&conn, &pi.url)?.is_none(),
2458 "should have been deleted"
2459 );
2460 assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2461 Ok(())
2462 }
2463
2464 #[test]
2465 fn test_apply_synced_deletion_normal() -> Result<()> {
2466 error_support::init_for_tests();
2467 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2468 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2469 assert_eq!(pi.sync_status, SyncStatus::New);
2470 conn.execute_cached(
2471 &format!(
2472 "UPDATE moz_places set sync_status = {}",
2473 (SyncStatus::Normal as u8)
2474 ),
2475 [],
2476 )?;
2477
2478 apply_synced_deletion(&conn, &pi.guid)?;
2479 assert!(
2480 fetch_page_info(&conn, &pi.url)?.is_none(),
2481 "should have been deleted"
2482 );
2483 assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2484 Ok(())
2485 }
2486
2487 #[test]
2488 fn test_apply_synced_deletions_deletes_visits_but_not_page_if_bookmark_exists() -> Result<()> {
2489 error_support::init_for_tests();
2490 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2491 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2492 let item = InsertableItem::Bookmark {
2493 b: crate::InsertableBookmark {
2494 parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
2495 position: crate::BookmarkPosition::Append,
2496 date_added: None,
2497 last_modified: None,
2498 guid: None,
2499 url: pi.url.clone(),
2500 title: Some("Title".to_string()),
2501 },
2502 };
2503 insert_bookmark(&conn, item).unwrap();
2504 apply_synced_deletion(&conn, &pi.guid)?;
2505 let page_info =
2506 fetch_page_info(&conn, &pi.url)?.expect("The places entry should have remained");
2507 assert!(
2508 page_info.last_visit_id.is_none(),
2509 "Should have no more visits"
2510 );
2511 Ok(())
2512 }
2513
2514 fn assert_tombstones(c: &PlacesDb, expected: &[(RowId, Timestamp)]) {
2515 let mut expected: Vec<(RowId, Timestamp)> = expected.into();
2516 expected.sort();
2517 let mut tombstones = c
2518 .query_rows_and_then(
2519 "SELECT place_id, visit_date FROM moz_historyvisit_tombstones",
2520 [],
2521 |row| -> Result<_> { Ok((row.get::<_, RowId>(0)?, row.get::<_, Timestamp>(1)?)) },
2522 )
2523 .unwrap();
2524 tombstones.sort();
2525 assert_eq!(expected, tombstones);
2526 }
2527
2528 #[test]
2529 fn test_visit_tombstones() {
2530 use url::Url;
2531 error_support::init_for_tests();
2532 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2533 let now = Timestamp::now();
2534
2535 let urls = &[
2536 Url::parse("http://example.com/1").unwrap(),
2537 Url::parse("http://example.com/2").unwrap(),
2538 ];
2539
2540 let dates = &[
2541 Timestamp(now.0 - 10000),
2542 Timestamp(now.0 - 5000),
2543 Timestamp(now.0),
2544 ];
2545 for url in urls {
2546 for &date in dates {
2547 get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2548 }
2549 }
2550 delete_place_visit_at_time(&conn, &urls[0], dates[1]).unwrap();
2551 delete_visits_between(&conn, Timestamp(now.0 - 4000), Timestamp::now()).unwrap();
2553
2554 let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2555 assert_eq!(
2556 visits0,
2557 &[FetchedVisit {
2558 is_local: true,
2559 visit_date: dates[0],
2560 visit_type: Some(VisitType::Link)
2561 },]
2562 );
2563
2564 assert!(
2565 !visits0.iter().any(|v| v.visit_date == dates[1]),
2566 "Shouldn't have deleted visit"
2567 );
2568
2569 let (info1, mut visits1) = fetch_visits(&conn, &urls[1], 100).unwrap().unwrap();
2570 visits1.sort_by_key(|v| v.visit_date);
2571 assert_eq!(
2574 visits1,
2575 &[
2576 FetchedVisit {
2577 is_local: true,
2578 visit_date: dates[0],
2579 visit_type: Some(VisitType::Link)
2580 },
2581 FetchedVisit {
2582 is_local: true,
2583 visit_date: dates[1],
2584 visit_type: Some(VisitType::Link)
2585 },
2586 ]
2587 );
2588
2589 apply_synced_visits(
2591 &conn,
2592 &info0.guid,
2593 &info0.url,
2594 &Some(info0.title.clone()),
2595 &dates
2597 .iter()
2598 .map(|&d| HistoryRecordVisit {
2599 date: d.into(),
2600 transition: VisitType::Link as u8,
2601 unknown_fields: UnknownFields::new(),
2602 })
2603 .collect::<Vec<_>>(),
2604 &UnknownFields::new(),
2605 )
2606 .unwrap();
2607
2608 let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2609 assert_eq!(
2610 visits0,
2611 &[FetchedVisit {
2612 is_local: true,
2613 visit_date: dates[0],
2614 visit_type: Some(VisitType::Link)
2615 }]
2616 );
2617
2618 assert_tombstones(
2619 &conn,
2620 &[
2621 (info0.row_id, dates[1]),
2622 (info0.row_id, dates[2]),
2623 (info1.row_id, dates[2]),
2624 ],
2625 );
2626
2627 delete_place_visit_at_time(&conn, &urls[0], dates[0]).unwrap();
2630
2631 assert!(fetch_visits(&conn, &urls[0], 100).unwrap().is_none());
2632
2633 assert_tombstones(&conn, &[(info1.row_id, dates[2])]);
2634 }
2635
2636 #[test]
2637 fn test_delete_local() {
2638 use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
2639 use crate::storage::bookmarks::{
2640 self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark, InsertableItem,
2641 };
2642 use url::Url;
2643 error_support::init_for_tests();
2644 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2645 let ts = Timestamp::now().0 - 5_000_000;
2646 for o in 0..10 {
2648 for i in 0..11 {
2649 for t in 0..3 {
2650 get_custom_observed_page(
2651 &mut conn,
2652 &format!("http://www.example{}.com/{}", o, i),
2653 |obs| obs.with_at(Timestamp(ts + t * 1000 + i * 10_000 + o * 100_000)),
2654 )
2655 .unwrap();
2656 }
2657 }
2658 }
2659 let b0 = (
2661 SyncGuid::from("aaaaaaaaaaaa"),
2662 Url::parse("http://www.example3.com/5").unwrap(),
2663 );
2664 let b1 = (
2665 SyncGuid::from("bbbbbbbbbbbb"),
2666 Url::parse("http://www.example6.com/10").unwrap(),
2667 );
2668 let b2 = (
2669 SyncGuid::from("cccccccccccc"),
2670 Url::parse("http://www.example9.com/4").unwrap(),
2671 );
2672 for (guid, url) in &[&b0, &b1, &b2] {
2673 bookmarks::insert_bookmark(
2674 &conn,
2675 InsertableItem::Bookmark {
2676 b: InsertableBookmark {
2677 parent_guid: BookmarkRootGuid::Unfiled.into(),
2678 position: BookmarkPosition::Append,
2679 date_added: None,
2680 last_modified: None,
2681 guid: Some(guid.clone()),
2682 url: url.clone(),
2683 title: None,
2684 },
2685 },
2686 )
2687 .unwrap();
2688 }
2689
2690 conn.execute_all(&[
2692 &format!(
2693 "UPDATE moz_places set sync_status = {}",
2694 (SyncStatus::Normal as u8)
2695 ),
2696 &format!(
2697 "UPDATE moz_bookmarks set syncStatus = {}",
2698 (SyncStatus::Normal as u8)
2699 ),
2700 ])
2701 .unwrap();
2702
2703 delete_visits_for(
2705 &conn,
2706 &url_to_guid(&conn, &Url::parse("http://www.example8.com/5").unwrap())
2707 .unwrap()
2708 .unwrap(),
2709 )
2710 .unwrap();
2711
2712 delete_place_visit_at_time(
2713 &conn,
2714 &Url::parse("http://www.example10.com/5").unwrap(),
2715 Timestamp(ts + 5 * 10_000 + 10 * 100_000),
2716 )
2717 .unwrap();
2718
2719 assert!(bookmarks::delete_bookmark(&conn, &b0.0).unwrap());
2720
2721 delete_everything(&conn).unwrap();
2722
2723 let places = conn
2724 .query_rows_and_then(
2725 "SELECT * FROM moz_places ORDER BY url ASC",
2726 [],
2727 PageInfo::from_row,
2728 )
2729 .unwrap();
2730 assert_eq!(places.len(), 2);
2731 assert_eq!(places[0].url, b1.1);
2732 assert_eq!(places[1].url, b2.1);
2733 for p in &places {
2734 assert_eq!(
2735 p.frecency,
2736 DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
2737 );
2738 assert_eq!(p.visit_count_local, 0);
2739 assert_eq!(p.visit_count_remote, 0);
2740 assert_eq!(p.last_visit_date_local, Timestamp(0));
2741 assert_eq!(p.last_visit_date_remote, Timestamp(0));
2742 }
2743
2744 let counts_sql = [
2745 (0i64, "SELECT COUNT(*) FROM moz_historyvisits"),
2746 (2, "SELECT COUNT(*) FROM moz_origins"),
2747 (7, "SELECT COUNT(*) FROM moz_bookmarks"), (1, "SELECT COUNT(*) FROM moz_bookmarks_deleted"),
2749 (0, "SELECT COUNT(*) FROM moz_historyvisit_tombstones"),
2750 (0, "SELECT COUNT(*) FROM moz_places_tombstones"),
2751 ];
2752 for (want, query) in &counts_sql {
2753 assert_eq!(
2754 *want,
2755 conn.query_one::<i64>(query).unwrap(),
2756 "Unexpected value for {}",
2757 query
2758 );
2759 }
2760 }
2761
2762 #[test]
2763 fn test_delete_everything() {
2764 use crate::storage::bookmarks::{
2765 self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2766 };
2767 use url::Url;
2768 error_support::init_for_tests();
2769 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2770 let start = Timestamp::now();
2771
2772 let urls = &[
2773 Url::parse("http://example.com/1").unwrap(),
2774 Url::parse("http://example.com/2").unwrap(),
2775 Url::parse("http://example.com/3").unwrap(),
2776 ];
2777
2778 let dates = &[
2779 Timestamp(start.0 - 10000),
2780 Timestamp(start.0 - 5000),
2781 Timestamp(start.0),
2782 ];
2783
2784 for url in urls {
2785 for &date in dates {
2786 get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2787 }
2788 }
2789
2790 bookmarks::insert_bookmark(
2791 &conn,
2792 InsertableBookmark {
2793 parent_guid: BookmarkRootGuid::Unfiled.into(),
2794 position: BookmarkPosition::Append,
2795 date_added: None,
2796 last_modified: None,
2797 guid: Some("bookmarkAAAA".into()),
2798 url: urls[2].clone(),
2799 title: Some("A".into()),
2800 }
2801 .into(),
2802 )
2803 .expect("Should insert bookmark with URL 3");
2804
2805 conn.execute(
2806 "WITH entries(url, input) AS (
2807 VALUES(:url1, 'hi'), (:url3, 'bye')
2808 )
2809 INSERT INTO moz_inputhistory(place_id, input, use_count)
2810 SELECT h.id, e.input, 1
2811 FROM entries e
2812 JOIN moz_places h ON h.url_hash = hash(e.url) AND
2813 h.url = e.url",
2814 &[(":url1", &urls[1].as_str()), (":url3", &urls[2].as_str())],
2815 )
2816 .expect("Should insert autocomplete history entries");
2817
2818 delete_everything(&conn).expect("Should delete everything except URL 3");
2819
2820 std::thread::sleep(std::time::Duration::from_millis(50));
2821
2822 let mut places_stmt = conn.prepare("SELECT url FROM moz_places").unwrap();
2825 let remaining_urls: Vec<String> = places_stmt
2826 .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2827 .expect("Should fetch remaining URLs")
2828 .map(std::result::Result::unwrap)
2829 .collect();
2830 assert_eq!(remaining_urls, &["http://example.com/3"]);
2831
2832 let mut input_stmt = conn.prepare("SELECT input FROM moz_inputhistory").unwrap();
2833 let remaining_inputs: Vec<String> = input_stmt
2834 .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2835 .expect("Should fetch remaining autocomplete history entries")
2836 .map(std::result::Result::unwrap)
2837 .collect();
2838 assert_eq!(remaining_inputs, &["bye"]);
2839
2840 bookmarks::delete_bookmark(&conn, &"bookmarkAAAA".into())
2841 .expect("Should delete bookmark with URL 3");
2842
2843 delete_everything(&conn).expect("Should delete all URLs");
2844
2845 assert_eq!(
2846 0,
2847 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2848 .unwrap(),
2849 );
2850
2851 apply_synced_visits(
2852 &conn,
2853 &SyncGuid::random(),
2854 &url::Url::parse("http://www.example.com/123").unwrap(),
2855 &None,
2856 &[
2857 HistoryRecordVisit {
2858 date: Timestamp::now().into(),
2860 transition: VisitType::Link as u8,
2861 unknown_fields: UnknownFields::new(),
2862 },
2863 HistoryRecordVisit {
2864 date: start.into(),
2866 transition: VisitType::Link as u8,
2867 unknown_fields: UnknownFields::new(),
2868 },
2869 ],
2870 &UnknownFields::new(),
2871 )
2872 .unwrap();
2873 assert_eq!(
2874 1,
2875 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2876 .unwrap(),
2877 );
2878 assert_eq!(
2880 1,
2881 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2882 .unwrap(),
2883 );
2884
2885 apply_synced_visits(
2887 &conn,
2888 &SyncGuid::random(),
2889 &url::Url::parse("http://www.example.com/1234").unwrap(),
2890 &None,
2891 &[HistoryRecordVisit {
2892 date: start.into(),
2893 transition: VisitType::Link as u8,
2894 unknown_fields: UnknownFields::new(),
2895 }],
2896 &UnknownFields::new(),
2897 )
2898 .unwrap();
2899 assert_eq!(
2901 1,
2902 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2903 .unwrap(),
2904 );
2905 assert_eq!(
2906 1,
2907 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2908 .unwrap(),
2909 );
2910 }
2911
2912 #[test]
2914 fn test_delete_everything_deletes_origins() {
2915 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2916
2917 let u = Url::parse("https://www.reddit.com/r/climbing").expect("Should parse URL");
2918 let ts = Timestamp::now().0 - 5_000_000;
2919 let obs = VisitObservation::new(u)
2920 .with_visit_type(VisitType::Link)
2921 .with_at(Timestamp(ts));
2922 apply_observation(&conn, obs).expect("Should apply observation");
2923
2924 delete_everything(&conn).expect("Should delete everything");
2925
2926 let origin_count = conn
2928 .query_one::<i64>("SELECT COUNT(*) FROM moz_origins")
2929 .expect("Should fetch origin count");
2930 assert_eq!(0, origin_count);
2931 }
2932
2933 #[test]
2934 fn test_apply_observation_updates_origins() {
2935 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2936
2937 let obs_for_a = VisitObservation::new(
2938 Url::parse("https://example1.com/a").expect("Should parse URL A"),
2939 )
2940 .with_visit_type(VisitType::Link)
2941 .with_at(Timestamp(Timestamp::now().0 - 5_000_000));
2942 apply_observation(&conn, obs_for_a).expect("Should apply observation for A");
2943
2944 let obs_for_b = VisitObservation::new(
2945 Url::parse("https://example2.com/b").expect("Should parse URL B"),
2946 )
2947 .with_visit_type(VisitType::Link)
2948 .with_at(Timestamp(Timestamp::now().0 - 2_500_000));
2949 apply_observation(&conn, obs_for_b).expect("Should apply observation for B");
2950
2951 let mut origins = conn
2952 .prepare("SELECT host FROM moz_origins")
2953 .expect("Should prepare origins statement")
2954 .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2955 .expect("Should fetch all origins")
2956 .map(|r| r.expect("Should get origin from row"))
2957 .collect::<Vec<_>>();
2958 origins.sort();
2959 assert_eq!(origins, &["example1.com", "example2.com",]);
2960 }
2961
2962 #[test]
2963 fn test_preview_url() {
2964 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2965
2966 let url1 = Url::parse("https://www.example.com/").unwrap();
2967 assert!(apply_observation(
2969 &conn,
2970 VisitObservation::new(url1.clone()).with_preview_image_url(Some(
2971 Url::parse("https://www.example.com/image.png").unwrap()
2972 ))
2973 )
2974 .unwrap()
2975 .is_none());
2976
2977 let mut db_preview_url = conn
2979 .query_row_and_then_cachable(
2980 "SELECT preview_image_url FROM moz_places WHERE id = 1",
2981 [],
2982 |row| row.get(0),
2983 false,
2984 )
2985 .unwrap();
2986 assert_eq!(
2987 Some("https://www.example.com/image.png".to_string()),
2988 db_preview_url
2989 );
2990
2991 let visit_id = apply_observation(
2993 &conn,
2994 VisitObservation::new(url1).with_visit_type(VisitType::Link),
2995 )
2996 .unwrap();
2997 assert!(visit_id.is_some());
2998
2999 db_preview_url = conn
3000 .query_row_and_then_cachable(
3001 "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3002 &[(":id", &visit_id.unwrap() as &dyn ToSql)],
3003 |row| row.get(0),
3004 false,
3005 )
3006 .unwrap();
3007 assert_eq!(
3008 Some("https://www.example.com/image.png".to_string()),
3009 db_preview_url
3010 );
3011
3012 let another_visit_id = apply_observation(
3014 &conn,
3015 VisitObservation::new(Url::parse("https://www.example.com/another/").unwrap())
3016 .with_preview_image_url(Some(
3017 Url::parse("https://www.example.com/funky/image.png").unwrap(),
3018 ))
3019 .with_visit_type(VisitType::Link),
3020 )
3021 .unwrap();
3022 assert!(another_visit_id.is_some());
3023
3024 db_preview_url = conn
3025 .query_row_and_then_cachable(
3026 "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3027 &[(":id", &another_visit_id.unwrap())],
3028 |row| row.get(0),
3029 false,
3030 )
3031 .unwrap();
3032 assert_eq!(
3033 Some("https://www.example.com/funky/image.png".to_string()),
3034 db_preview_url
3035 );
3036 }
3037
3038 #[test]
3039 fn test_long_strings() {
3040 error_support::init_for_tests();
3041 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3042 let mut url = "http://www.example.com".to_string();
3043 while url.len() < crate::storage::URL_LENGTH_MAX {
3044 url += "/garbage";
3045 }
3046 let maybe_row = apply_observation(
3047 &conn,
3048 VisitObservation::new(Url::parse(&url).unwrap())
3049 .with_visit_type(VisitType::Link)
3050 .with_at(Timestamp::now()),
3051 )
3052 .unwrap();
3053 assert!(maybe_row.is_none(), "Shouldn't insert overlong URL");
3054
3055 let maybe_row_preview = apply_observation(
3056 &conn,
3057 VisitObservation::new(Url::parse("https://www.example.com/").unwrap())
3058 .with_visit_type(VisitType::Link)
3059 .with_preview_image_url(Url::parse(&url).unwrap()),
3060 )
3061 .unwrap();
3062 assert!(
3063 maybe_row_preview.is_some(),
3064 "Shouldn't avoid a visit observation due to an overly long preview url"
3065 );
3066
3067 let mut title = "example 1 2 3".to_string();
3068 while title.len() < crate::storage::TITLE_LENGTH_MAX + 10 {
3070 title += " test test";
3071 }
3072 let maybe_visit_row = apply_observation(
3073 &conn,
3074 VisitObservation::new(Url::parse("http://www.example.com/123").unwrap())
3075 .with_title(title.clone())
3076 .with_visit_type(VisitType::Link)
3077 .with_at(Timestamp::now()),
3078 )
3079 .unwrap();
3080
3081 assert!(maybe_visit_row.is_some());
3082 let db_title: String = conn
3083 .query_row_and_then_cachable(
3084 "SELECT h.title FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3085 &[(":id", &maybe_visit_row.unwrap())],
3086 |row| row.get(0),
3087 false,
3088 )
3089 .unwrap();
3090 assert_eq!(db_title.len(), crate::storage::TITLE_LENGTH_MAX);
3092 assert!(title.starts_with(&db_title));
3093 }
3094
3095 #[test]
3096 fn test_get_visit_page_with_bound() {
3097 use std::time::SystemTime;
3098 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3099 let now: Timestamp = SystemTime::now().into();
3100 let now_u64 = now.0;
3101 let now_i64 = now.0 as i64;
3102 let to_add = [
3104 (
3105 "https://www.example.com/0",
3106 "older 2",
3107 now_u64 - 200_200,
3108 false,
3109 (true, false),
3110 ),
3111 (
3112 "https://www.example.com/1",
3113 "older 1",
3114 now_u64 - 200_100,
3115 true,
3116 (true, false),
3117 ),
3118 (
3119 "https://www.example.com/2",
3120 "same time",
3121 now_u64 - 200_000,
3122 false,
3123 (true, false),
3124 ),
3125 (
3126 "https://www.example.com/3",
3127 "same time",
3128 now_u64 - 200_000,
3129 false,
3130 (true, false),
3131 ),
3132 (
3133 "https://www.example.com/4",
3134 "same time",
3135 now_u64 - 200_000,
3136 false,
3137 (true, false),
3138 ),
3139 (
3140 "https://www.example.com/5",
3141 "same time",
3142 now_u64 - 200_000,
3143 false,
3144 (true, false),
3145 ),
3146 (
3147 "https://www.example.com/6",
3148 "same time",
3149 now_u64 - 200_000,
3150 false,
3151 (true, false),
3152 ),
3153 (
3154 "https://www.example.com/7",
3155 "same time",
3156 now_u64 - 200_000,
3157 false,
3158 (true, false),
3159 ),
3160 (
3161 "https://www.example.com/8",
3162 "same time",
3163 now_u64 - 200_000,
3164 false,
3165 (true, false),
3166 ),
3167 (
3168 "https://www.example.com/9",
3169 "same time",
3170 now_u64 - 200_000,
3171 false,
3172 (true, false),
3173 ),
3174 (
3175 "https://www.example.com/10",
3176 "more recent 2",
3177 now_u64 - 199_000,
3178 false,
3179 (true, false),
3180 ),
3181 (
3182 "https://www.example.com/11",
3183 "more recent 1",
3184 now_u64 - 198_000,
3185 false,
3186 (true, false),
3187 ),
3188 ];
3189
3190 for &(url, title, when, remote, _) in &to_add {
3191 apply_observation(
3192 &conn,
3193 VisitObservation::new(Url::parse(url).unwrap())
3194 .with_title(title.to_owned())
3195 .with_at(Timestamp(when))
3196 .with_is_remote(remote)
3197 .with_visit_type(VisitType::Link),
3198 )
3199 .expect("Should apply visit");
3200 }
3201
3202 let infos_with_bound =
3204 get_visit_page_with_bound(&conn, now_i64 - 200_000, 8, 2, VisitTransitionSet::empty())
3205 .unwrap();
3206 let infos = infos_with_bound.infos;
3207 assert_eq!(infos[0].title.as_ref().unwrap().as_str(), "older 1",);
3208 assert!(infos[0].is_remote); assert_eq!(infos[1].title.as_ref().unwrap().as_str(), "older 2",);
3210 assert!(!infos[1].is_remote); assert_eq!(infos_with_bound.bound, now_i64 - 200_200,);
3212 assert_eq!(infos_with_bound.offset, 1,);
3213
3214 let infos_with_bound =
3216 get_visit_page_with_bound(&conn, now_i64 - 200_000, 7, 1, VisitTransitionSet::empty())
3217 .unwrap();
3218 assert_eq!(
3219 infos_with_bound.infos[0].url,
3220 Url::parse("https://www.example.com/9").unwrap(),
3221 );
3222
3223 let infos_with_bound =
3225 get_visit_page_with_bound(&conn, now_i64 - 200_000, 9, 1, VisitTransitionSet::empty())
3226 .unwrap();
3227 assert_eq!(
3228 infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3229 "older 2",
3230 );
3231
3232 let count = 2;
3234 let mut bound = now_i64 - 199_000;
3235 let mut offset = 1;
3236 for _i in 0..4 {
3237 let infos_with_bound =
3238 get_visit_page_with_bound(&conn, bound, offset, count, VisitTransitionSet::empty())
3239 .unwrap();
3240 assert_eq!(
3241 infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3242 "same time",
3243 );
3244 assert_eq!(
3245 infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3246 "same time",
3247 );
3248 bound = infos_with_bound.bound;
3249 offset = infos_with_bound.offset;
3250 }
3251 assert_eq!(bound, now_i64 - 200_000,);
3253 assert_eq!(offset, 8,);
3254
3255 let infos_with_bound =
3257 get_visit_page_with_bound(&conn, now_i64, 0, 2, VisitTransitionSet::empty()).unwrap();
3258 assert_eq!(
3259 infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3260 "more recent 1",
3261 );
3262 assert_eq!(
3263 infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3264 "more recent 2",
3265 );
3266 assert_eq!(infos_with_bound.bound, now_i64 - 199_000);
3267 assert_eq!(infos_with_bound.offset, 1);
3268 }
3269
3270 #[test]
3272 fn test_normal_visit_pruning() {
3273 use std::time::{Duration, SystemTime};
3274 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3275 let one_day = Duration::from_secs(60 * 60 * 24);
3276 let now: Timestamp = SystemTime::now().into();
3277 let url = Url::parse("https://mozilla.com/").unwrap();
3278
3279 let mut visits: Vec<_> = (0..30)
3281 .map(|i| {
3282 apply_observation(
3283 &conn,
3284 VisitObservation::new(url.clone())
3285 .with_at(now.checked_sub(one_day * i))
3286 .with_visit_type(VisitType::Link),
3287 )
3288 .unwrap()
3289 .unwrap()
3290 })
3291 .collect();
3292 visits.reverse();
3294
3295 check_visits_to_prune(
3296 &conn,
3297 find_normal_visits_to_prune(&conn, 4, now).unwrap(),
3298 &visits[..4],
3299 );
3300
3301 check_visits_to_prune(
3303 &conn,
3304 find_normal_visits_to_prune(&conn, 30, now).unwrap(),
3305 &visits[..22],
3306 );
3307 }
3308
3309 #[test]
3311 fn test_exotic_visit_pruning() {
3312 use std::time::{Duration, SystemTime};
3313 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3314 let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3315 let now: Timestamp = SystemTime::now().into();
3316 let short_url = Url::parse("https://mozilla.com/").unwrap();
3317 let long_url = Url::parse(&format!(
3318 "https://mozilla.com/{}",
3319 (0..255).map(|_| "x").collect::<String>()
3320 ))
3321 .unwrap();
3322
3323 let visit_with_long_url = apply_observation(
3324 &conn,
3325 VisitObservation::new(long_url.clone())
3326 .with_at(now.checked_sub(one_month * 2))
3327 .with_visit_type(VisitType::Link),
3328 )
3329 .unwrap()
3330 .unwrap();
3331
3332 let visit_for_download = apply_observation(
3333 &conn,
3334 VisitObservation::new(short_url)
3335 .with_at(now.checked_sub(one_month * 3))
3336 .with_visit_type(VisitType::Download),
3337 )
3338 .unwrap()
3339 .unwrap();
3340
3341 apply_observation(
3343 &conn,
3344 VisitObservation::new(long_url)
3345 .with_at(now.checked_sub(one_month))
3346 .with_visit_type(VisitType::Download),
3347 )
3348 .unwrap()
3349 .unwrap();
3350
3351 check_visits_to_prune(
3352 &conn,
3353 find_exotic_visits_to_prune(&conn, 2, now).unwrap(),
3354 &[visit_for_download, visit_with_long_url],
3355 );
3356
3357 check_visits_to_prune(
3359 &conn,
3360 find_exotic_visits_to_prune(&conn, 1, now).unwrap(),
3361 &[visit_for_download],
3362 );
3363
3364 check_visits_to_prune(
3366 &conn,
3367 find_exotic_visits_to_prune(&conn, 3, now).unwrap(),
3368 &[visit_for_download, visit_with_long_url],
3369 );
3370 }
3371 #[test]
3374 fn test_visit_pruning() {
3375 use std::time::{Duration, SystemTime};
3376 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3377 let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3378 let now: Timestamp = SystemTime::now().into();
3379 let short_url = Url::parse("https://mozilla.com/").unwrap();
3380 let long_url = Url::parse(&format!(
3381 "https://mozilla.com/{}",
3382 (0..255).map(|_| "x").collect::<String>()
3383 ))
3384 .unwrap();
3385
3386 let excotic_visit = apply_observation(
3388 &conn,
3389 VisitObservation::new(long_url)
3390 .with_at(now.checked_sub(one_month * 3))
3391 .with_visit_type(VisitType::Link),
3392 )
3393 .unwrap()
3394 .unwrap();
3395
3396 let old_visit = apply_observation(
3398 &conn,
3399 VisitObservation::new(short_url.clone())
3400 .with_at(now.checked_sub(one_month * 4))
3401 .with_visit_type(VisitType::Link),
3402 )
3403 .unwrap()
3404 .unwrap();
3405 let really_old_visit = apply_observation(
3406 &conn,
3407 VisitObservation::new(short_url.clone())
3408 .with_at(now.checked_sub(one_month * 12))
3409 .with_visit_type(VisitType::Link),
3410 )
3411 .unwrap()
3412 .unwrap();
3413
3414 apply_observation(
3416 &conn,
3417 VisitObservation::new(short_url)
3418 .with_at(now.checked_sub(Duration::from_secs(100)))
3419 .with_visit_type(VisitType::Link),
3420 )
3421 .unwrap()
3422 .unwrap();
3423
3424 check_visits_to_prune(
3425 &conn,
3426 find_visits_to_prune(&conn, 2, now).unwrap(),
3427 &[excotic_visit, really_old_visit],
3428 );
3429
3430 check_visits_to_prune(
3431 &conn,
3432 find_visits_to_prune(&conn, 10, now).unwrap(),
3433 &[excotic_visit, really_old_visit, old_visit],
3434 );
3435 }
3436
3437 fn check_visits_to_prune(
3438 db: &PlacesDb,
3439 visits_to_delete: Vec<VisitToDelete>,
3440 correct_visits: &[RowId],
3441 ) {
3442 assert_eq!(
3443 correct_visits.iter().collect::<HashSet<_>>(),
3444 visits_to_delete
3445 .iter()
3446 .map(|v| &v.visit_id)
3447 .collect::<HashSet<_>>()
3448 );
3449
3450 let correct_place_ids: HashSet<RowId> = correct_visits
3451 .iter()
3452 .map(|vid| {
3453 db.query_one(&format!(
3454 "SELECT v.place_id FROM moz_historyvisits v WHERE v.id = {}",
3455 vid
3456 ))
3457 .unwrap()
3458 })
3459 .collect();
3460 assert_eq!(
3461 correct_place_ids,
3462 visits_to_delete
3463 .iter()
3464 .map(|v| v.page_id)
3465 .collect::<HashSet<_>>()
3466 );
3467 }
3468
3469 #[test]
3470 fn test_get_visit_count_for_host() {
3471 error_support::init_for_tests();
3472 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3473 let start_timestamp = Timestamp::now();
3474 let to_add = [
3475 (
3476 "http://example.com/0",
3477 start_timestamp.0 - 200_200,
3478 VisitType::Link,
3479 ),
3480 (
3481 "http://example.com/1",
3482 start_timestamp.0 - 200_100,
3483 VisitType::Link,
3484 ),
3485 (
3486 "https://example.com/0",
3487 start_timestamp.0 - 200_000,
3488 VisitType::Link,
3489 ),
3490 (
3491 "https://example1.com/0",
3492 start_timestamp.0 - 100_600,
3493 VisitType::Link,
3494 ),
3495 (
3496 "https://example1.com/0",
3497 start_timestamp.0 - 100_500,
3498 VisitType::Reload,
3499 ),
3500 (
3501 "https://example1.com/1",
3502 start_timestamp.0 - 100_400,
3503 VisitType::Link,
3504 ),
3505 (
3506 "https://example.com/2",
3507 start_timestamp.0 - 100_300,
3508 VisitType::Link,
3509 ),
3510 (
3511 "https://example.com/1",
3512 start_timestamp.0 - 100_200,
3513 VisitType::Link,
3514 ),
3515 (
3516 "https://example.com/0",
3517 start_timestamp.0 - 100_100,
3518 VisitType::Link,
3519 ),
3520 ];
3521
3522 for &(url, when, visit_type) in &to_add {
3523 apply_observation(
3524 &conn,
3525 VisitObservation::new(Url::parse(url).unwrap())
3526 .with_at(Timestamp(when))
3527 .with_visit_type(visit_type),
3528 )
3529 .unwrap()
3530 .unwrap();
3531 }
3532
3533 assert_eq!(
3534 get_visit_count_for_host(
3535 &conn,
3536 "example.com",
3537 Timestamp(start_timestamp.0 - 100_000),
3538 VisitTransitionSet::for_specific(&[])
3539 )
3540 .unwrap(),
3541 6
3542 );
3543 assert_eq!(
3544 get_visit_count_for_host(
3545 &conn,
3546 "example1.com",
3547 Timestamp(start_timestamp.0 - 100_000),
3548 VisitTransitionSet::for_specific(&[])
3549 )
3550 .unwrap(),
3551 3
3552 );
3553 assert_eq!(
3554 get_visit_count_for_host(
3555 &conn,
3556 "example.com",
3557 Timestamp(start_timestamp.0 - 200_000),
3558 VisitTransitionSet::for_specific(&[])
3559 )
3560 .unwrap(),
3561 2
3562 );
3563 assert_eq!(
3564 get_visit_count_for_host(
3565 &conn,
3566 "example1.com",
3567 Timestamp(start_timestamp.0 - 100_500),
3568 VisitTransitionSet::for_specific(&[])
3569 )
3570 .unwrap(),
3571 1
3572 );
3573 assert_eq!(
3574 get_visit_count_for_host(
3575 &conn,
3576 "example1.com",
3577 Timestamp(start_timestamp.0 - 100_000),
3578 VisitTransitionSet::for_specific(&[VisitType::Reload])
3579 )
3580 .unwrap(),
3581 2
3582 );
3583 }
3584}