1mod actions;
6
7use super::{fetch_page_info, new_page_info, PageInfo, RowId};
8use crate::db::PlacesDb;
9use crate::error::{debug, trace, warn, Result};
10use crate::ffi::{HistoryVisitInfo, HistoryVisitInfosWithBound, TopFrecentSiteInfo};
11use crate::frecency;
12use crate::hash;
13use crate::history_sync::engine::{
14 COLLECTION_SYNCID_META_KEY, GLOBAL_SYNCID_META_KEY, LAST_SYNC_META_KEY,
15};
16use crate::observation::VisitObservation;
17use crate::storage::{
18 delete_meta, delete_pending_temp_tables, get_meta, history_metadata, put_meta,
19};
20use crate::types::{
21 serialize_unknown_fields, SyncStatus, UnknownFields, VisitTransitionSet, VisitType,
22};
23use actions::*;
24use rusqlite::types::ToSql;
25use rusqlite::Result as RusqliteResult;
26use rusqlite::Row;
27use sql_support::{self, ConnExt};
28use std::collections::HashSet;
29use std::time::Duration;
30use sync15::bso::OutgoingBso;
31use sync15::engine::EngineSyncAssociation;
32use sync_guid::Guid as SyncGuid;
33use types::Timestamp;
34use url::Url;
35
36static DELETION_HIGH_WATER_MARK_META_KEY: &str = "history_deleted_hwm";
43
44pub fn apply_observation(db: &PlacesDb, visit_ob: VisitObservation) -> Result<Option<RowId>> {
46 let tx = db.begin_transaction()?;
47 let result = apply_observation_direct(db, visit_ob)?;
48 delete_pending_temp_tables(db)?;
49 tx.commit()?;
50 Ok(result)
51}
52
53pub fn apply_observation_direct(
55 db: &PlacesDb,
56 visit_ob: VisitObservation,
57) -> Result<Option<RowId>> {
58 if visit_ob.url.as_str().len() > super::URL_LENGTH_MAX {
60 return Ok(None);
61 }
62 let preview_image_url = if let Some(ref piu) = visit_ob.preview_image_url {
66 if piu.as_str().len() > super::URL_LENGTH_MAX {
67 None
68 } else {
69 Some(piu.clone())
70 }
71 } else {
72 None
73 };
74 let mut page_info = match fetch_page_info(db, &visit_ob.url)? {
75 Some(info) => info.page,
76 None => new_page_info(db, &visit_ob.url, None)?,
77 };
78 let mut update_change_counter = false;
79 let mut update_frec = false;
80 let mut updates: Vec<(&str, &str, &dyn ToSql)> = Vec::new();
81
82 if let Some(ref title) = visit_ob.title {
83 page_info.title = crate::util::slice_up_to(title, super::TITLE_LENGTH_MAX).into();
84 updates.push(("title", ":title", &page_info.title));
85 update_change_counter = true;
86 }
87 let preview_image_url_str;
88 if let Some(ref preview_image_url) = preview_image_url {
89 preview_image_url_str = preview_image_url.as_str();
90 updates.push((
91 "preview_image_url",
92 ":preview_image_url",
93 &preview_image_url_str,
94 ));
95 }
96 let visit_row_id = match visit_ob.visit_type {
99 Some(visit_type) => {
100 if !visit_ob.get_is_hidden() {
102 updates.push(("hidden", ":hidden", &false));
103 }
104 if visit_type == VisitType::Typed {
105 page_info.typed += 1;
106 updates.push(("typed", ":typed", &page_info.typed));
107 }
108
109 let at = visit_ob.at.unwrap_or_else(Timestamp::now);
110 let is_remote = visit_ob.is_remote.unwrap_or(false);
111 let row_id = add_visit(db, page_info.row_id, None, at, visit_type, !is_remote, None)?;
112 if !visit_ob.is_error.unwrap_or(false) {
114 update_frec = true;
115 }
116 update_change_counter = true;
117 Some(row_id)
118 }
119 None => None,
120 };
121
122 if update_change_counter {
123 page_info.sync_change_counter += 1;
124 updates.push((
125 "sync_change_counter",
126 ":sync_change_counter",
127 &page_info.sync_change_counter,
128 ));
129 }
130
131 if !updates.is_empty() {
132 let mut params: Vec<(&str, &dyn ToSql)> = Vec::with_capacity(updates.len() + 1);
133 let mut sets: Vec<String> = Vec::with_capacity(updates.len());
134 for (col, name, val) in updates {
135 sets.push(format!("{} = {}", col, name));
136 params.push((name, val))
137 }
138 params.push((":row_id", &page_info.row_id.0));
139 let sql = format!(
140 "UPDATE moz_places
141 SET {}
142 WHERE id == :row_id",
143 sets.join(",")
144 );
145 db.execute(&sql, ¶ms[..])?;
146 }
147 if update_frec {
149 update_frecency(
150 db,
151 page_info.row_id,
152 Some(visit_ob.get_redirect_frecency_boost()),
153 )?;
154 }
155 Ok(visit_row_id)
156}
157
158pub fn update_frecency(db: &PlacesDb, id: RowId, redirect_boost: Option<bool>) -> Result<()> {
159 let score = frecency::calculate_frecency(
160 db.conn(),
161 &frecency::DEFAULT_FRECENCY_SETTINGS,
162 id.0, redirect_boost,
164 )?;
165
166 db.execute(
167 "
168 UPDATE moz_places
169 SET frecency = :frecency
170 WHERE id = :page_id",
171 &[
172 (":frecency", &score as &dyn rusqlite::ToSql),
173 (":page_id", &id.0),
174 ],
175 )?;
176
177 Ok(())
178}
179
180pub fn frecency_stale_at(db: &PlacesDb, url: &Url) -> Result<Option<Timestamp>> {
182 let result = db.try_query_row(
183 "SELECT stale_at FROM moz_places_stale_frecencies s
184 JOIN moz_places h ON h.id = s.place_id
185 WHERE h.url_hash = hash(:url) AND
186 h.url = :url",
187 &[(":url", &url.as_str())],
188 |row| -> rusqlite::Result<_> { row.get::<_, Timestamp>(0) },
189 true,
190 )?;
191 Ok(result)
192}
193
194fn add_visit(
198 db: &PlacesDb,
199 page_id: RowId,
200 from_visit: Option<RowId>,
201 visit_date: Timestamp,
202 visit_type: VisitType,
203 is_local: bool,
204 unknown_fields: Option<String>,
205) -> Result<RowId> {
206 let sql = "INSERT INTO moz_historyvisits
207 (from_visit, place_id, visit_date, visit_type, is_local, unknown_fields)
208 VALUES (:from_visit, :page_id, :visit_date, :visit_type, :is_local, :unknown_fields)";
209 db.execute_cached(
210 sql,
211 &[
212 (":from_visit", &from_visit as &dyn rusqlite::ToSql),
213 (":page_id", &page_id),
214 (":visit_date", &visit_date),
215 (":visit_type", &visit_type),
216 (":is_local", &is_local),
217 (":unknown_fields", &unknown_fields),
218 ],
219 )?;
220 let rid = db.conn().last_insert_rowid();
221 db.execute_cached(
223 "DELETE FROM moz_historyvisit_tombstones
224 WHERE place_id = :place_id
225 AND visit_date = :visit_date",
226 &[
227 (":place_id", &page_id as &dyn rusqlite::ToSql),
228 (":visit_date", &visit_date),
229 ],
230 )?;
231 Ok(RowId(rid))
232}
233
234pub fn url_to_guid(db: &PlacesDb, url: &Url) -> Result<Option<SyncGuid>> {
236 href_to_guid(db, url.clone().as_str())
237}
238
239pub fn href_to_guid(db: &PlacesDb, url: &str) -> Result<Option<SyncGuid>> {
241 let sql = "SELECT guid FROM moz_places WHERE url_hash = hash(:url) AND url = :url";
242 let result: Option<SyncGuid> = db.try_query_row(
243 sql,
244 &[(":url", &url.to_owned())],
245 |row| -> rusqlite::Result<_> { row.get::<_, SyncGuid>(0) },
248 true,
249 )?;
250 Ok(result)
251}
252
253fn delete_visits_for_in_tx(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
256 let to_clean = db.conn().try_query_row(
259 "SELECT id,
260 (foreign_count != 0) AS has_foreign,
261 1 as has_visits,
262 sync_status
263 FROM moz_places
264 WHERE guid = :guid",
265 &[(":guid", guid)],
266 PageToClean::from_row,
267 true,
268 )?;
269 match to_clean {
273 Some(PageToClean {
274 id,
275 has_foreign: true,
276 sync_status: SyncStatus::Normal,
277 ..
278 }) => {
279 insert_tombstones_for_all_page_visits(db, id)?;
286 delete_all_visits_for_page(db, id)?;
287 history_metadata::delete_all_metadata_for_page(db, id)?;
288 }
289 Some(PageToClean {
290 id,
291 has_foreign: false,
292 sync_status: SyncStatus::Normal,
293 ..
294 }) => {
295 insert_tombstone_for_page(db, guid)?;
299 delete_page(db, id)?;
300 }
301 Some(PageToClean {
302 id,
303 has_foreign: true,
304 ..
305 }) => {
306 delete_all_visits_for_page(db, id)?;
310 history_metadata::delete_all_metadata_for_page(db, id)?;
312 }
313 Some(PageToClean {
314 id,
315 has_foreign: false,
316 ..
317 }) => {
318 delete_page(db, id)?;
321 }
322 None => {}
323 }
324 delete_pending_temp_tables(db)?;
325 Ok(())
326}
327
328fn insert_tombstones_for_all_page_visits(db: &PlacesDb, page_id: RowId) -> Result<()> {
330 db.execute_cached(
331 "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date)
332 SELECT place_id, visit_date
333 FROM moz_historyvisits
334 WHERE place_id = :page_id",
335 &[(":page_id", &page_id)],
336 )?;
337 Ok(())
338}
339
340fn delete_all_visits_for_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
343 db.execute_cached(
344 "DELETE FROM moz_historyvisits
345 WHERE place_id = :page_id",
346 &[(":page_id", &page_id)],
347 )?;
348 Ok(())
349}
350
351fn insert_tombstone_for_page(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
353 db.execute_cached(
354 "INSERT OR IGNORE INTO moz_places_tombstones (guid)
355 VALUES(:guid)",
356 &[(":guid", guid)],
357 )?;
358 Ok(())
359}
360
361fn delete_page(db: &PlacesDb, page_id: RowId) -> Result<()> {
364 db.execute_cached(
365 "DELETE FROM moz_places
366 WHERE id = :page_id",
367 &[(":page_id", &page_id)],
368 )?;
369 Ok(())
370}
371
372pub fn delete_visits_for(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
375 let tx = db.begin_transaction()?;
376 let result = delete_visits_for_in_tx(db, guid);
377 tx.commit()?;
378 result
379}
380
381pub fn delete_visits_between(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
383 let tx = db.begin_transaction()?;
384 delete_visits_between_in_tx(db, start, end)?;
385 tx.commit()?;
386 Ok(())
387}
388
389pub fn delete_place_visit_at_time(db: &PlacesDb, place: &Url, visit: Timestamp) -> Result<()> {
390 delete_place_visit_at_time_by_href(db, place.as_str(), visit)
391}
392
393pub fn delete_place_visit_at_time_by_href(
394 db: &PlacesDb,
395 place: &str,
396 visit: Timestamp,
397) -> Result<()> {
398 let tx = db.begin_transaction()?;
399 delete_place_visit_at_time_in_tx(db, place, visit)?;
400 tx.commit()?;
401 Ok(())
402}
403
404pub fn prune_older_visits(db: &PlacesDb, limit: u32) -> Result<()> {
405 let tx = db.begin_transaction()?;
406
407 let result = DbAction::apply_all(
408 db,
409 db_actions_from_visits_to_delete(find_visits_to_prune(
410 db,
411 limit as usize,
412 Timestamp::now(),
413 )?),
414 );
415 tx.commit()?;
416 result
417}
418
419fn find_visits_to_prune(db: &PlacesDb, limit: usize, now: Timestamp) -> Result<Vec<VisitToDelete>> {
420 let mut to_delete: HashSet<_> = find_exotic_visits_to_prune(db, limit, now)?
422 .into_iter()
423 .collect();
424 if to_delete.len() < limit {
427 for delete_visit in find_normal_visits_to_prune(db, limit, now)? {
428 to_delete.insert(delete_visit);
429 if to_delete.len() >= limit {
430 break;
431 }
432 }
433 }
434 Ok(Vec::from_iter(to_delete))
435}
436
437fn find_normal_visits_to_prune(
438 db: &PlacesDb,
439 limit: usize,
440 now: Timestamp,
441) -> Result<Vec<VisitToDelete>> {
442 let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 7));
444 db.query_rows_and_then(
445 "
446 SELECT v.id, v.place_id
447 FROM moz_places p
448 JOIN moz_historyvisits v ON v.place_id = p.id
449 WHERE v.visit_date < :visit_date_cuttoff
450 ORDER BY v.visit_date
451 LIMIT :limit
452 ",
453 rusqlite::named_params! {
454 ":visit_date_cuttoff": visit_date_cutoff,
455 ":limit": limit,
456 },
457 VisitToDelete::from_row,
458 )
459}
460
461fn find_exotic_visits_to_prune(
469 db: &PlacesDb,
470 limit: usize,
471 now: Timestamp,
472) -> Result<Vec<VisitToDelete>> {
473 let visit_date_cutoff = now.checked_sub(Duration::from_secs(60 * 60 * 24 * 60));
475 db.query_rows_and_then(
476 "
477 SELECT v.id, v.place_id
478 FROM moz_places p
479 JOIN moz_historyvisits v ON v.place_id = p.id
480 WHERE v.visit_date < :visit_date_cuttoff
481 AND (LENGTH(p.url) > 255 OR v.visit_type = :download_visit_type)
482 ORDER BY v.visit_date
483 LIMIT :limit
484 ",
485 rusqlite::named_params! {
486 ":visit_date_cuttoff": visit_date_cutoff,
487 ":download_visit_type": VisitType::Download,
488 ":limit": limit,
489 },
490 VisitToDelete::from_row,
491 )
492}
493
494fn wipe_local_in_tx(db: &PlacesDb) -> Result<()> {
495 use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
496 db.execute_all(&[
497 "DELETE FROM moz_places WHERE foreign_count == 0",
498 "DELETE FROM moz_places_metadata",
499 "DELETE FROM moz_places_metadata_search_queries",
500 "DELETE FROM moz_historyvisits",
501 "DELETE FROM moz_places_tombstones",
502 "DELETE FROM moz_inputhistory AS i WHERE NOT EXISTS(
503 SELECT 1 FROM moz_places h
504 WHERE h.id = i.place_id)",
505 "DELETE FROM moz_historyvisit_tombstones",
506 "DELETE FROM moz_origins
507 WHERE id NOT IN (SELECT origin_id FROM moz_places)",
508 &format!(
509 r#"UPDATE moz_places SET
510 frecency = (CASE WHEN url_hash BETWEEN hash("place", "prefix_lo") AND
511 hash("place", "prefix_hi")
512 THEN 0
513 ELSE {unvisited_bookmark_frec}
514 END),
515 sync_change_counter = 0"#,
516 unvisited_bookmark_frec = DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
517 ),
518 ])?;
519
520 let need_frecency_update =
521 db.query_rows_and_then("SELECT id FROM moz_places", [], |r| r.get::<_, RowId>(0))?;
522 for row_id in need_frecency_update {
525 update_frecency(db, row_id, None)?;
526 }
527 delete_pending_temp_tables(db)?;
528 Ok(())
529}
530
531pub fn delete_everything(db: &PlacesDb) -> Result<()> {
532 let tx = db.begin_transaction()?;
533
534 let most_recent_known_visit_time = db
536 .try_query_one::<Timestamp, _>("SELECT MAX(visit_date) FROM moz_historyvisits", [], false)?
537 .unwrap_or_default();
538
539 let previous_mark =
541 get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
542
543 let new_mark = Timestamp::now()
544 .max(previous_mark)
545 .max(most_recent_known_visit_time);
546
547 put_meta(db, DELETION_HIGH_WATER_MARK_META_KEY, &new_mark)?;
548
549 wipe_local_in_tx(db)?;
550
551 reset_in_tx(db, &EngineSyncAssociation::Disconnected)?;
553
554 tx.commit()?;
555
556 db.execute_batch("VACUUM")?;
558 Ok(())
559}
560
561fn delete_place_visit_at_time_in_tx(db: &PlacesDb, url: &str, visit_date: Timestamp) -> Result<()> {
562 DbAction::apply_all(
563 db,
564 db_actions_from_visits_to_delete(db.query_rows_and_then(
565 "SELECT v.id, v.place_id
566 FROM moz_places h
567 JOIN moz_historyvisits v
568 ON v.place_id = h.id
569 WHERE v.visit_date = :visit_date
570 AND h.url_hash = hash(:url)
571 AND h.url = :url",
572 &[
573 (":url", &url as &dyn rusqlite::ToSql),
574 (":visit_date", &visit_date),
575 ],
576 VisitToDelete::from_row,
577 )?),
578 )
579}
580
581pub fn delete_visits_between_in_tx(db: &PlacesDb, start: Timestamp, end: Timestamp) -> Result<()> {
582 let sql = "
586 SELECT id, place_id, visit_date
587 FROM moz_historyvisits
588 WHERE visit_date
589 BETWEEN :start AND :end
590 ";
591 let visits = db.query_rows_and_then(
592 sql,
593 &[(":start", &start), (":end", &end)],
594 |row| -> rusqlite::Result<_> {
595 Ok((
596 row.get::<_, RowId>(0)?,
597 row.get::<_, RowId>(1)?,
598 row.get::<_, Timestamp>(2)?,
599 ))
600 },
601 )?;
602
603 sql_support::each_chunk_mapped(
604 &visits,
605 |(visit_id, _, _)| visit_id,
606 |chunk, _| -> Result<()> {
607 db.conn().execute(
608 &format!(
609 "DELETE from moz_historyvisits WHERE id IN ({})",
610 sql_support::repeat_sql_vars(chunk.len()),
611 ),
612 rusqlite::params_from_iter(chunk),
613 )?;
614 Ok(())
615 },
616 )?;
617
618 if !visits.is_empty() {
620 let sql = format!(
621 "INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date) VALUES {}",
622 sql_support::repeat_display(visits.len(), ",", |i, f| {
623 let (_, place_id, visit_date) = visits[i];
624 write!(f, "({},{})", place_id.0, visit_date.0)
625 })
626 );
627 db.conn().execute(&sql, [])?;
628 }
629
630 sql_support::each_chunk_mapped(
632 &visits,
633 |(_, place_id, _)| place_id.0,
634 |chunk, _| -> Result<()> {
635 let query = format!(
636 "SELECT id,
637 (foreign_count != 0) AS has_foreign,
638 ((last_visit_date_local + last_visit_date_remote) != 0) as has_visits,
639 sync_status
640 FROM moz_places
641 WHERE id IN ({})",
642 sql_support::repeat_sql_vars(chunk.len()),
643 );
644
645 let mut stmt = db.conn().prepare(&query)?;
646 let page_results =
647 stmt.query_and_then(rusqlite::params_from_iter(chunk), PageToClean::from_row)?;
648 let pages: Vec<PageToClean> = page_results.collect::<Result<_>>()?;
649 cleanup_pages(db, &pages)
650 },
651 )?;
652
653 history_metadata::delete_between(db, start.as_millis_i64(), end.as_millis_i64())?;
655 delete_pending_temp_tables(db)?;
656 Ok(())
657}
658
659#[derive(Debug)]
660struct PageToClean {
661 id: RowId,
662 has_foreign: bool,
663 has_visits: bool,
664 sync_status: SyncStatus,
665}
666
667impl PageToClean {
668 pub fn from_row(row: &Row<'_>) -> Result<Self> {
669 Ok(Self {
670 id: row.get("id")?,
671 has_foreign: row.get("has_foreign")?,
672 has_visits: row.get("has_visits")?,
673 sync_status: row.get("sync_status")?,
674 })
675 }
676}
677
678fn cleanup_pages(db: &PlacesDb, pages: &[PageToClean]) -> Result<()> {
684 let frec_ids = pages
687 .iter()
688 .filter(|&p| p.has_foreign || p.has_visits)
689 .map(|p| p.id);
690
691 for id in frec_ids {
692 update_frecency(db, id, None)?;
693 }
694
695 let remove_ids: Vec<RowId> = pages
701 .iter()
702 .filter(|p| !p.has_foreign && !p.has_visits)
703 .map(|p| p.id)
704 .collect();
705 sql_support::each_chunk(&remove_ids, |chunk, _| -> Result<()> {
706 db.conn().execute(
708 &format!(
709 "
710 INSERT OR IGNORE INTO moz_places_tombstones (guid)
711 SELECT guid FROM moz_places
712 WHERE id in ({ids}) AND sync_status = {status}
713 AND foreign_count = 0
714 AND last_visit_date_local = 0
715 AND last_visit_date_remote = 0",
716 ids = sql_support::repeat_sql_vars(chunk.len()),
717 status = SyncStatus::Normal as u8,
718 ),
719 rusqlite::params_from_iter(chunk),
720 )?;
721 db.conn().execute(
722 &format!(
723 "
724 DELETE FROM moz_places
725 WHERE id IN ({ids})
726 AND foreign_count = 0
727 AND last_visit_date_local = 0
728 AND last_visit_date_remote = 0",
729 ids = sql_support::repeat_sql_vars(chunk.len())
730 ),
731 rusqlite::params_from_iter(chunk),
732 )?;
733 Ok(())
734 })?;
735
736 Ok(())
737}
738
739fn reset_in_tx(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
740 db.execute_cached(
742 &format!(
743 "
744 UPDATE moz_places
745 SET sync_change_counter = 0,
746 sync_status = {}",
747 (SyncStatus::New as u8)
748 ),
749 [],
750 )?;
751
752 put_meta(db, LAST_SYNC_META_KEY, &0)?;
755
756 match assoc {
759 EngineSyncAssociation::Disconnected => {
760 delete_meta(db, GLOBAL_SYNCID_META_KEY)?;
761 delete_meta(db, COLLECTION_SYNCID_META_KEY)?;
762 }
763 EngineSyncAssociation::Connected(ids) => {
764 put_meta(db, GLOBAL_SYNCID_META_KEY, &ids.global)?;
765 put_meta(db, COLLECTION_SYNCID_META_KEY, &ids.coll)?;
766 }
767 }
768
769 Ok(())
770}
771
772pub mod history_sync {
774 use sync15::bso::OutgoingEnvelope;
775
776 use super::*;
777 use crate::history_sync::record::{HistoryRecord, HistoryRecordVisit};
778 use crate::history_sync::HISTORY_TTL;
779 use std::collections::HashSet;
780
781 #[derive(Debug, Clone, PartialEq, Eq)]
782 pub struct FetchedVisit {
783 pub is_local: bool,
784 pub visit_date: Timestamp,
785 pub visit_type: Option<VisitType>,
786 }
787
788 impl FetchedVisit {
789 pub fn from_row(row: &Row<'_>) -> Result<Self> {
790 Ok(Self {
791 is_local: row.get("is_local")?,
792 visit_date: row
793 .get::<_, Option<Timestamp>>("visit_date")?
794 .unwrap_or_default(),
795 visit_type: VisitType::from_primitive(
796 row.get::<_, Option<u8>>("visit_type")?.unwrap_or(0),
797 ),
798 })
799 }
800 }
801
802 #[derive(Debug)]
803 pub struct FetchedVisitPage {
804 pub url: Url,
805 pub guid: SyncGuid,
806 pub row_id: RowId,
807 pub title: String,
808 pub unknown_fields: UnknownFields,
809 }
810
811 impl FetchedVisitPage {
812 pub fn from_row(row: &Row<'_>) -> Result<Self> {
813 Ok(Self {
814 url: Url::parse(&row.get::<_, String>("url")?)?,
815 guid: row.get::<_, String>("guid")?.into(),
816 row_id: row.get("id")?,
817 title: row.get::<_, Option<String>>("title")?.unwrap_or_default(),
818 unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
819 None => UnknownFields::new(),
820 Some(v) => serde_json::from_str(&v)?,
821 },
822 })
823 }
824 }
825
826 pub fn fetch_visits(
827 db: &PlacesDb,
828 url: &Url,
829 limit: usize,
830 ) -> Result<Option<(FetchedVisitPage, Vec<FetchedVisit>)>> {
831 let page_sql = "
833 SELECT guid, url, id, title, unknown_fields
834 FROM moz_places h
835 WHERE url_hash = hash(:url) AND url = :url";
836
837 let page_info = match db.try_query_row(
838 page_sql,
839 &[(":url", &url.to_string())],
840 FetchedVisitPage::from_row,
841 true,
842 )? {
843 None => return Ok(None),
844 Some(pi) => pi,
845 };
846
847 let visits = db.query_rows_and_then(
848 "SELECT is_local, visit_type, visit_date
849 FROM moz_historyvisits
850 WHERE place_id = :place_id
851 LIMIT :limit",
852 &[
853 (":place_id", &page_info.row_id as &dyn rusqlite::ToSql),
854 (":limit", &(limit as u32)),
855 ],
856 FetchedVisit::from_row,
857 )?;
858 Ok(Some((page_info, visits)))
859 }
860
861 pub fn apply_synced_visits(
864 db: &PlacesDb,
865 incoming_guid: &SyncGuid,
866 url: &Url,
867 title: &Option<String>,
868 visits: &[HistoryRecordVisit],
869 unknown_fields: &UnknownFields,
870 ) -> Result<()> {
871 let visit_ignored_mark =
875 get_meta::<Timestamp>(db, DELETION_HIGH_WATER_MARK_META_KEY)?.unwrap_or_default();
876
877 let visits = visits
878 .iter()
879 .filter(|v| Timestamp::from(v.date) > visit_ignored_mark)
880 .collect::<Vec<_>>();
881
882 let mut counter_incr = 0;
883 let page_info = match fetch_page_info(db, url)? {
884 Some(mut info) => {
885 if &info.page.guid != incoming_guid {
890 if info.page.sync_status == SyncStatus::New {
891 db.execute_cached(
892 "UPDATE moz_places SET guid = :new_guid WHERE id = :row_id",
893 &[
894 (":new_guid", incoming_guid as &dyn rusqlite::ToSql),
895 (":row_id", &info.page.row_id),
896 ],
897 )?;
898 info.page.guid = incoming_guid.clone();
899 }
900 counter_incr = 1;
904 }
905 info.page
906 }
907 None => {
908 if visits.is_empty() {
911 return Ok(());
912 }
913 new_page_info(db, url, Some(incoming_guid.clone()))?
914 }
915 };
916
917 if !visits.is_empty() {
918 let mut visits_to_skip: HashSet<Timestamp> = db.query_rows_into(
926 &format!(
927 "SELECT t.visit_date AS visit_date
928 FROM moz_historyvisit_tombstones t
929 WHERE t.place_id = {place}
930 AND t.visit_date IN ({dates})
931 UNION ALL
932 SELECT v.visit_date AS visit_date
933 FROM moz_historyvisits v
934 WHERE v.place_id = {place}
935 AND v.visit_date IN ({dates})",
936 place = page_info.row_id,
937 dates = sql_support::repeat_display(visits.len(), ",", |i, f| write!(
938 f,
939 "{}",
940 Timestamp::from(visits[i].date).0
941 )),
942 ),
943 [],
944 |row| row.get::<_, Timestamp>(0),
945 )?;
946
947 visits_to_skip.reserve(visits.len());
948
949 for visit in visits {
950 let timestamp = Timestamp::from(visit.date);
951 if visits_to_skip.contains(×tamp) {
953 continue;
954 }
955 let transition = VisitType::from_primitive(visit.transition)
956 .expect("these should already be validated");
957 add_visit(
958 db,
959 page_info.row_id,
960 None,
961 timestamp,
962 transition,
963 false,
964 serialize_unknown_fields(&visit.unknown_fields)?,
965 )?;
966 visits_to_skip.insert(timestamp);
970 }
971 }
972 update_frecency(db, page_info.row_id, None)?;
975
976 let new_title = title.as_ref().unwrap_or(&page_info.title);
978 db.execute_cached(
983 "UPDATE moz_places
984 SET title = :title,
985 unknown_fields = :unknown_fields,
986 sync_status = :status,
987 sync_change_counter = :sync_change_counter
988 WHERE id == :row_id",
989 &[
990 (":title", new_title as &dyn rusqlite::ToSql),
991 (":row_id", &page_info.row_id),
992 (":status", &SyncStatus::Normal),
993 (
994 ":unknown_fields",
995 &serialize_unknown_fields(unknown_fields)?,
996 ),
997 (
998 ":sync_change_counter",
999 &(page_info.sync_change_counter + counter_incr),
1000 ),
1001 ],
1002 )?;
1003
1004 Ok(())
1005 }
1006
1007 pub fn apply_synced_reconciliation(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1008 db.execute_cached(
1009 "UPDATE moz_places
1010 SET sync_status = :status,
1011 sync_change_counter = 0
1012 WHERE guid == :guid",
1013 &[
1014 (":guid", guid as &dyn rusqlite::ToSql),
1015 (":status", &SyncStatus::Normal),
1016 ],
1017 )?;
1018 Ok(())
1019 }
1020
1021 pub fn apply_synced_deletion(db: &PlacesDb, guid: &SyncGuid) -> Result<()> {
1022 db.execute_cached(
1026 "DELETE FROM moz_historyvisits
1027 WHERE place_id IN (
1028 SELECT id
1029 FROM moz_places
1030 WHERE guid = :guid
1031 )",
1032 &[(":guid", guid)],
1033 )?;
1034 db.execute_cached(
1035 "DELETE FROM moz_places WHERE guid = :guid AND foreign_count = 0",
1036 &[(":guid", guid)],
1037 )?;
1038 Ok(())
1039 }
1040
1041 pub fn fetch_outgoing(
1042 db: &PlacesDb,
1043 max_places: usize,
1044 max_visits: usize,
1045 ) -> Result<Vec<OutgoingBso>> {
1046 let places_sql = format!(
1051 "
1052 SELECT guid, url, id, title, hidden, typed, frecency,
1053 visit_count_local, visit_count_remote,
1054 last_visit_date_local, last_visit_date_remote,
1055 sync_status, sync_change_counter, preview_image_url,
1056 unknown_fields
1057 FROM moz_places
1058 WHERE (sync_change_counter > 0 OR sync_status != {}) AND
1059 NOT hidden
1060 ORDER BY frecency DESC
1061 LIMIT :max_places",
1062 (SyncStatus::Normal as u8)
1063 );
1064 let visits_sql = "
1065 SELECT visit_date as date, visit_type as transition, unknown_fields
1066 FROM moz_historyvisits
1067 WHERE place_id = :place_id
1068 ORDER BY visit_date DESC
1069 LIMIT :max_visits";
1070 let tombstones_sql = "SELECT guid FROM moz_places_tombstones LIMIT :max_places";
1072
1073 let mut tombstone_ids = HashSet::new();
1074 let mut result = Vec::new();
1075
1076 let ts_rows = db.query_rows_and_then(
1079 tombstones_sql,
1080 &[(":max_places", &(max_places as u32))],
1081 |row| -> rusqlite::Result<SyncGuid> { Ok(row.get::<_, String>("guid")?.into()) },
1082 )?;
1083 result.reserve(ts_rows.len());
1087 tombstone_ids.reserve(ts_rows.len());
1088 for guid in ts_rows {
1089 trace!("outgoing tombstone {:?}", &guid);
1090 let envelope = OutgoingEnvelope {
1091 id: guid.clone(),
1092 ttl: Some(HISTORY_TTL),
1093 ..Default::default()
1094 };
1095 result.push(OutgoingBso::new_tombstone(envelope));
1096 tombstone_ids.insert(guid);
1097 }
1098
1099 let max_places_left = max_places - result.len();
1101
1102 db.execute(
1108 "CREATE TEMP TABLE IF NOT EXISTS temp_sync_updated_meta
1109 (id INTEGER PRIMARY KEY,
1110 change_delta INTEGER NOT NULL)",
1111 [],
1112 )?;
1113
1114 let insert_meta_sql = "
1115 INSERT INTO temp_sync_updated_meta VALUES (:row_id, :change_delta)";
1116
1117 let rows = db.query_rows_and_then(
1118 &places_sql,
1119 &[(":max_places", &(max_places_left as u32))],
1120 PageInfo::from_row,
1121 )?;
1122 result.reserve(rows.len());
1123 let mut ids_to_update = Vec::with_capacity(rows.len());
1124 for page in rows {
1125 let visits = db.query_rows_and_then_cached(
1126 visits_sql,
1127 &[
1128 (":max_visits", &(max_visits as u32) as &dyn rusqlite::ToSql),
1129 (":place_id", &page.row_id),
1130 ],
1131 |row| -> Result<_> {
1132 Ok(HistoryRecordVisit {
1133 date: row.get::<_, Timestamp>("date")?.into(),
1134 transition: row.get::<_, u8>("transition")?,
1135 unknown_fields: match row.get::<_, Option<String>>("unknown_fields")? {
1136 None => UnknownFields::new(),
1137 Some(v) => serde_json::from_str(&v)?,
1138 },
1139 })
1140 },
1141 )?;
1142 if tombstone_ids.contains(&page.guid) {
1143 warn!("Found {:?} in both tombstones and live records", &page.guid);
1145 continue;
1146 }
1147 if visits.is_empty() {
1148 trace!(
1152 "Page {:?} is flagged to be uploaded, but has no visits - skipping",
1153 &page.guid
1154 );
1155 continue;
1156 }
1157 trace!("outgoing record {:?}", &page.guid);
1158 ids_to_update.push(page.row_id);
1159 db.execute_cached(
1160 insert_meta_sql,
1161 &[
1162 (":row_id", &page.row_id as &dyn rusqlite::ToSql),
1163 (":change_delta", &page.sync_change_counter),
1164 ],
1165 )?;
1166
1167 let content = HistoryRecord {
1168 id: page.guid.clone(),
1169 title: page.title,
1170 hist_uri: page.url.to_string(),
1171 visits,
1172 unknown_fields: page.unknown_fields,
1173 };
1174
1175 let envelope = OutgoingEnvelope {
1176 id: page.guid,
1177 sortindex: Some(page.frecency),
1178 ttl: Some(HISTORY_TTL),
1179 };
1180 let bso = OutgoingBso::from_content(envelope, content)?;
1181 result.push(bso);
1182 }
1183
1184 sql_support::each_chunk(&ids_to_update, |chunk, _| -> Result<()> {
1189 db.conn().execute(
1190 &format!(
1191 "UPDATE moz_places SET sync_status={status}
1192 WHERE id IN ({vars})",
1193 vars = sql_support::repeat_sql_vars(chunk.len()),
1194 status = SyncStatus::Normal as u8
1195 ),
1196 rusqlite::params_from_iter(chunk),
1197 )?;
1198 Ok(())
1199 })?;
1200
1201 Ok(result)
1202 }
1203
1204 pub fn finish_outgoing(db: &PlacesDb) -> Result<()> {
1205 debug!("Updating all synced rows");
1216 db.conn().execute_cached(
1219 "
1220 UPDATE moz_places
1221 SET sync_change_counter = sync_change_counter -
1222 (SELECT change_delta FROM temp_sync_updated_meta m WHERE moz_places.id = m.id)
1223 WHERE id IN (SELECT id FROM temp_sync_updated_meta)
1224 ",
1225 [],
1226 )?;
1227
1228 debug!("Updating all non-synced rows");
1229 db.execute_all(&[
1230 &format!(
1231 "UPDATE moz_places
1232 SET sync_change_counter = 0, sync_status = {}
1233 WHERE id NOT IN (SELECT id from temp_sync_updated_meta)",
1234 (SyncStatus::Normal as u8)
1235 ),
1236 "DELETE FROM temp_sync_updated_meta",
1237 ])?;
1238
1239 debug!("Removing local tombstones");
1240 db.conn()
1241 .execute_cached("DELETE from moz_places_tombstones", [])?;
1242
1243 Ok(())
1244 }
1245
1246 pub(crate) fn reset(db: &PlacesDb, assoc: &EngineSyncAssociation) -> Result<()> {
1250 let tx = db.begin_transaction()?;
1251 reset_in_tx(db, assoc)?;
1252 tx.commit()?;
1253 Ok(())
1254 }
1255} pub fn get_visited<I>(db: &PlacesDb, urls: I) -> Result<Vec<bool>>
1258where
1259 I: IntoIterator<Item = Url>,
1260 I::IntoIter: ExactSizeIterator,
1261{
1262 let iter = urls.into_iter();
1263 let mut result = vec![false; iter.len()];
1264 let url_idxs = iter.enumerate().collect::<Vec<_>>();
1265 get_visited_into(db, &url_idxs, &mut result)?;
1266 Ok(result)
1267}
1268
1269pub fn get_visited_into(
1276 db: &PlacesDb,
1277 urls_idxs: &[(usize, Url)],
1278 result: &mut [bool],
1279) -> Result<()> {
1280 sql_support::each_chunk_mapped(
1281 urls_idxs,
1282 |(_, url)| url.as_str(),
1283 |chunk, offset| -> Result<()> {
1284 let values_with_idx = sql_support::repeat_display(chunk.len(), ",", |i, f| {
1285 let (idx, url) = &urls_idxs[i + offset];
1286 write!(f, "({},{},?)", *idx, hash::hash_url(url.as_str()))
1287 });
1288 let sql = format!(
1289 "WITH to_fetch(fetch_url_index, url_hash, url) AS (VALUES {})
1290 SELECT fetch_url_index
1291 FROM moz_places h
1292 JOIN to_fetch f ON h.url_hash = f.url_hash
1293 AND h.url = f.url
1294 AND (h.last_visit_date_local + h.last_visit_date_remote) != 0",
1295 values_with_idx
1296 );
1297 let mut stmt = db.prepare(&sql)?;
1298 for idx_r in stmt.query_and_then(
1299 rusqlite::params_from_iter(chunk),
1300 |row| -> rusqlite::Result<_> { Ok(row.get::<_, i64>(0)? as usize) },
1301 )? {
1302 let idx = idx_r?;
1303 result[idx] = true;
1304 }
1305 Ok(())
1306 },
1307 )?;
1308 Ok(())
1309}
1310
1311pub fn get_visited_urls(
1314 db: &PlacesDb,
1315 start: Timestamp,
1316 end: Timestamp,
1317 include_remote: bool,
1318) -> Result<Vec<String>> {
1319 let sql = format!(
1323 "SELECT h.url
1324 FROM moz_places h
1325 WHERE EXISTS (
1326 SELECT 1 FROM moz_historyvisits v
1327 WHERE place_id = h.id
1328 AND visit_date BETWEEN :start AND :end
1329 {and_is_local}
1330 LIMIT 1
1331 )",
1332 and_is_local = if include_remote { "" } else { "AND is_local" }
1333 );
1334 Ok(db.query_rows_and_then_cached(
1335 &sql,
1336 &[(":start", &start), (":end", &end)],
1337 |row| -> RusqliteResult<_> { row.get::<_, String>(0) },
1338 )?)
1339}
1340
1341pub fn get_top_frecent_site_infos(
1342 db: &PlacesDb,
1343 num_items: i32,
1344 frecency_threshold: i64,
1345) -> Result<Vec<TopFrecentSiteInfo>> {
1346 let allowed_types = VisitTransitionSet::for_specific(&[
1348 VisitType::Download,
1349 VisitType::Embed,
1350 VisitType::RedirectPermanent,
1351 VisitType::RedirectTemporary,
1352 VisitType::FramedLink,
1353 VisitType::Reload,
1354 ])
1355 .complement();
1356
1357 let infos = db.query_rows_and_then_cached(
1358 "SELECT h.frecency, h.title, h.url
1359 FROM moz_places h
1360 WHERE EXISTS (
1361 SELECT v.visit_type
1362 FROM moz_historyvisits v
1363 WHERE h.id = v.place_id
1364 AND (SUBSTR(h.url, 1, 6) == 'https:' OR SUBSTR(h.url, 1, 5) == 'http:')
1365 AND (h.last_visit_date_local + h.last_visit_date_remote) != 0
1366 AND ((1 << v.visit_type) & :allowed_types) != 0
1367 AND h.frecency >= :frecency_threshold AND
1368 NOT h.hidden
1369 )
1370 ORDER BY h.frecency DESC
1371 LIMIT :limit",
1372 rusqlite::named_params! {
1373 ":limit": num_items,
1374 ":allowed_types": allowed_types,
1375 ":frecency_threshold": frecency_threshold,
1376 },
1377 TopFrecentSiteInfo::from_row,
1378 )?;
1379 Ok(infos)
1380}
1381
1382pub fn get_visit_infos(
1383 db: &PlacesDb,
1384 start: Timestamp,
1385 end: Timestamp,
1386 exclude_types: VisitTransitionSet,
1387) -> Result<Vec<HistoryVisitInfo>> {
1388 let allowed_types = exclude_types.complement();
1389 let infos = db.query_rows_and_then_cached(
1390 "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1391 v.is_local
1392 FROM moz_places h
1393 JOIN moz_historyvisits v
1394 ON h.id = v.place_id
1395 WHERE v.visit_date BETWEEN :start AND :end
1396 AND ((1 << visit_type) & :allowed_types) != 0 AND
1397 NOT h.hidden
1398 ORDER BY v.visit_date",
1399 rusqlite::named_params! {
1400 ":start": start,
1401 ":end": end,
1402 ":allowed_types": allowed_types,
1403 },
1404 HistoryVisitInfo::from_row,
1405 )?;
1406 Ok(infos)
1407}
1408
1409pub fn get_visit_count(db: &PlacesDb, exclude_types: VisitTransitionSet) -> Result<i64> {
1410 let count = if exclude_types.is_empty() {
1411 db.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")?
1412 } else {
1413 let allowed_types = exclude_types.complement();
1414 db.query_row_and_then_cachable(
1415 "SELECT COUNT(*)
1416 FROM moz_historyvisits
1417 WHERE ((1 << visit_type) & :allowed_types) != 0",
1418 rusqlite::named_params! {
1419 ":allowed_types": allowed_types,
1420 },
1421 |r| r.get(0),
1422 true,
1423 )?
1424 };
1425 Ok(count)
1426}
1427
1428pub fn get_visit_count_for_host(
1429 db: &PlacesDb,
1430 host: &str,
1431 before: Timestamp,
1432 exclude_types: VisitTransitionSet,
1433) -> Result<i64> {
1434 let allowed_types = exclude_types.complement();
1435 let count = db.query_row_and_then_cachable(
1436 "SELECT COUNT(*)
1437 FROM moz_historyvisits
1438 JOIN moz_places ON moz_places.id = moz_historyvisits.place_id
1439 JOIN moz_origins ON moz_origins.id = moz_places.origin_id
1440 WHERE moz_origins.host = :host
1441 AND visit_date < :before
1442 AND ((1 << visit_type) & :allowed_types) != 0",
1443 rusqlite::named_params! {
1444 ":host": host,
1445 ":before": before,
1446 ":allowed_types": allowed_types,
1447 },
1448 |r| r.get(0),
1449 true,
1450 )?;
1451 Ok(count)
1452}
1453
1454pub fn get_visit_page(
1455 db: &PlacesDb,
1456 offset: i64,
1457 count: i64,
1458 exclude_types: VisitTransitionSet,
1459) -> Result<Vec<HistoryVisitInfo>> {
1460 let allowed_types = exclude_types.complement();
1461 let infos = db.query_rows_and_then_cached(
1462 "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1463 v.is_local
1464 FROM moz_places h
1465 JOIN moz_historyvisits v
1466 ON h.id = v.place_id
1467 WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1468 NOT h.hidden
1469 ORDER BY v.visit_date DESC, v.id
1470 LIMIT :count
1471 OFFSET :offset",
1472 rusqlite::named_params! {
1473 ":count": count,
1474 ":offset": offset,
1475 ":allowed_types": allowed_types,
1476 },
1477 HistoryVisitInfo::from_row,
1478 )?;
1479 Ok(infos)
1480}
1481
1482pub fn get_visit_page_with_bound(
1483 db: &PlacesDb,
1484 bound: i64,
1485 offset: i64,
1486 count: i64,
1487 exclude_types: VisitTransitionSet,
1488) -> Result<HistoryVisitInfosWithBound> {
1489 let allowed_types = exclude_types.complement();
1490 let infos = db.query_rows_and_then_cached(
1491 "SELECT h.url, h.title, v.visit_date, v.visit_type, h.hidden, h.preview_image_url,
1492 v.is_local
1493 FROM moz_places h
1494 JOIN moz_historyvisits v
1495 ON h.id = v.place_id
1496 WHERE ((1 << v.visit_type) & :allowed_types) != 0 AND
1497 NOT h.hidden
1498 AND v.visit_date <= :bound
1499 ORDER BY v.visit_date DESC, v.id
1500 LIMIT :count
1501 OFFSET :offset",
1502 rusqlite::named_params! {
1503 ":allowed_types": allowed_types,
1504 ":bound": bound,
1505 ":count": count,
1506 ":offset": offset,
1507 },
1508 HistoryVisitInfo::from_row,
1509 )?;
1510
1511 if let Some(l) = infos.last() {
1512 if l.timestamp.as_millis_i64() == bound {
1513 let offset = offset + infos.len() as i64;
1515 Ok(HistoryVisitInfosWithBound {
1516 infos,
1517 bound,
1518 offset,
1519 })
1520 } else {
1521 let bound = l.timestamp;
1522 let offset = infos
1523 .iter()
1524 .rev()
1525 .take_while(|i| i.timestamp == bound)
1526 .count() as i64;
1527 Ok(HistoryVisitInfosWithBound {
1528 infos,
1529 bound: bound.as_millis_i64(),
1530 offset,
1531 })
1532 }
1533 } else {
1534 Ok(HistoryVisitInfosWithBound {
1536 infos,
1537 bound: 0,
1538 offset: 0,
1539 })
1540 }
1541}
1542
1543#[cfg(test)]
1544mod tests {
1545 use super::history_sync::*;
1546 use super::*;
1547 use crate::history_sync::record::HistoryRecordVisit;
1548 use crate::storage::bookmarks::{insert_bookmark, InsertableItem};
1549 use crate::types::VisitTransitionSet;
1550 use crate::{api::places_api::ConnectionType, storage::bookmarks::BookmarkRootGuid};
1551 use pretty_assertions::assert_eq;
1552 use std::time::{Duration, SystemTime};
1553 use sync15::engine::CollSyncIds;
1554 use types::Timestamp;
1555
1556 #[test]
1557 fn test_get_visited_urls() {
1558 use std::collections::HashSet;
1559 use std::time::SystemTime;
1560 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1561 let now: Timestamp = SystemTime::now().into();
1562 let now_u64 = now.0;
1563 let to_add = [
1565 (
1566 "https://www.example.com/1",
1567 now_u64 - 200_100,
1568 false,
1569 (false, false),
1570 ),
1571 (
1572 "https://www.example.com/12",
1573 now_u64 - 200_000,
1574 false,
1575 (true, true),
1576 ),
1577 (
1578 "https://www.example.com/123",
1579 now_u64 - 10_000,
1580 true,
1581 (true, false),
1582 ),
1583 (
1584 "https://www.example.com/1234",
1585 now_u64 - 1000,
1586 false,
1587 (true, true),
1588 ),
1589 (
1590 "https://www.mozilla.com",
1591 now_u64 - 500,
1592 false,
1593 (false, false),
1594 ),
1595 ];
1596
1597 for &(url, when, remote, _) in &to_add {
1598 apply_observation(
1599 &conn,
1600 VisitObservation::new(Url::parse(url).unwrap())
1601 .with_at(Timestamp(when))
1602 .with_is_remote(remote)
1603 .with_visit_type(VisitType::Link),
1604 )
1605 .expect("Should apply visit");
1606 }
1607
1608 let visited_all = get_visited_urls(
1609 &conn,
1610 Timestamp(now_u64 - 200_000),
1611 Timestamp(now_u64 - 1000),
1612 true,
1613 )
1614 .unwrap()
1615 .into_iter()
1616 .collect::<HashSet<_>>();
1617
1618 let visited_local = get_visited_urls(
1619 &conn,
1620 Timestamp(now_u64 - 200_000),
1621 Timestamp(now_u64 - 1000),
1622 false,
1623 )
1624 .unwrap()
1625 .into_iter()
1626 .collect::<HashSet<_>>();
1627
1628 for &(url, ts, is_remote, (expected_in_all, expected_in_local)) in &to_add {
1629 let url = Url::parse(url).unwrap().to_string();
1631 assert_eq!(
1632 expected_in_local,
1633 visited_local.contains(&url),
1634 "Failed in local for {:?}",
1635 (url, ts, is_remote)
1636 );
1637 assert_eq!(
1638 expected_in_all,
1639 visited_all.contains(&url),
1640 "Failed in all for {:?}",
1641 (url, ts, is_remote)
1642 );
1643 }
1644 }
1645
1646 fn get_custom_observed_page<F>(conn: &mut PlacesDb, url: &str, custom: F) -> Result<PageInfo>
1647 where
1648 F: Fn(VisitObservation) -> VisitObservation,
1649 {
1650 let u = Url::parse(url)?;
1651 let obs = VisitObservation::new(u.clone()).with_visit_type(VisitType::Link);
1652 apply_observation(conn, custom(obs))?;
1653 Ok(fetch_page_info(conn, &u)?
1654 .expect("should have the page")
1655 .page)
1656 }
1657
1658 fn get_observed_page(conn: &mut PlacesDb, url: &str) -> Result<PageInfo> {
1659 get_custom_observed_page(conn, url, |o| o)
1660 }
1661
1662 fn get_tombstone_count(conn: &PlacesDb) -> u32 {
1663 let result: Result<Option<u32>> = conn.try_query_row(
1664 "SELECT COUNT(*) from moz_places_tombstones;",
1665 [],
1666 |row| Ok(row.get::<_, u32>(0)?),
1667 true,
1668 );
1669 result
1670 .expect("should have worked")
1671 .expect("should have got a value")
1672 }
1673
1674 #[test]
1675 fn test_visit_counts() -> Result<()> {
1676 error_support::init_for_tests();
1677 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1678 let url = Url::parse("https://www.example.com").expect("it's a valid url");
1679 let early_time = SystemTime::now() - Duration::new(60, 0);
1680 let late_time = SystemTime::now();
1681
1682 let rid1 = apply_observation(
1684 &conn,
1685 VisitObservation::new(url.clone())
1686 .with_visit_type(VisitType::Link)
1687 .with_at(Some(late_time.into())),
1688 )?
1689 .expect("should get a rowid");
1690
1691 let rid2 = apply_observation(
1692 &conn,
1693 VisitObservation::new(url.clone())
1694 .with_visit_type(VisitType::Link)
1695 .with_at(Some(early_time.into())),
1696 )?
1697 .expect("should get a rowid");
1698
1699 let mut pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1700 assert_eq!(pi.page.visit_count_local, 2);
1701 assert_eq!(pi.page.last_visit_date_local, late_time.into());
1702 assert_eq!(pi.page.visit_count_remote, 0);
1703 assert_eq!(pi.page.last_visit_date_remote.0, 0);
1704
1705 let rid3 = apply_observation(
1707 &conn,
1708 VisitObservation::new(url.clone())
1709 .with_visit_type(VisitType::Link)
1710 .with_at(Some(early_time.into()))
1711 .with_is_remote(true),
1712 )?
1713 .expect("should get a rowid");
1714
1715 let rid4 = apply_observation(
1716 &conn,
1717 VisitObservation::new(url.clone())
1718 .with_visit_type(VisitType::Link)
1719 .with_at(Some(late_time.into()))
1720 .with_is_remote(true),
1721 )?
1722 .expect("should get a rowid");
1723
1724 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1725 assert_eq!(pi.page.visit_count_local, 2);
1726 assert_eq!(pi.page.last_visit_date_local, late_time.into());
1727 assert_eq!(pi.page.visit_count_remote, 2);
1728 assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1729
1730 let sql = "DELETE FROM moz_historyvisits WHERE id = :row_id";
1734 conn.execute_cached(sql, &[(":row_id", &rid1)])?;
1736 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1737 assert_eq!(pi.page.visit_count_local, 1);
1738 assert_eq!(pi.page.last_visit_date_local, early_time.into());
1739 assert_eq!(pi.page.visit_count_remote, 2);
1740 assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1741
1742 conn.execute_cached(sql, &[(":row_id", &rid3)])?;
1744 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1745 assert_eq!(pi.page.visit_count_local, 1);
1746 assert_eq!(pi.page.last_visit_date_local, early_time.into());
1747 assert_eq!(pi.page.visit_count_remote, 1);
1748 assert_eq!(pi.page.last_visit_date_remote, late_time.into());
1749
1750 conn.execute_cached(sql, &[(":row_id", &rid2)])?;
1752 conn.execute_cached(sql, &[(":row_id", &rid4)])?;
1753 pi = fetch_page_info(&conn, &url)?.expect("should have the page");
1756 assert_eq!(pi.page.visit_count_local, 0);
1757 assert_eq!(pi.page.last_visit_date_local, Timestamp(0));
1758 assert_eq!(pi.page.visit_count_remote, 0);
1759 assert_eq!(pi.page.last_visit_date_remote, Timestamp(0));
1760 Ok(())
1761 }
1762
1763 #[test]
1764 fn test_get_visited() -> Result<()> {
1765 error_support::init_for_tests();
1766 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
1767
1768 let unicode_in_path = "http://www.example.com/tëst😀abc";
1769 let escaped_unicode_in_path = "http://www.example.com/t%C3%ABst%F0%9F%98%80abc";
1770
1771 let unicode_in_domain = "http://www.exämple😀123.com";
1772 let escaped_unicode_in_domain = "http://www.xn--exmple123-w2a24222l.com";
1773
1774 let to_add = [
1775 "https://www.example.com/1".to_string(),
1776 "https://www.example.com/12".to_string(),
1777 "https://www.example.com/123".to_string(),
1778 "https://www.example.com/1234".to_string(),
1779 "https://www.mozilla.com".to_string(),
1780 "https://www.firefox.com".to_string(),
1781 unicode_in_path.to_string() + "/1",
1782 escaped_unicode_in_path.to_string() + "/2",
1783 unicode_in_domain.to_string() + "/1",
1784 escaped_unicode_in_domain.to_string() + "/2",
1785 ];
1786
1787 for item in &to_add {
1788 apply_observation(
1789 &conn,
1790 VisitObservation::new(Url::parse(item).unwrap()).with_visit_type(VisitType::Link),
1791 )?;
1792 }
1793
1794 let to_search = [
1795 ("https://www.example.com".to_string(), false),
1796 ("https://www.example.com/1".to_string(), true),
1797 ("https://www.example.com/12".to_string(), true),
1798 ("https://www.example.com/123".to_string(), true),
1799 ("https://www.example.com/1234".to_string(), true),
1800 ("https://www.example.com/12345".to_string(), false),
1801 ("https://www.mozilla.com".to_string(), true),
1802 ("https://www.firefox.com".to_string(), true),
1803 ("https://www.mozilla.org".to_string(), false),
1804 ("https://www.example.com/1234".to_string(), true),
1806 ("https://www.example.com/12345".to_string(), false),
1807 (unicode_in_path.to_string() + "/1", true),
1810 (escaped_unicode_in_path.to_string() + "/2", true),
1811 (unicode_in_domain.to_string() + "/1", true),
1812 (escaped_unicode_in_domain.to_string() + "/2", true),
1813 (unicode_in_path.to_string() + "/2", true),
1815 (escaped_unicode_in_path.to_string() + "/1", true),
1816 (unicode_in_domain.to_string() + "/2", true),
1817 (escaped_unicode_in_domain.to_string() + "/1", true),
1818 ];
1819
1820 let urls = to_search
1821 .iter()
1822 .map(|(url, _expect)| Url::parse(url).unwrap())
1823 .collect::<Vec<_>>();
1824
1825 let visited = get_visited(&conn, urls).unwrap();
1826
1827 assert_eq!(visited.len(), to_search.len());
1828
1829 for (i, &did_see) in visited.iter().enumerate() {
1830 assert_eq!(
1831 did_see,
1832 to_search[i].1,
1833 "Wrong value in get_visited for '{}' (idx {}), want {}, have {}",
1834 to_search[i].0,
1835 i, to_search[i].1,
1837 did_see
1838 );
1839 }
1840 Ok(())
1841 }
1842
1843 #[test]
1844 fn test_get_visited_into() {
1845 error_support::init_for_tests();
1846 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
1847
1848 let u0 = Url::parse("https://www.example.com/1").unwrap();
1849 let u1 = Url::parse("https://www.example.com/12").unwrap();
1850 let u2 = Url::parse("https://www.example.com/123").unwrap();
1851 let u3 = Url::parse("https://www.example.com/1234").unwrap();
1852 let u4 = Url::parse("https://www.example.com/12345").unwrap();
1853
1854 let to_add = [(&u0, false), (&u1, false), (&u2, false), (&u3, true)];
1855 for (item, is_remote) in to_add {
1856 apply_observation(
1857 &conn,
1858 VisitObservation::new(item.clone())
1859 .with_visit_type(VisitType::Link)
1860 .with_is_remote(is_remote),
1861 )
1862 .unwrap();
1863 }
1864 insert_bookmark(
1867 &conn,
1868 crate::InsertableBookmark {
1869 parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
1870 position: crate::BookmarkPosition::Append,
1871 date_added: None,
1872 last_modified: None,
1873 guid: None,
1874 url: u4.clone(),
1875 title: Some("Title".to_string()),
1876 }
1877 .into(),
1878 )
1879 .unwrap();
1880
1881 let mut results = [false; 12];
1882
1883 let get_visited_request = [
1884 (2, u1.clone()),
1886 (1, u0),
1887 (4, u2),
1889 (6, Url::parse("https://www.example.com/123456").unwrap()),
1892 (8, u1),
1895 (10, u3),
1897 (11, u4),
1898 ];
1899
1900 get_visited_into(&conn, &get_visited_request, &mut results).unwrap();
1901 let expect = [
1902 false, true, true, false, true, false, false, false, true, false, true, false, ];
1915
1916 assert_eq!(expect, results);
1917 }
1918
1919 #[test]
1920 fn test_delete_visited() {
1921 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
1922 let late: Timestamp = SystemTime::now().into();
1923 let early: Timestamp = (SystemTime::now() - Duration::from_secs(30)).into();
1924 let url1 = Url::parse("https://www.example.com/1").unwrap();
1925 let url2 = Url::parse("https://www.example.com/2").unwrap();
1926 let url3 = Url::parse("https://www.example.com/3").unwrap();
1927 let url4 = Url::parse("https://www.example.com/4").unwrap();
1928 let to_add = [
1930 (&url1, early),
1932 (&url1, late),
1933 (&url2, late),
1935 (&url3, early),
1937 (&url4, late),
1939 ];
1940
1941 for &(url, when) in &to_add {
1942 apply_observation(
1943 &conn,
1944 VisitObservation::new(url.clone())
1945 .with_at(when)
1946 .with_visit_type(VisitType::Link),
1947 )
1948 .expect("Should apply visit");
1949 }
1950 let pi = fetch_page_info(&conn, &url1)
1952 .expect("should work")
1953 .expect("should get the page");
1954 assert_eq!(pi.page.visit_count_local, 2);
1955
1956 let pi2 = fetch_page_info(&conn, &url2)
1957 .expect("should work")
1958 .expect("should get the page");
1959 assert_eq!(pi2.page.visit_count_local, 1);
1960
1961 let pi3 = fetch_page_info(&conn, &url3)
1962 .expect("should work")
1963 .expect("should get the page");
1964 assert_eq!(pi3.page.visit_count_local, 1);
1965
1966 let pi4 = fetch_page_info(&conn, &url4)
1967 .expect("should work")
1968 .expect("should get the page");
1969 assert_eq!(pi4.page.visit_count_local, 1);
1970
1971 conn.execute_cached(
1972 &format!(
1973 "UPDATE moz_places set sync_status = {}
1974 WHERE url = 'https://www.example.com/4'",
1975 (SyncStatus::Normal as u8)
1976 ),
1977 [],
1978 )
1979 .expect("should work");
1980
1981 delete_visits_between(&conn, late, Timestamp::now()).expect("should work");
1983 let pi = fetch_page_info(&conn, &url1)
1985 .expect("should work")
1986 .expect("should get the page");
1987 assert_eq!(pi.page.visit_count_local, 1);
1988
1989 assert!(fetch_page_info(&conn, &url2)
1991 .expect("should work")
1992 .is_none());
1993
1994 let pi3 = fetch_page_info(&conn, &url3)
1996 .expect("should work")
1997 .expect("should get the page");
1998 assert_eq!(pi3.page.visit_count_local, 1);
1999
2000 assert!(fetch_page_info(&conn, &url4)
2002 .expect("should work")
2003 .is_none());
2004 assert_eq!(get_tombstone_count(&conn), 1);
2006 }
2009
2010 #[test]
2011 fn test_change_counter() -> Result<()> {
2012 error_support::init_for_tests();
2013 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2014 let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2015 apply_observation(
2017 &conn,
2018 VisitObservation::new(pi.url.clone()).with_title(Some("new title".into())),
2019 )?;
2020 pi = fetch_page_info(&conn, &pi.url)?
2021 .expect("page should exist")
2022 .page;
2023 assert_eq!(pi.title, "new title");
2024 assert_eq!(pi.preview_image_url, None);
2025 assert_eq!(pi.sync_change_counter, 2);
2026 apply_observation(
2028 &conn,
2029 VisitObservation::new(pi.url.clone()).with_preview_image_url(Some(
2030 Url::parse("https://www.example.com/preview.png").unwrap(),
2031 )),
2032 )?;
2033 pi = fetch_page_info(&conn, &pi.url)?
2034 .expect("page should exist")
2035 .page;
2036 assert_eq!(pi.title, "new title");
2037 assert_eq!(
2038 pi.preview_image_url,
2039 Some(Url::parse("https://www.example.com/preview.png").expect("parsed"))
2040 );
2041 assert_eq!(pi.sync_change_counter, 2);
2042 Ok(())
2043 }
2044
2045 #[test]
2046 fn test_status_columns() -> Result<()> {
2047 error_support::init_for_tests();
2048 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2049 let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2051 assert_eq!(pi.sync_change_counter, 1);
2052 conn.execute_cached(
2053 "UPDATE moz_places
2054 SET frecency = 100
2055 WHERE id = :id",
2056 &[(":id", &pi.row_id)],
2057 )?;
2058 let mut pi2 = get_observed_page(&mut conn, "http://example.com/2")?;
2060 conn.execute_cached(
2061 "UPDATE moz_places
2062 SET sync_status = :status,
2063 sync_change_counter = 0,
2064 frecency = 50
2065 WHERE id = :id",
2066 &[
2067 (":status", &(SyncStatus::New as u8) as &dyn rusqlite::ToSql),
2068 (":id", &pi2.row_id),
2069 ],
2070 )?;
2071
2072 let mut pi3 = get_observed_page(&mut conn, "http://example.com/3")?;
2075 conn.execute_cached(
2076 "UPDATE moz_places
2077 SET sync_status = :status,
2078 sync_change_counter = 1,
2079 frecency = 10
2080 WHERE id = :id",
2081 &[
2082 (":status", &(SyncStatus::New as u8) as &dyn ToSql),
2083 (":id", &pi3.row_id),
2084 ],
2085 )?;
2086
2087 let outgoing = fetch_outgoing(&conn, 2, 3)?;
2088 assert_eq!(outgoing.len(), 2, "should have restricted to the limit");
2089 assert!(outgoing[0].envelope.id != outgoing[1].envelope.id);
2091 assert!(outgoing[0].envelope.id == pi.guid || outgoing[0].envelope.id == pi2.guid);
2092 assert!(outgoing[1].envelope.id == pi.guid || outgoing[1].envelope.id == pi2.guid);
2093 finish_outgoing(&conn)?;
2094
2095 pi = fetch_page_info(&conn, &pi.url)?
2096 .expect("page should exist")
2097 .page;
2098 assert_eq!(pi.sync_change_counter, 0);
2099 pi2 = fetch_page_info(&conn, &pi2.url)?
2100 .expect("page should exist")
2101 .page;
2102 assert_eq!(pi2.sync_change_counter, 0);
2103 assert_eq!(pi2.sync_status, SyncStatus::Normal);
2104
2105 pi3 = fetch_page_info(&conn, &pi3.url)?
2108 .expect("page should exist")
2109 .page;
2110 assert_eq!(pi3.sync_change_counter, 0);
2111 assert_eq!(pi3.sync_status, SyncStatus::Normal);
2112 Ok(())
2113 }
2114
2115 #[test]
2116 fn test_delete_visits_for() -> Result<()> {
2117 use crate::storage::bookmarks::{
2118 self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2119 };
2120
2121 let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2122
2123 struct TestPage {
2124 href: &'static str,
2125 synced: bool,
2126 bookmark_title: Option<&'static str>,
2127 keyword: Option<&'static str>,
2128 }
2129
2130 fn page_has_tombstone(conn: &PlacesDb, guid: &SyncGuid) -> Result<bool> {
2131 let exists = conn
2132 .try_query_one::<bool, _>(
2133 "SELECT EXISTS(SELECT 1 FROM moz_places_tombstones
2134 WHERE guid = :guid)",
2135 rusqlite::named_params! { ":guid" : guid },
2136 false,
2137 )?
2138 .unwrap_or_default();
2139 Ok(exists)
2140 }
2141
2142 fn page_has_visit_tombstones(conn: &PlacesDb, page_id: RowId) -> Result<bool> {
2143 let exists = conn
2144 .try_query_one::<bool, _>(
2145 "SELECT EXISTS(SELECT 1 FROM moz_historyvisit_tombstones
2146 WHERE place_id = :page_id)",
2147 rusqlite::named_params! { ":page_id": page_id },
2148 false,
2149 )?
2150 .unwrap_or_default();
2151 Ok(exists)
2152 }
2153
2154 let pages = &[
2155 TestPage {
2158 href: "http://example.com/a",
2159 synced: true,
2160 bookmark_title: Some("A"),
2161 keyword: None,
2162 },
2163 TestPage {
2166 href: "http://example.com/b",
2167 synced: true,
2168 bookmark_title: None,
2169 keyword: None,
2170 },
2171 TestPage {
2174 href: "http://example.com/c",
2175 synced: false,
2176 bookmark_title: None,
2177 keyword: Some("one"),
2178 },
2179 TestPage {
2182 href: "http://example.com/d",
2183 synced: false,
2184 bookmark_title: None,
2185 keyword: None,
2186 },
2187 ];
2188 for page in pages {
2189 let url = Url::parse(page.href)?;
2190 let obs = VisitObservation::new(url.clone())
2191 .with_visit_type(VisitType::Link)
2192 .with_at(Some(SystemTime::now().into()));
2193 apply_observation(&db, obs)?;
2194
2195 if page.synced {
2196 db.execute_cached(
2197 &format!(
2198 "UPDATE moz_places
2199 SET sync_status = {}
2200 WHERE url_hash = hash(:url) AND
2201 url = :url",
2202 (SyncStatus::Normal as u8)
2203 ),
2204 &[(":url", &url.as_str())],
2205 )?;
2206 }
2207
2208 if let Some(title) = page.bookmark_title {
2209 bookmarks::insert_bookmark(
2210 &db,
2211 InsertableBookmark {
2212 parent_guid: BookmarkRootGuid::Unfiled.into(),
2213 position: BookmarkPosition::Append,
2214 date_added: None,
2215 last_modified: None,
2216 guid: None,
2217 url: url.clone(),
2218 title: Some(title.to_owned()),
2219 }
2220 .into(),
2221 )?;
2222 }
2223
2224 if let Some(keyword) = page.keyword {
2225 db.execute_cached(
2228 "INSERT INTO moz_keywords(place_id, keyword)
2229 SELECT id, :keyword
2230 FROM moz_places
2231 WHERE url_hash = hash(:url) AND
2232 url = :url",
2233 &[(":url", &url.as_str()), (":keyword", &keyword)],
2234 )?;
2235 }
2236
2237 let (info, _) =
2239 fetch_visits(&db, &url, 0)?.expect("Should return visits for test page");
2240 delete_visits_for(&db, &info.guid)?;
2241
2242 match (
2243 page.synced,
2244 page.bookmark_title.is_some() || page.keyword.is_some(),
2245 ) {
2246 (true, true) => {
2247 let (_, visits) = fetch_visits(&db, &url, 0)?
2248 .expect("Shouldn't delete synced page with foreign count");
2249 assert!(
2250 visits.is_empty(),
2251 "Should delete all visits from synced page with foreign count"
2252 );
2253 assert!(
2254 !page_has_tombstone(&db, &info.guid)?,
2255 "Shouldn't insert tombstone for synced page with foreign count"
2256 );
2257 assert!(
2258 page_has_visit_tombstones(&db, info.row_id)?,
2259 "Should insert visit tombstones for synced page with foreign count"
2260 );
2261 }
2262 (true, false) => {
2263 assert!(
2264 fetch_visits(&db, &url, 0)?.is_none(),
2265 "Should delete synced page"
2266 );
2267 assert!(
2268 page_has_tombstone(&db, &info.guid)?,
2269 "Should insert tombstone for synced page"
2270 );
2271 assert!(
2272 !page_has_visit_tombstones(&db, info.row_id)?,
2273 "Shouldn't insert visit tombstones for synced page"
2274 );
2275 }
2276 (false, true) => {
2277 let (_, visits) = fetch_visits(&db, &url, 0)?
2278 .expect("Shouldn't delete page with foreign count");
2279 assert!(
2280 visits.is_empty(),
2281 "Should delete all visits from page with foreign count"
2282 );
2283 assert!(
2284 !page_has_tombstone(&db, &info.guid)?,
2285 "Shouldn't insert tombstone for page with foreign count"
2286 );
2287 assert!(
2288 !page_has_visit_tombstones(&db, info.row_id)?,
2289 "Shouldn't insert visit tombstones for page with foreign count"
2290 );
2291 }
2292 (false, false) => {
2293 assert!(fetch_visits(&db, &url, 0)?.is_none(), "Should delete page");
2294 assert!(
2295 !page_has_tombstone(&db, &info.guid)?,
2296 "Shouldn't insert tombstone for page"
2297 );
2298 assert!(
2299 !page_has_visit_tombstones(&db, info.row_id)?,
2300 "Shouldn't insert visit tombstones for page"
2301 );
2302 }
2303 }
2304 }
2305
2306 Ok(())
2307 }
2308
2309 #[test]
2310 fn test_tombstones() -> Result<()> {
2311 error_support::init_for_tests();
2312 let db = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2313 let url = Url::parse("https://example.com")?;
2314 let obs = VisitObservation::new(url.clone())
2315 .with_visit_type(VisitType::Link)
2316 .with_at(Some(SystemTime::now().into()));
2317 apply_observation(&db, obs)?;
2318 let guid = url_to_guid(&db, &url)?.expect("should exist");
2319
2320 delete_visits_for(&db, &guid)?;
2321
2322 assert_eq!(get_tombstone_count(&db), 0);
2324
2325 let obs = VisitObservation::new(url.clone())
2326 .with_visit_type(VisitType::Link)
2327 .with_at(Some(SystemTime::now().into()));
2328 apply_observation(&db, obs)?;
2329 let new_guid = url_to_guid(&db, &url)?.expect("should exist");
2330
2331 db.execute_cached(
2333 &format!(
2334 "UPDATE moz_places
2335 SET sync_status = {}
2336 WHERE guid = :guid",
2337 (SyncStatus::Normal as u8)
2338 ),
2339 &[(":guid", &new_guid)],
2340 )?;
2341 delete_visits_for(&db, &new_guid)?;
2342 assert_eq!(get_tombstone_count(&db), 1);
2343 Ok(())
2344 }
2345
2346 #[test]
2347 fn test_reset() -> Result<()> {
2348 fn mark_all_as_synced(db: &PlacesDb) -> Result<()> {
2349 db.execute_cached(
2350 &format!(
2351 "UPDATE moz_places set sync_status = {}",
2352 (SyncStatus::Normal as u8)
2353 ),
2354 [],
2355 )?;
2356 Ok(())
2357 }
2358
2359 error_support::init_for_tests();
2360 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2361
2362 put_meta(&conn, GLOBAL_SYNCID_META_KEY, &"syncAAAAAAAA")?;
2364 put_meta(&conn, COLLECTION_SYNCID_META_KEY, &"syncBBBBBBBB")?;
2365 put_meta(&conn, LAST_SYNC_META_KEY, &12345)?;
2366
2367 delete_everything(&conn)?;
2370
2371 let mut pi = get_observed_page(&mut conn, "http://example.com")?;
2372 mark_all_as_synced(&conn)?;
2373 pi = fetch_page_info(&conn, &pi.url)?
2374 .expect("page should exist")
2375 .page;
2376 assert_eq!(pi.sync_change_counter, 1);
2377 assert_eq!(pi.sync_status, SyncStatus::Normal);
2378
2379 let sync_ids = CollSyncIds {
2380 global: SyncGuid::random(),
2381 coll: SyncGuid::random(),
2382 };
2383 history_sync::reset(&conn, &EngineSyncAssociation::Connected(sync_ids.clone()))?;
2384
2385 assert_eq!(
2386 get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?,
2387 Some(sync_ids.global)
2388 );
2389 assert_eq!(
2390 get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2391 Some(sync_ids.coll)
2392 );
2393 assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2394 assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2395
2396 pi = fetch_page_info(&conn, &pi.url)?
2397 .expect("page should exist")
2398 .page;
2399 assert_eq!(pi.sync_change_counter, 0);
2400 assert_eq!(pi.sync_status, SyncStatus::New);
2401 let outgoing = fetch_outgoing(&conn, 100, 100)?;
2403 assert_eq!(outgoing.len(), 1);
2404
2405 mark_all_as_synced(&conn)?;
2406 assert!(fetch_outgoing(&conn, 100, 100)?.is_empty());
2407 history_sync::reset(&conn, &EngineSyncAssociation::Disconnected)?;
2412
2413 assert_eq!(get_meta::<SyncGuid>(&conn, GLOBAL_SYNCID_META_KEY)?, None);
2414 assert_eq!(
2415 get_meta::<SyncGuid>(&conn, COLLECTION_SYNCID_META_KEY)?,
2416 None
2417 );
2418 assert_eq!(get_meta::<i64>(&conn, LAST_SYNC_META_KEY)?, Some(0));
2419 assert!(get_meta::<Timestamp>(&conn, DELETION_HIGH_WATER_MARK_META_KEY)?.is_some());
2420
2421 Ok(())
2422 }
2423
2424 #[test]
2425 fn test_fetch_visits() -> Result<()> {
2426 error_support::init_for_tests();
2427 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
2428 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2429 assert_eq!(fetch_visits(&conn, &pi.url, 0).unwrap().unwrap().1.len(), 0);
2430 assert_eq!(fetch_visits(&conn, &pi.url, 1).unwrap().unwrap().1.len(), 1);
2431 Ok(())
2432 }
2433
2434 #[test]
2435 fn test_apply_synced_reconciliation() -> Result<()> {
2436 error_support::init_for_tests();
2437 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2438 let mut pi = get_observed_page(&mut conn, "http://example.com/1")?;
2439 assert_eq!(pi.sync_status, SyncStatus::New);
2440 assert_eq!(pi.sync_change_counter, 1);
2441 apply_synced_reconciliation(&conn, &pi.guid)?;
2442 pi = fetch_page_info(&conn, &pi.url)?
2443 .expect("page should exist")
2444 .page;
2445 assert_eq!(pi.sync_status, SyncStatus::Normal);
2446 assert_eq!(pi.sync_change_counter, 0);
2447 Ok(())
2448 }
2449
2450 #[test]
2451 fn test_apply_synced_deletion_new() -> Result<()> {
2452 error_support::init_for_tests();
2453 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2454 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2455 assert_eq!(pi.sync_status, SyncStatus::New);
2456 apply_synced_deletion(&conn, &pi.guid)?;
2457 assert!(
2458 fetch_page_info(&conn, &pi.url)?.is_none(),
2459 "should have been deleted"
2460 );
2461 assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2462 Ok(())
2463 }
2464
2465 #[test]
2466 fn test_apply_synced_deletion_normal() -> Result<()> {
2467 error_support::init_for_tests();
2468 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2469 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2470 assert_eq!(pi.sync_status, SyncStatus::New);
2471 conn.execute_cached(
2472 &format!(
2473 "UPDATE moz_places set sync_status = {}",
2474 (SyncStatus::Normal as u8)
2475 ),
2476 [],
2477 )?;
2478
2479 apply_synced_deletion(&conn, &pi.guid)?;
2480 assert!(
2481 fetch_page_info(&conn, &pi.url)?.is_none(),
2482 "should have been deleted"
2483 );
2484 assert_eq!(get_tombstone_count(&conn), 0, "should be no tombstones");
2485 Ok(())
2486 }
2487
2488 #[test]
2489 fn test_apply_synced_deletions_deletes_visits_but_not_page_if_bookmark_exists() -> Result<()> {
2490 error_support::init_for_tests();
2491 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite)?;
2492 let pi = get_observed_page(&mut conn, "http://example.com/1")?;
2493 let item = InsertableItem::Bookmark {
2494 b: crate::InsertableBookmark {
2495 parent_guid: BookmarkRootGuid::Unfiled.as_guid(),
2496 position: crate::BookmarkPosition::Append,
2497 date_added: None,
2498 last_modified: None,
2499 guid: None,
2500 url: pi.url.clone(),
2501 title: Some("Title".to_string()),
2502 },
2503 };
2504 insert_bookmark(&conn, item).unwrap();
2505 apply_synced_deletion(&conn, &pi.guid)?;
2506 let page_info =
2507 fetch_page_info(&conn, &pi.url)?.expect("The places entry should have remained");
2508 assert!(
2509 page_info.last_visit_id.is_none(),
2510 "Should have no more visits"
2511 );
2512 Ok(())
2513 }
2514
2515 fn assert_tombstones(c: &PlacesDb, expected: &[(RowId, Timestamp)]) {
2516 let mut expected: Vec<(RowId, Timestamp)> = expected.into();
2517 expected.sort();
2518 let mut tombstones = c
2519 .query_rows_and_then(
2520 "SELECT place_id, visit_date FROM moz_historyvisit_tombstones",
2521 [],
2522 |row| -> Result<_> { Ok((row.get::<_, RowId>(0)?, row.get::<_, Timestamp>(1)?)) },
2523 )
2524 .unwrap();
2525 tombstones.sort();
2526 assert_eq!(expected, tombstones);
2527 }
2528
2529 #[test]
2530 fn test_visit_tombstones() {
2531 use url::Url;
2532 error_support::init_for_tests();
2533 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2534 let now = Timestamp::now();
2535
2536 let urls = &[
2537 Url::parse("http://example.com/1").unwrap(),
2538 Url::parse("http://example.com/2").unwrap(),
2539 ];
2540
2541 let dates = &[
2542 Timestamp(now.0 - 10000),
2543 Timestamp(now.0 - 5000),
2544 Timestamp(now.0),
2545 ];
2546 for url in urls {
2547 for &date in dates {
2548 get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2549 }
2550 }
2551 delete_place_visit_at_time(&conn, &urls[0], dates[1]).unwrap();
2552 delete_visits_between(&conn, Timestamp(now.0 - 4000), Timestamp::now()).unwrap();
2554
2555 let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2556 assert_eq!(
2557 visits0,
2558 &[FetchedVisit {
2559 is_local: true,
2560 visit_date: dates[0],
2561 visit_type: Some(VisitType::Link)
2562 },]
2563 );
2564
2565 assert!(
2566 !visits0.iter().any(|v| v.visit_date == dates[1]),
2567 "Shouldn't have deleted visit"
2568 );
2569
2570 let (info1, mut visits1) = fetch_visits(&conn, &urls[1], 100).unwrap().unwrap();
2571 visits1.sort_by_key(|v| v.visit_date);
2572 assert_eq!(
2575 visits1,
2576 &[
2577 FetchedVisit {
2578 is_local: true,
2579 visit_date: dates[0],
2580 visit_type: Some(VisitType::Link)
2581 },
2582 FetchedVisit {
2583 is_local: true,
2584 visit_date: dates[1],
2585 visit_type: Some(VisitType::Link)
2586 },
2587 ]
2588 );
2589
2590 apply_synced_visits(
2592 &conn,
2593 &info0.guid,
2594 &info0.url,
2595 &Some(info0.title.clone()),
2596 &dates
2598 .iter()
2599 .map(|&d| HistoryRecordVisit {
2600 date: d.into(),
2601 transition: VisitType::Link as u8,
2602 unknown_fields: UnknownFields::new(),
2603 })
2604 .collect::<Vec<_>>(),
2605 &UnknownFields::new(),
2606 )
2607 .unwrap();
2608
2609 let (info0, visits0) = fetch_visits(&conn, &urls[0], 100).unwrap().unwrap();
2610 assert_eq!(
2611 visits0,
2612 &[FetchedVisit {
2613 is_local: true,
2614 visit_date: dates[0],
2615 visit_type: Some(VisitType::Link)
2616 }]
2617 );
2618
2619 assert_tombstones(
2620 &conn,
2621 &[
2622 (info0.row_id, dates[1]),
2623 (info0.row_id, dates[2]),
2624 (info1.row_id, dates[2]),
2625 ],
2626 );
2627
2628 delete_place_visit_at_time(&conn, &urls[0], dates[0]).unwrap();
2631
2632 assert!(fetch_visits(&conn, &urls[0], 100).unwrap().is_none());
2633
2634 assert_tombstones(&conn, &[(info1.row_id, dates[2])]);
2635 }
2636
2637 #[test]
2638 fn test_delete_local() {
2639 use crate::frecency::DEFAULT_FRECENCY_SETTINGS;
2640 use crate::storage::bookmarks::{
2641 self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark, InsertableItem,
2642 };
2643 use url::Url;
2644 error_support::init_for_tests();
2645 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2646 let ts = Timestamp::now().0 - 5_000_000;
2647 for o in 0..10 {
2649 for i in 0..11 {
2650 for t in 0..3 {
2651 get_custom_observed_page(
2652 &mut conn,
2653 &format!("http://www.example{}.com/{}", o, i),
2654 |obs| obs.with_at(Timestamp(ts + t * 1000 + i * 10_000 + o * 100_000)),
2655 )
2656 .unwrap();
2657 }
2658 }
2659 }
2660 let b0 = (
2662 SyncGuid::from("aaaaaaaaaaaa"),
2663 Url::parse("http://www.example3.com/5").unwrap(),
2664 );
2665 let b1 = (
2666 SyncGuid::from("bbbbbbbbbbbb"),
2667 Url::parse("http://www.example6.com/10").unwrap(),
2668 );
2669 let b2 = (
2670 SyncGuid::from("cccccccccccc"),
2671 Url::parse("http://www.example9.com/4").unwrap(),
2672 );
2673 for (guid, url) in &[&b0, &b1, &b2] {
2674 bookmarks::insert_bookmark(
2675 &conn,
2676 InsertableItem::Bookmark {
2677 b: InsertableBookmark {
2678 parent_guid: BookmarkRootGuid::Unfiled.into(),
2679 position: BookmarkPosition::Append,
2680 date_added: None,
2681 last_modified: None,
2682 guid: Some(guid.clone()),
2683 url: url.clone(),
2684 title: None,
2685 },
2686 },
2687 )
2688 .unwrap();
2689 }
2690
2691 conn.execute_all(&[
2693 &format!(
2694 "UPDATE moz_places set sync_status = {}",
2695 (SyncStatus::Normal as u8)
2696 ),
2697 &format!(
2698 "UPDATE moz_bookmarks set syncStatus = {}",
2699 (SyncStatus::Normal as u8)
2700 ),
2701 ])
2702 .unwrap();
2703
2704 delete_visits_for(
2706 &conn,
2707 &url_to_guid(&conn, &Url::parse("http://www.example8.com/5").unwrap())
2708 .unwrap()
2709 .unwrap(),
2710 )
2711 .unwrap();
2712
2713 delete_place_visit_at_time(
2714 &conn,
2715 &Url::parse("http://www.example10.com/5").unwrap(),
2716 Timestamp(ts + 5 * 10_000 + 10 * 100_000),
2717 )
2718 .unwrap();
2719
2720 assert!(bookmarks::delete_bookmark(&conn, &b0.0).unwrap());
2721
2722 delete_everything(&conn).unwrap();
2723
2724 let places = conn
2725 .query_rows_and_then(
2726 "SELECT * FROM moz_places ORDER BY url ASC",
2727 [],
2728 PageInfo::from_row,
2729 )
2730 .unwrap();
2731 assert_eq!(places.len(), 2);
2732 assert_eq!(places[0].url, b1.1);
2733 assert_eq!(places[1].url, b2.1);
2734 for p in &places {
2735 assert_eq!(
2736 p.frecency,
2737 DEFAULT_FRECENCY_SETTINGS.unvisited_bookmark_bonus
2738 );
2739 assert_eq!(p.visit_count_local, 0);
2740 assert_eq!(p.visit_count_remote, 0);
2741 assert_eq!(p.last_visit_date_local, Timestamp(0));
2742 assert_eq!(p.last_visit_date_remote, Timestamp(0));
2743 }
2744
2745 let counts_sql = [
2746 (0i64, "SELECT COUNT(*) FROM moz_historyvisits"),
2747 (2, "SELECT COUNT(*) FROM moz_origins"),
2748 (7, "SELECT COUNT(*) FROM moz_bookmarks"), (1, "SELECT COUNT(*) FROM moz_bookmarks_deleted"),
2750 (0, "SELECT COUNT(*) FROM moz_historyvisit_tombstones"),
2751 (0, "SELECT COUNT(*) FROM moz_places_tombstones"),
2752 ];
2753 for (want, query) in &counts_sql {
2754 assert_eq!(
2755 *want,
2756 conn.query_one::<i64>(query).unwrap(),
2757 "Unexpected value for {}",
2758 query
2759 );
2760 }
2761 }
2762
2763 #[test]
2764 fn test_delete_everything() {
2765 use crate::storage::bookmarks::{
2766 self, BookmarkPosition, BookmarkRootGuid, InsertableBookmark,
2767 };
2768 use url::Url;
2769 error_support::init_for_tests();
2770 let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2771 let start = Timestamp::now();
2772
2773 let urls = &[
2774 Url::parse("http://example.com/1").unwrap(),
2775 Url::parse("http://example.com/2").unwrap(),
2776 Url::parse("http://example.com/3").unwrap(),
2777 ];
2778
2779 let dates = &[
2780 Timestamp(start.0 - 10000),
2781 Timestamp(start.0 - 5000),
2782 Timestamp(start.0),
2783 ];
2784
2785 for url in urls {
2786 for &date in dates {
2787 get_custom_observed_page(&mut conn, url.as_str(), |o| o.with_at(date)).unwrap();
2788 }
2789 }
2790
2791 bookmarks::insert_bookmark(
2792 &conn,
2793 InsertableBookmark {
2794 parent_guid: BookmarkRootGuid::Unfiled.into(),
2795 position: BookmarkPosition::Append,
2796 date_added: None,
2797 last_modified: None,
2798 guid: Some("bookmarkAAAA".into()),
2799 url: urls[2].clone(),
2800 title: Some("A".into()),
2801 }
2802 .into(),
2803 )
2804 .expect("Should insert bookmark with URL 3");
2805
2806 conn.execute(
2807 "WITH entries(url, input) AS (
2808 VALUES(:url1, 'hi'), (:url3, 'bye')
2809 )
2810 INSERT INTO moz_inputhistory(place_id, input, use_count)
2811 SELECT h.id, e.input, 1
2812 FROM entries e
2813 JOIN moz_places h ON h.url_hash = hash(e.url) AND
2814 h.url = e.url",
2815 &[(":url1", &urls[1].as_str()), (":url3", &urls[2].as_str())],
2816 )
2817 .expect("Should insert autocomplete history entries");
2818
2819 delete_everything(&conn).expect("Should delete everything except URL 3");
2820
2821 std::thread::sleep(std::time::Duration::from_millis(50));
2822
2823 let mut places_stmt = conn.prepare("SELECT url FROM moz_places").unwrap();
2826 let remaining_urls: Vec<String> = places_stmt
2827 .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2828 .expect("Should fetch remaining URLs")
2829 .map(std::result::Result::unwrap)
2830 .collect();
2831 assert_eq!(remaining_urls, &["http://example.com/3"]);
2832
2833 let mut input_stmt = conn.prepare("SELECT input FROM moz_inputhistory").unwrap();
2834 let remaining_inputs: Vec<String> = input_stmt
2835 .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2836 .expect("Should fetch remaining autocomplete history entries")
2837 .map(std::result::Result::unwrap)
2838 .collect();
2839 assert_eq!(remaining_inputs, &["bye"]);
2840
2841 bookmarks::delete_bookmark(&conn, &"bookmarkAAAA".into())
2842 .expect("Should delete bookmark with URL 3");
2843
2844 delete_everything(&conn).expect("Should delete all URLs");
2845
2846 assert_eq!(
2847 0,
2848 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2849 .unwrap(),
2850 );
2851
2852 apply_synced_visits(
2853 &conn,
2854 &SyncGuid::random(),
2855 &url::Url::parse("http://www.example.com/123").unwrap(),
2856 &None,
2857 &[
2858 HistoryRecordVisit {
2859 date: Timestamp::now().into(),
2861 transition: VisitType::Link as u8,
2862 unknown_fields: UnknownFields::new(),
2863 },
2864 HistoryRecordVisit {
2865 date: start.into(),
2867 transition: VisitType::Link as u8,
2868 unknown_fields: UnknownFields::new(),
2869 },
2870 ],
2871 &UnknownFields::new(),
2872 )
2873 .unwrap();
2874 assert_eq!(
2875 1,
2876 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2877 .unwrap(),
2878 );
2879 assert_eq!(
2881 1,
2882 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2883 .unwrap(),
2884 );
2885
2886 apply_synced_visits(
2888 &conn,
2889 &SyncGuid::random(),
2890 &url::Url::parse("http://www.example.com/1234").unwrap(),
2891 &None,
2892 &[HistoryRecordVisit {
2893 date: start.into(),
2894 transition: VisitType::Link as u8,
2895 unknown_fields: UnknownFields::new(),
2896 }],
2897 &UnknownFields::new(),
2898 )
2899 .unwrap();
2900 assert_eq!(
2902 1,
2903 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_places")
2904 .unwrap(),
2905 );
2906 assert_eq!(
2907 1,
2908 conn.query_one::<i64>("SELECT COUNT(*) FROM moz_historyvisits")
2909 .unwrap(),
2910 );
2911 }
2912
2913 #[test]
2915 fn test_delete_everything_deletes_origins() {
2916 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2917
2918 let u = Url::parse("https://www.reddit.com/r/climbing").expect("Should parse URL");
2919 let ts = Timestamp::now().0 - 5_000_000;
2920 let obs = VisitObservation::new(u)
2921 .with_visit_type(VisitType::Link)
2922 .with_at(Timestamp(ts));
2923 apply_observation(&conn, obs).expect("Should apply observation");
2924
2925 delete_everything(&conn).expect("Should delete everything");
2926
2927 let origin_count = conn
2929 .query_one::<i64>("SELECT COUNT(*) FROM moz_origins")
2930 .expect("Should fetch origin count");
2931 assert_eq!(0, origin_count);
2932 }
2933
2934 #[test]
2935 fn test_apply_observation_updates_origins() {
2936 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2937
2938 let obs_for_a = VisitObservation::new(
2939 Url::parse("https://example1.com/a").expect("Should parse URL A"),
2940 )
2941 .with_visit_type(VisitType::Link)
2942 .with_at(Timestamp(Timestamp::now().0 - 5_000_000));
2943 apply_observation(&conn, obs_for_a).expect("Should apply observation for A");
2944
2945 let obs_for_b = VisitObservation::new(
2946 Url::parse("https://example2.com/b").expect("Should parse URL B"),
2947 )
2948 .with_visit_type(VisitType::Link)
2949 .with_at(Timestamp(Timestamp::now().0 - 2_500_000));
2950 apply_observation(&conn, obs_for_b).expect("Should apply observation for B");
2951
2952 let mut origins = conn
2953 .prepare("SELECT host FROM moz_origins")
2954 .expect("Should prepare origins statement")
2955 .query_and_then([], |row| -> rusqlite::Result<_> { row.get::<_, String>(0) })
2956 .expect("Should fetch all origins")
2957 .map(|r| r.expect("Should get origin from row"))
2958 .collect::<Vec<_>>();
2959 origins.sort();
2960 assert_eq!(origins, &["example1.com", "example2.com",]);
2961 }
2962
2963 #[test]
2964 fn test_preview_url() {
2965 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
2966
2967 let url1 = Url::parse("https://www.example.com/").unwrap();
2968 assert!(apply_observation(
2970 &conn,
2971 VisitObservation::new(url1.clone()).with_preview_image_url(Some(
2972 Url::parse("https://www.example.com/image.png").unwrap()
2973 ))
2974 )
2975 .unwrap()
2976 .is_none());
2977
2978 let mut db_preview_url = conn
2980 .query_row_and_then_cachable(
2981 "SELECT preview_image_url FROM moz_places WHERE id = 1",
2982 [],
2983 |row| row.get(0),
2984 false,
2985 )
2986 .unwrap();
2987 assert_eq!(
2988 Some("https://www.example.com/image.png".to_string()),
2989 db_preview_url
2990 );
2991
2992 let visit_id = apply_observation(
2994 &conn,
2995 VisitObservation::new(url1).with_visit_type(VisitType::Link),
2996 )
2997 .unwrap();
2998 assert!(visit_id.is_some());
2999
3000 db_preview_url = conn
3001 .query_row_and_then_cachable(
3002 "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3003 &[(":id", &visit_id.unwrap() as &dyn ToSql)],
3004 |row| row.get(0),
3005 false,
3006 )
3007 .unwrap();
3008 assert_eq!(
3009 Some("https://www.example.com/image.png".to_string()),
3010 db_preview_url
3011 );
3012
3013 let another_visit_id = apply_observation(
3015 &conn,
3016 VisitObservation::new(Url::parse("https://www.example.com/another/").unwrap())
3017 .with_preview_image_url(Some(
3018 Url::parse("https://www.example.com/funky/image.png").unwrap(),
3019 ))
3020 .with_visit_type(VisitType::Link),
3021 )
3022 .unwrap();
3023 assert!(another_visit_id.is_some());
3024
3025 db_preview_url = conn
3026 .query_row_and_then_cachable(
3027 "SELECT h.preview_image_url FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3028 &[(":id", &another_visit_id.unwrap())],
3029 |row| row.get(0),
3030 false,
3031 )
3032 .unwrap();
3033 assert_eq!(
3034 Some("https://www.example.com/funky/image.png".to_string()),
3035 db_preview_url
3036 );
3037 }
3038
3039 #[test]
3040 fn test_long_strings() {
3041 error_support::init_for_tests();
3042 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3043 let mut url = "http://www.example.com".to_string();
3044 while url.len() < crate::storage::URL_LENGTH_MAX {
3045 url += "/garbage";
3046 }
3047 let maybe_row = apply_observation(
3048 &conn,
3049 VisitObservation::new(Url::parse(&url).unwrap())
3050 .with_visit_type(VisitType::Link)
3051 .with_at(Timestamp::now()),
3052 )
3053 .unwrap();
3054 assert!(maybe_row.is_none(), "Shouldn't insert overlong URL");
3055
3056 let maybe_row_preview = apply_observation(
3057 &conn,
3058 VisitObservation::new(Url::parse("https://www.example.com/").unwrap())
3059 .with_visit_type(VisitType::Link)
3060 .with_preview_image_url(Url::parse(&url).unwrap()),
3061 )
3062 .unwrap();
3063 assert!(
3064 maybe_row_preview.is_some(),
3065 "Shouldn't avoid a visit observation due to an overly long preview url"
3066 );
3067
3068 let mut title = "example 1 2 3".to_string();
3069 while title.len() < crate::storage::TITLE_LENGTH_MAX + 10 {
3071 title += " test test";
3072 }
3073 let maybe_visit_row = apply_observation(
3074 &conn,
3075 VisitObservation::new(Url::parse("http://www.example.com/123").unwrap())
3076 .with_title(title.clone())
3077 .with_visit_type(VisitType::Link)
3078 .with_at(Timestamp::now()),
3079 )
3080 .unwrap();
3081
3082 assert!(maybe_visit_row.is_some());
3083 let db_title: String = conn
3084 .query_row_and_then_cachable(
3085 "SELECT h.title FROM moz_places AS h JOIN moz_historyvisits AS v ON h.id = v.place_id WHERE v.id = :id",
3086 &[(":id", &maybe_visit_row.unwrap())],
3087 |row| row.get(0),
3088 false,
3089 )
3090 .unwrap();
3091 assert_eq!(db_title.len(), crate::storage::TITLE_LENGTH_MAX);
3093 assert!(title.starts_with(&db_title));
3094 }
3095
3096 #[test]
3097 fn test_get_visit_page_with_bound() {
3098 use std::time::SystemTime;
3099 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3100 let now: Timestamp = SystemTime::now().into();
3101 let now_u64 = now.0;
3102 let now_i64 = now.0 as i64;
3103 let to_add = [
3105 (
3106 "https://www.example.com/0",
3107 "older 2",
3108 now_u64 - 200_200,
3109 false,
3110 (true, false),
3111 ),
3112 (
3113 "https://www.example.com/1",
3114 "older 1",
3115 now_u64 - 200_100,
3116 true,
3117 (true, false),
3118 ),
3119 (
3120 "https://www.example.com/2",
3121 "same time",
3122 now_u64 - 200_000,
3123 false,
3124 (true, false),
3125 ),
3126 (
3127 "https://www.example.com/3",
3128 "same time",
3129 now_u64 - 200_000,
3130 false,
3131 (true, false),
3132 ),
3133 (
3134 "https://www.example.com/4",
3135 "same time",
3136 now_u64 - 200_000,
3137 false,
3138 (true, false),
3139 ),
3140 (
3141 "https://www.example.com/5",
3142 "same time",
3143 now_u64 - 200_000,
3144 false,
3145 (true, false),
3146 ),
3147 (
3148 "https://www.example.com/6",
3149 "same time",
3150 now_u64 - 200_000,
3151 false,
3152 (true, false),
3153 ),
3154 (
3155 "https://www.example.com/7",
3156 "same time",
3157 now_u64 - 200_000,
3158 false,
3159 (true, false),
3160 ),
3161 (
3162 "https://www.example.com/8",
3163 "same time",
3164 now_u64 - 200_000,
3165 false,
3166 (true, false),
3167 ),
3168 (
3169 "https://www.example.com/9",
3170 "same time",
3171 now_u64 - 200_000,
3172 false,
3173 (true, false),
3174 ),
3175 (
3176 "https://www.example.com/10",
3177 "more recent 2",
3178 now_u64 - 199_000,
3179 false,
3180 (true, false),
3181 ),
3182 (
3183 "https://www.example.com/11",
3184 "more recent 1",
3185 now_u64 - 198_000,
3186 false,
3187 (true, false),
3188 ),
3189 ];
3190
3191 for &(url, title, when, remote, _) in &to_add {
3192 apply_observation(
3193 &conn,
3194 VisitObservation::new(Url::parse(url).unwrap())
3195 .with_title(title.to_owned())
3196 .with_at(Timestamp(when))
3197 .with_is_remote(remote)
3198 .with_visit_type(VisitType::Link),
3199 )
3200 .expect("Should apply visit");
3201 }
3202
3203 let infos_with_bound =
3205 get_visit_page_with_bound(&conn, now_i64 - 200_000, 8, 2, VisitTransitionSet::empty())
3206 .unwrap();
3207 let infos = infos_with_bound.infos;
3208 assert_eq!(infos[0].title.as_ref().unwrap().as_str(), "older 1",);
3209 assert!(infos[0].is_remote); assert_eq!(infos[1].title.as_ref().unwrap().as_str(), "older 2",);
3211 assert!(!infos[1].is_remote); assert_eq!(infos_with_bound.bound, now_i64 - 200_200,);
3213 assert_eq!(infos_with_bound.offset, 1,);
3214
3215 let infos_with_bound =
3217 get_visit_page_with_bound(&conn, now_i64 - 200_000, 7, 1, VisitTransitionSet::empty())
3218 .unwrap();
3219 assert_eq!(
3220 infos_with_bound.infos[0].url,
3221 Url::parse("https://www.example.com/9").unwrap(),
3222 );
3223
3224 let infos_with_bound =
3226 get_visit_page_with_bound(&conn, now_i64 - 200_000, 9, 1, VisitTransitionSet::empty())
3227 .unwrap();
3228 assert_eq!(
3229 infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3230 "older 2",
3231 );
3232
3233 let count = 2;
3235 let mut bound = now_i64 - 199_000;
3236 let mut offset = 1;
3237 for _i in 0..4 {
3238 let infos_with_bound =
3239 get_visit_page_with_bound(&conn, bound, offset, count, VisitTransitionSet::empty())
3240 .unwrap();
3241 assert_eq!(
3242 infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3243 "same time",
3244 );
3245 assert_eq!(
3246 infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3247 "same time",
3248 );
3249 bound = infos_with_bound.bound;
3250 offset = infos_with_bound.offset;
3251 }
3252 assert_eq!(bound, now_i64 - 200_000,);
3254 assert_eq!(offset, 8,);
3255
3256 let infos_with_bound =
3258 get_visit_page_with_bound(&conn, now_i64, 0, 2, VisitTransitionSet::empty()).unwrap();
3259 assert_eq!(
3260 infos_with_bound.infos[0].title.as_ref().unwrap().as_str(),
3261 "more recent 1",
3262 );
3263 assert_eq!(
3264 infos_with_bound.infos[1].title.as_ref().unwrap().as_str(),
3265 "more recent 2",
3266 );
3267 assert_eq!(infos_with_bound.bound, now_i64 - 199_000);
3268 assert_eq!(infos_with_bound.offset, 1);
3269 }
3270
3271 #[test]
3273 fn test_normal_visit_pruning() {
3274 use std::time::{Duration, SystemTime};
3275 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3276 let one_day = Duration::from_secs(60 * 60 * 24);
3277 let now: Timestamp = SystemTime::now().into();
3278 let url = Url::parse("https://mozilla.com/").unwrap();
3279
3280 let mut visits: Vec<_> = (0..30)
3282 .map(|i| {
3283 apply_observation(
3284 &conn,
3285 VisitObservation::new(url.clone())
3286 .with_at(now.checked_sub(one_day * i))
3287 .with_visit_type(VisitType::Link),
3288 )
3289 .unwrap()
3290 .unwrap()
3291 })
3292 .collect();
3293 visits.reverse();
3295
3296 check_visits_to_prune(
3297 &conn,
3298 find_normal_visits_to_prune(&conn, 4, now).unwrap(),
3299 &visits[..4],
3300 );
3301
3302 check_visits_to_prune(
3304 &conn,
3305 find_normal_visits_to_prune(&conn, 30, now).unwrap(),
3306 &visits[..22],
3307 );
3308 }
3309
3310 #[test]
3312 fn test_exotic_visit_pruning() {
3313 use std::time::{Duration, SystemTime};
3314 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3315 let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3316 let now: Timestamp = SystemTime::now().into();
3317 let short_url = Url::parse("https://mozilla.com/").unwrap();
3318 let long_url = Url::parse(&format!(
3319 "https://mozilla.com/{}",
3320 (0..255).map(|_| "x").collect::<String>()
3321 ))
3322 .unwrap();
3323
3324 let visit_with_long_url = apply_observation(
3325 &conn,
3326 VisitObservation::new(long_url.clone())
3327 .with_at(now.checked_sub(one_month * 2))
3328 .with_visit_type(VisitType::Link),
3329 )
3330 .unwrap()
3331 .unwrap();
3332
3333 let visit_for_download = apply_observation(
3334 &conn,
3335 VisitObservation::new(short_url)
3336 .with_at(now.checked_sub(one_month * 3))
3337 .with_visit_type(VisitType::Download),
3338 )
3339 .unwrap()
3340 .unwrap();
3341
3342 apply_observation(
3344 &conn,
3345 VisitObservation::new(long_url)
3346 .with_at(now.checked_sub(one_month))
3347 .with_visit_type(VisitType::Download),
3348 )
3349 .unwrap()
3350 .unwrap();
3351
3352 check_visits_to_prune(
3353 &conn,
3354 find_exotic_visits_to_prune(&conn, 2, now).unwrap(),
3355 &[visit_for_download, visit_with_long_url],
3356 );
3357
3358 check_visits_to_prune(
3360 &conn,
3361 find_exotic_visits_to_prune(&conn, 1, now).unwrap(),
3362 &[visit_for_download],
3363 );
3364
3365 check_visits_to_prune(
3367 &conn,
3368 find_exotic_visits_to_prune(&conn, 3, now).unwrap(),
3369 &[visit_for_download, visit_with_long_url],
3370 );
3371 }
3372 #[test]
3375 fn test_visit_pruning() {
3376 use std::time::{Duration, SystemTime};
3377 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).expect("no memory db");
3378 let one_month = Duration::from_secs(60 * 60 * 24 * 31);
3379 let now: Timestamp = SystemTime::now().into();
3380 let short_url = Url::parse("https://mozilla.com/").unwrap();
3381 let long_url = Url::parse(&format!(
3382 "https://mozilla.com/{}",
3383 (0..255).map(|_| "x").collect::<String>()
3384 ))
3385 .unwrap();
3386
3387 let excotic_visit = apply_observation(
3389 &conn,
3390 VisitObservation::new(long_url)
3391 .with_at(now.checked_sub(one_month * 3))
3392 .with_visit_type(VisitType::Link),
3393 )
3394 .unwrap()
3395 .unwrap();
3396
3397 let old_visit = apply_observation(
3399 &conn,
3400 VisitObservation::new(short_url.clone())
3401 .with_at(now.checked_sub(one_month * 4))
3402 .with_visit_type(VisitType::Link),
3403 )
3404 .unwrap()
3405 .unwrap();
3406 let really_old_visit = apply_observation(
3407 &conn,
3408 VisitObservation::new(short_url.clone())
3409 .with_at(now.checked_sub(one_month * 12))
3410 .with_visit_type(VisitType::Link),
3411 )
3412 .unwrap()
3413 .unwrap();
3414
3415 apply_observation(
3417 &conn,
3418 VisitObservation::new(short_url)
3419 .with_at(now.checked_sub(Duration::from_secs(100)))
3420 .with_visit_type(VisitType::Link),
3421 )
3422 .unwrap()
3423 .unwrap();
3424
3425 check_visits_to_prune(
3426 &conn,
3427 find_visits_to_prune(&conn, 2, now).unwrap(),
3428 &[excotic_visit, really_old_visit],
3429 );
3430
3431 check_visits_to_prune(
3432 &conn,
3433 find_visits_to_prune(&conn, 10, now).unwrap(),
3434 &[excotic_visit, really_old_visit, old_visit],
3435 );
3436 }
3437
3438 fn check_visits_to_prune(
3439 db: &PlacesDb,
3440 visits_to_delete: Vec<VisitToDelete>,
3441 correct_visits: &[RowId],
3442 ) {
3443 assert_eq!(
3444 correct_visits.iter().collect::<HashSet<_>>(),
3445 visits_to_delete
3446 .iter()
3447 .map(|v| &v.visit_id)
3448 .collect::<HashSet<_>>()
3449 );
3450
3451 let correct_place_ids: HashSet<RowId> = correct_visits
3452 .iter()
3453 .map(|vid| {
3454 db.query_one(&format!(
3455 "SELECT v.place_id FROM moz_historyvisits v WHERE v.id = {}",
3456 vid
3457 ))
3458 .unwrap()
3459 })
3460 .collect();
3461 assert_eq!(
3462 correct_place_ids,
3463 visits_to_delete
3464 .iter()
3465 .map(|v| v.page_id)
3466 .collect::<HashSet<_>>()
3467 );
3468 }
3469
3470 #[test]
3471 fn test_get_visit_count_for_host() {
3472 error_support::init_for_tests();
3473 let conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap();
3474 let start_timestamp = Timestamp::now();
3475 let to_add = [
3476 (
3477 "http://example.com/0",
3478 start_timestamp.0 - 200_200,
3479 VisitType::Link,
3480 ),
3481 (
3482 "http://example.com/1",
3483 start_timestamp.0 - 200_100,
3484 VisitType::Link,
3485 ),
3486 (
3487 "https://example.com/0",
3488 start_timestamp.0 - 200_000,
3489 VisitType::Link,
3490 ),
3491 (
3492 "https://example1.com/0",
3493 start_timestamp.0 - 100_600,
3494 VisitType::Link,
3495 ),
3496 (
3497 "https://example1.com/0",
3498 start_timestamp.0 - 100_500,
3499 VisitType::Reload,
3500 ),
3501 (
3502 "https://example1.com/1",
3503 start_timestamp.0 - 100_400,
3504 VisitType::Link,
3505 ),
3506 (
3507 "https://example.com/2",
3508 start_timestamp.0 - 100_300,
3509 VisitType::Link,
3510 ),
3511 (
3512 "https://example.com/1",
3513 start_timestamp.0 - 100_200,
3514 VisitType::Link,
3515 ),
3516 (
3517 "https://example.com/0",
3518 start_timestamp.0 - 100_100,
3519 VisitType::Link,
3520 ),
3521 ];
3522
3523 for &(url, when, visit_type) in &to_add {
3524 apply_observation(
3525 &conn,
3526 VisitObservation::new(Url::parse(url).unwrap())
3527 .with_at(Timestamp(when))
3528 .with_visit_type(visit_type),
3529 )
3530 .unwrap()
3531 .unwrap();
3532 }
3533
3534 assert_eq!(
3535 get_visit_count_for_host(
3536 &conn,
3537 "example.com",
3538 Timestamp(start_timestamp.0 - 100_000),
3539 VisitTransitionSet::for_specific(&[])
3540 )
3541 .unwrap(),
3542 6
3543 );
3544 assert_eq!(
3545 get_visit_count_for_host(
3546 &conn,
3547 "example1.com",
3548 Timestamp(start_timestamp.0 - 100_000),
3549 VisitTransitionSet::for_specific(&[])
3550 )
3551 .unwrap(),
3552 3
3553 );
3554 assert_eq!(
3555 get_visit_count_for_host(
3556 &conn,
3557 "example.com",
3558 Timestamp(start_timestamp.0 - 200_000),
3559 VisitTransitionSet::for_specific(&[])
3560 )
3561 .unwrap(),
3562 2
3563 );
3564 assert_eq!(
3565 get_visit_count_for_host(
3566 &conn,
3567 "example1.com",
3568 Timestamp(start_timestamp.0 - 100_500),
3569 VisitTransitionSet::for_specific(&[])
3570 )
3571 .unwrap(),
3572 1
3573 );
3574 assert_eq!(
3575 get_visit_count_for_host(
3576 &conn,
3577 "example1.com",
3578 Timestamp(start_timestamp.0 - 100_000),
3579 VisitTransitionSet::for_specific(&[VisitType::Reload])
3580 )
3581 .unwrap(),
3582 2
3583 );
3584 }
3585}