places/history_sync/
plan.rs

Help
1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5use super::record::{HistoryRecord, HistoryRecordVisit};
6use super::{MAX_OUTGOING_PLACES, MAX_VISITS};
7use crate::api::history::can_add_url;
8use crate::db::PlacesDb;
9use crate::error::*;
10use crate::storage::{
11    delete_pending_temp_tables,
12    history::history_sync::{
13        apply_synced_deletion, apply_synced_reconciliation, apply_synced_visits, fetch_outgoing,
14        fetch_visits, finish_outgoing, FetchedVisit, FetchedVisitPage,
15    },
16};
17use crate::types::{UnknownFields, VisitType};
18use interrupt_support::Interruptee;
19use std::collections::HashSet;
20use std::time::{SystemTime, UNIX_EPOCH};
21use sync15::bso::{IncomingBso, IncomingKind, OutgoingBso};
22use sync15::telemetry;
23use sync_guid::Guid as SyncGuid;
24use types::Timestamp;
25use url::Url;
26
27/// Clamps a history visit date between the current date and the earliest
28/// sensible date.
29fn clamp_visit_date(visit_date: Timestamp) -> std::result::Result<Timestamp, ()> {
30    let now = Timestamp::now();
31    if visit_date > now {
32        return Ok(now);
33    }
34    if visit_date < Timestamp::EARLIEST {
35        return Err(());
36    }
37    Ok(visit_date)
38}
39
40/// This is the action we will take *locally* for each incoming record.
41/// For example, IncomingPlan::Delete means we will be deleting a local record
42/// and not that we will be uploading a tombstone or deleting the record itself.
43#[allow(clippy::large_enum_variant)]
44#[derive(Debug)]
45pub enum IncomingPlan {
46    /// An entry we just want to ignore - either due to the URL etc, or because no changes.
47    Skip,
48    /// Something's wrong with this entry.
49    Invalid(Error),
50    /// The entry appears sane, but there was some error.
51    Failed(Error),
52    /// We should locally delete this.
53    Delete,
54    /// We should apply this.
55    Apply {
56        url: Url,
57        new_title: Option<String>,
58        visits: Vec<HistoryRecordVisit>,
59        unknown_fields: UnknownFields,
60    },
61    /// Entry exists locally and it's the same as the incoming record. This is
62    /// subtly different from Skip as we may still need to write metadata to
63    /// the local DB for reconciled items.
64    Reconciled,
65}
66
67fn plan_incoming_record(conn: &PlacesDb, record: HistoryRecord, max_visits: usize) -> IncomingPlan {
68    let url = match Url::parse(&record.hist_uri) {
69        Ok(u) => u,
70        Err(e) => return IncomingPlan::Invalid(e.into()),
71    };
72
73    if !record.id.is_valid_for_places() {
74        return IncomingPlan::Invalid(InvalidPlaceInfo::InvalidGuid.into());
75    }
76
77    match can_add_url(&url) {
78        Ok(can) => {
79            if !can {
80                return IncomingPlan::Skip;
81            }
82        }
83        Err(e) => return IncomingPlan::Failed(e),
84    }
85    // Let's get what we know about it, if anything - last 20, like desktop?
86    let visit_tuple = match fetch_visits(conn, &url, max_visits) {
87        Ok(v) => v,
88        Err(e) => return IncomingPlan::Failed(e),
89    };
90
91    // This all seems more messy than it should be - struggling to find the
92    // correct signature for fetch_visits.
93    // An improvement might be to do this via a temp table so we can dedupe
94    // and apply in one operation rather than the fetch, rust-merge and update
95    // we are doing here.
96    let (existing_page, existing_visits): (Option<FetchedVisitPage>, Vec<FetchedVisit>) =
97        match visit_tuple {
98            None => (None, Vec::new()),
99            Some((p, v)) => (Some(p), v),
100        };
101
102    let guid_changed = match existing_page {
103        Some(p) => p.guid != record.id,
104        None => false,
105    };
106
107    let mut cur_visit_map: HashSet<(VisitType, Timestamp)> =
108        HashSet::with_capacity(existing_visits.len());
109    for visit in &existing_visits {
110        // it should be impossible for us to have invalid visits locally, but...
111        let transition = match visit.visit_type {
112            Some(t) => t,
113            None => continue,
114        };
115        match clamp_visit_date(visit.visit_date) {
116            Ok(date_use) => {
117                cur_visit_map.insert((transition, date_use));
118            }
119            Err(_) => {
120                warn!("Ignored visit before 1993-01-23");
121            }
122        }
123    }
124    // If we already have MAX_RECORDS visits, then we will ignore incoming
125    // visits older than that, to avoid adding dupes of earlier visits.
126    // (Not really clear why 20 is magic, but what's good enough for desktop
127    // is good enough for us at this stage.)
128    // We should also consider pushing this deduping down into storage, where
129    // it can possibly do a better job directly in SQL or similar.
130    let earliest_allowed: SystemTime = if existing_visits.len() == max_visits {
131        existing_visits[existing_visits.len() - 1].visit_date.into()
132    } else {
133        UNIX_EPOCH
134    };
135
136    // work out which of the incoming visits we should apply.
137    let mut to_apply = Vec::with_capacity(record.visits.len());
138    for incoming_visit in record.visits {
139        let transition = match VisitType::from_primitive(incoming_visit.transition) {
140            Some(v) => v,
141            None => continue,
142        };
143        match clamp_visit_date(incoming_visit.date.into()) {
144            Ok(timestamp) => {
145                if earliest_allowed > timestamp.into() {
146                    continue;
147                }
148                // If the entry isn't in our map we should add it.
149                let key = (transition, timestamp);
150                if !cur_visit_map.contains(&key) {
151                    to_apply.push(HistoryRecordVisit {
152                        date: timestamp.into(),
153                        transition: transition as u8,
154                        unknown_fields: incoming_visit.unknown_fields,
155                    });
156                    cur_visit_map.insert(key);
157                }
158            }
159            Err(()) => {
160                warn!("Ignored visit before 1993-01-23");
161            }
162        }
163    }
164    // Now we need to check the other attributes.
165    // Check if we should update title? For now, assume yes. It appears
166    // as though desktop always updates it.
167    if guid_changed || !to_apply.is_empty() {
168        let new_title = Some(record.title);
169        IncomingPlan::Apply {
170            url,
171            new_title,
172            visits: to_apply,
173            unknown_fields: record.unknown_fields,
174        }
175    } else {
176        IncomingPlan::Reconciled
177    }
178}
179
180pub fn apply_plan(
181    db: &PlacesDb,
182    inbound: Vec<IncomingBso>,
183    telem: &mut telemetry::EngineIncoming,
184    interruptee: &impl Interruptee,
185) -> Result<()> {
186    // for a first-cut, let's do this in the most naive way possible...
187    let mut plans: Vec<(SyncGuid, IncomingPlan)> = Vec::with_capacity(inbound.len());
188    for incoming in inbound {
189        interruptee.err_if_interrupted()?;
190        let content = incoming.into_content::<HistoryRecord>();
191        let plan = match content.kind {
192            IncomingKind::Tombstone => IncomingPlan::Delete,
193            IncomingKind::Content(record) => plan_incoming_record(db, record, MAX_VISITS),
194            IncomingKind::Malformed => {
195                // We could push IncomingPlan::Invalid here, but the code before the IncomingKind
196                // refactor didn't know what `id` to use, so skipped it - so we do too.
197                warn!(
198                    "Error deserializing incoming record: {}",
199                    content.envelope.id
200                );
201                telem.failed(1);
202                continue;
203            }
204        };
205        plans.push((content.envelope.id.clone(), plan));
206    }
207
208    let mut tx = db.begin_transaction()?;
209
210    for (guid, plan) in plans {
211        interruptee.err_if_interrupted()?;
212        match &plan {
213            IncomingPlan::Skip => {
214                trace!("incoming: skipping item {:?}", guid);
215                // XXX - should we `telem.reconciled(1);` here?
216            }
217            IncomingPlan::Invalid(err) => {
218                warn!(
219                    "incoming: record {:?} skipped because it is invalid: {}",
220                    guid, err
221                );
222                telem.failed(1);
223            }
224            IncomingPlan::Failed(err) => {
225                error_support::report_error!(
226                    "places-failed-to-apply",
227                    "incoming: record {:?} failed to apply: {}",
228                    guid,
229                    err
230                );
231                telem.failed(1);
232            }
233            IncomingPlan::Delete => {
234                trace!("incoming: deleting {:?}", guid);
235                apply_synced_deletion(db, &guid)?;
236                telem.applied(1);
237            }
238            IncomingPlan::Apply {
239                url,
240                new_title,
241                visits,
242                unknown_fields,
243            } => {
244                trace!(
245                    "incoming: will apply {guid:?}: url={url:?}, title={new_title:?}, to_add={visits:?}, unknown_fields={unknown_fields:?}"
246                );
247                apply_synced_visits(db, &guid, url, new_title, visits, unknown_fields)?;
248                telem.applied(1);
249            }
250            IncomingPlan::Reconciled => {
251                telem.reconciled(1);
252                trace!("incoming: reconciled {:?}", guid);
253                apply_synced_reconciliation(db, &guid)?;
254            }
255        };
256        if tx.should_commit() {
257            // Trigger frecency and origin updates before committing the
258            // transaction, so that our origins table is consistent even
259            // if we're interrupted.
260            delete_pending_temp_tables(db)?;
261        }
262        tx.maybe_commit()?;
263    }
264    // ...And commit the final chunk of plans, making sure we trigger
265    // frecency and origin updates.
266    delete_pending_temp_tables(db)?;
267    tx.commit()?;
268    info!("incoming: {}", serde_json::to_string(&telem).unwrap());
269    Ok(())
270}
271
272pub fn get_planned_outgoing(db: &PlacesDb) -> Result<Vec<OutgoingBso>> {
273    // It might make sense for fetch_outgoing to manage its own
274    // begin_transaction - even though doesn't seem a large bottleneck
275    // at this time, the fact we hold a single transaction for the entire call
276    // really is used only for performance, so it's certainly a candidate.
277    let tx = db.begin_transaction()?;
278    let outgoing = fetch_outgoing(db, MAX_OUTGOING_PLACES, MAX_VISITS)?;
279    tx.commit()?;
280    Ok(outgoing)
281}
282
283pub fn finish_plan(db: &PlacesDb) -> Result<()> {
284    let tx = db.begin_transaction()?;
285    finish_outgoing(db)?;
286    trace!("Committing final sync plan");
287    tx.commit()?;
288    Ok(())
289}
290
291#[cfg(test)]
292mod tests {
293    use super::*;
294    use crate::api::matcher::{search_frecent, SearchParams};
295    use crate::api::places_api::ConnectionType;
296    use crate::db::PlacesDb;
297    use crate::history_sync::ServerVisitTimestamp;
298    use crate::observation::VisitObservation;
299    use crate::storage::history::history_sync::fetch_visits;
300    use crate::storage::history::{apply_observation, delete_visits_for, url_to_guid};
301    use crate::types::SyncStatus;
302    use interrupt_support::NeverInterrupts;
303    use serde_json::json;
304    use sql_support::ConnExt;
305    use std::time::Duration;
306    use sync15::bso::IncomingBso;
307    use types::Timestamp;
308    use url::Url;
309
310    fn get_existing_guid(conn: &PlacesDb, url: &Url) -> SyncGuid {
311        url_to_guid(conn, url)
312            .expect("should have worked")
313            .expect("should have got a value")
314    }
315
316    fn get_tombstone_count(conn: &PlacesDb) -> u32 {
317        let result: Result<Option<u32>> = conn.try_query_row(
318            "SELECT COUNT(*) from moz_places_tombstones;",
319            [],
320            |row| Ok(row.get::<_, u32>(0)?),
321            true,
322        );
323        result
324            .expect("should have worked")
325            .expect("should have got a value")
326    }
327
328    fn get_sync(conn: &PlacesDb, url: &Url) -> (SyncStatus, u32) {
329        let guid_result: Result<Option<(SyncStatus, u32)>> = conn.try_query_row(
330            "SELECT sync_status, sync_change_counter
331                     FROM moz_places
332                     WHERE url = :url;",
333            &[(":url", &String::from(url.clone()))],
334            |row| {
335                Ok((
336                    SyncStatus::from_u8(row.get::<_, u8>(0)?),
337                    row.get::<_, u32>(1)?,
338                ))
339            },
340            true,
341        );
342        guid_result
343            .expect("should have worked")
344            .expect("should have got values")
345    }
346
347    fn apply_and_get_outgoing(db: &PlacesDb, incoming: Vec<IncomingBso>) -> Vec<OutgoingBso> {
348        apply_plan(
349            db,
350            incoming,
351            &mut telemetry::EngineIncoming::new(),
352            &NeverInterrupts,
353        )
354        .expect("should apply");
355        get_planned_outgoing(db).expect("should get outgoing")
356    }
357
358    #[test]
359    fn test_invalid_guid() -> Result<()> {
360        error_support::init_for_tests();
361        let conn = PlacesDb::open_in_memory(ConnectionType::Sync)?;
362        let record = HistoryRecord {
363            id: "foo".into(),
364            title: "title".into(),
365            hist_uri: "http://example.com".into(),
366            visits: vec![],
367            unknown_fields: UnknownFields::new(),
368        };
369
370        assert!(matches!(
371            plan_incoming_record(&conn, record, 10),
372            IncomingPlan::Invalid(_)
373        ));
374        Ok(())
375    }
376
377    #[test]
378    fn test_invalid_url() -> Result<()> {
379        error_support::init_for_tests();
380        let conn = PlacesDb::open_in_memory(ConnectionType::Sync)?;
381        let record = HistoryRecord {
382            id: "aaaaaaaaaaaa".into(),
383            title: "title".into(),
384            hist_uri: "invalid".into(),
385            visits: vec![],
386            unknown_fields: UnknownFields::new(),
387        };
388
389        assert!(matches!(
390            plan_incoming_record(&conn, record, 10),
391            IncomingPlan::Invalid(_)
392        ));
393        Ok(())
394    }
395
396    #[test]
397    fn test_new() -> Result<()> {
398        error_support::init_for_tests();
399        let conn = PlacesDb::open_in_memory(ConnectionType::Sync)?;
400        let visits = vec![HistoryRecordVisit {
401            date: SystemTime::now().into(),
402            transition: 1,
403            unknown_fields: UnknownFields::new(),
404        }];
405        let record = HistoryRecord {
406            id: "aaaaaaaaaaaa".into(),
407            title: "title".into(),
408            hist_uri: "https://example.com".into(),
409            visits,
410            unknown_fields: UnknownFields::new(),
411        };
412
413        assert!(matches!(
414            plan_incoming_record(&conn, record, 10),
415            IncomingPlan::Apply { .. }
416        ));
417        Ok(())
418    }
419
420    #[test]
421    fn test_plan_dupe_visit_same_guid() {
422        error_support::init_for_tests();
423        let conn = PlacesDb::open_in_memory(ConnectionType::Sync).expect("no memory db");
424        let now = SystemTime::now();
425        let url = Url::parse("https://example.com").expect("is valid");
426        // add it locally
427        let obs = VisitObservation::new(url.clone())
428            .with_visit_type(VisitType::Link)
429            .with_at(Some(now.into()));
430        apply_observation(&conn, obs).expect("should apply");
431        // should be New with a change counter.
432        assert_eq!(get_sync(&conn, &url), (SyncStatus::New, 1));
433
434        let guid = get_existing_guid(&conn, &url);
435
436        // try and add it remotely.
437        let visits = vec![HistoryRecordVisit {
438            date: now.into(),
439            transition: 1,
440            unknown_fields: UnknownFields::new(),
441        }];
442        let record = HistoryRecord {
443            id: guid,
444            title: "title".into(),
445            hist_uri: "https://example.com".into(),
446            visits,
447            unknown_fields: UnknownFields::new(),
448        };
449        // We should have reconciled it.
450        assert!(matches!(
451            plan_incoming_record(&conn, record, 10),
452            IncomingPlan::Reconciled
453        ));
454    }
455
456    #[test]
457    fn test_plan_dupe_visit_different_guid_no_visits() {
458        error_support::init_for_tests();
459        let conn = PlacesDb::open_in_memory(ConnectionType::Sync).expect("no memory db");
460        let now = SystemTime::now();
461        let url = Url::parse("https://example.com").expect("is valid");
462        // add it locally
463        let obs = VisitObservation::new(url.clone())
464            .with_visit_type(VisitType::Link)
465            .with_at(Some(now.into()));
466        apply_observation(&conn, obs).expect("should apply");
467
468        assert_eq!(get_sync(&conn, &url), (SyncStatus::New, 1));
469
470        // try and add an incoming record with the same URL but different guid.
471        let record = HistoryRecord {
472            id: SyncGuid::random(),
473            title: "title".into(),
474            hist_uri: "https://example.com".into(),
475            visits: vec![],
476            unknown_fields: UnknownFields::new(),
477        };
478        // Even though there are no visits we should record that it will be
479        // applied with the guid change.
480        assert!(matches!(
481            plan_incoming_record(&conn, record, 10),
482            IncomingPlan::Apply { .. }
483        ));
484    }
485
486    // These "dupe" tests all do the full application of the plan and checks
487    // the end state of the db.
488    #[test]
489    fn test_apply_dupe_no_local_visits() -> Result<()> {
490        // There's a chance the server ends up with different records but
491        // which reference the same URL.
492        // This is testing the case when there are no local visits to that URL.
493        error_support::init_for_tests();
494        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
495        let guid1 = SyncGuid::random();
496        let ts1: Timestamp = (SystemTime::now() - Duration::new(5, 0)).into();
497
498        let guid2 = SyncGuid::random();
499        let ts2: Timestamp = SystemTime::now().into();
500        let url = Url::parse("https://example.com")?;
501
502        // 2 incoming records with the same URL.
503        let incoming = vec![
504            IncomingBso::from_test_content(json!({
505                "id": guid1,
506                "title": "title",
507                "histUri": url.as_str(),
508                "visits": [ {"date": ServerVisitTimestamp::from(ts1), "type": 1}]
509            })),
510            IncomingBso::from_test_content(json!({
511                "id": guid2,
512                "title": "title",
513                "histUri": url.as_str(),
514                "visits": [ {"date": ServerVisitTimestamp::from(ts2), "type": 1}]
515            })),
516            IncomingBso::from_test_content(json!({
517                "id": guid2,
518                "title": "title2",
519                "histUri": url.as_str(),
520                "visits": [ {"date": ServerVisitTimestamp::from(ts2), "type": 1}]
521            })),
522        ];
523
524        let outgoing = apply_and_get_outgoing(&db, incoming);
525        assert_eq!(
526            outgoing.len(),
527            1,
528            "should have guid1 as outgoing with both visits."
529        );
530        assert_eq!(outgoing[0].envelope.id, guid1);
531
532        // should have 1 URL with both visits locally.
533        let (page, visits) = fetch_visits(&db, &url, 3)?.expect("page exists");
534        assert_eq!(
535            page.guid, guid1,
536            "page should have the guid from the first record"
537        );
538        assert_eq!(
539            page.title, "title2",
540            "page should have the title from the second record"
541        );
542        assert_eq!(visits.len(), 2, "page should have 2 visits");
543
544        Ok(())
545    }
546
547    #[test]
548    fn test_apply_dupe_local_unsynced_visits() -> Result<()> {
549        // There's a chance the server ends up with different records but
550        // which reference the same URL.
551        // This is testing the case when there are a local visits to that URL,
552        // but they are yet to be synced - the local guid should change and
553        // all visits should be applied.
554        error_support::init_for_tests();
555        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
556
557        let guid1 = SyncGuid::random();
558        let ts1: Timestamp = (SystemTime::now() - Duration::new(5, 0)).into();
559
560        let guid2 = SyncGuid::random();
561        let ts2: Timestamp = SystemTime::now().into();
562        let url = Url::parse("https://example.com")?;
563
564        let ts_local: Timestamp = (SystemTime::now() - Duration::new(10, 0)).into();
565        let obs = VisitObservation::new(url.clone())
566            .with_visit_type(VisitType::Link)
567            .with_at(Some(ts_local));
568        apply_observation(&db, obs)?;
569
570        // 2 incoming records with the same URL.
571        let incoming = vec![
572            IncomingBso::from_test_content(json!({
573                "id": guid1,
574                "title": "title",
575                "histUri": url.as_str(),
576                "visits": [ {"date": ServerVisitTimestamp::from(ts1), "type": 1}]
577            })),
578            IncomingBso::from_test_content(json!({
579                "id": guid2,
580                "title": "title",
581                "histUri": url.as_str(),
582                "visits": [ {"date": ServerVisitTimestamp::from(ts2), "type": 1}]
583            })),
584        ];
585
586        let outgoing = apply_and_get_outgoing(&db, incoming);
587        assert_eq!(outgoing.len(), 1, "should have guid1 as outgoing");
588        assert_eq!(outgoing[0].envelope.id, guid1);
589
590        // should have 1 URL with all visits locally, but with the first incoming guid.
591        let (page, visits) = fetch_visits(&db, &url, 3)?.expect("page exists");
592        assert_eq!(page.guid, guid1, "should have the expected guid");
593        assert_eq!(visits.len(), 3, "should have all visits");
594
595        Ok(())
596    }
597
598    #[test]
599    fn test_apply_dupe_local_synced_visits() -> Result<()> {
600        // There's a chance the server ends up with different records but
601        // which reference the same URL.
602        // This is testing the case when there are a local visits to that URL,
603        // and they have been synced - the existing guid should not change,
604        // although all visits should still be applied.
605        error_support::init_for_tests();
606        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
607
608        let guid1 = SyncGuid::random();
609        let ts1: Timestamp = (SystemTime::now() - Duration::new(5, 0)).into();
610
611        let guid2 = SyncGuid::random();
612        let ts2: Timestamp = SystemTime::now().into();
613        let url = Url::parse("https://example.com")?;
614
615        let ts_local: Timestamp = (SystemTime::now() - Duration::new(10, 0)).into();
616        let obs = VisitObservation::new(url.clone())
617            .with_visit_type(VisitType::Link)
618            .with_at(Some(ts_local));
619        apply_observation(&db, obs)?;
620
621        // 2 incoming records with the same URL.
622        let incoming = vec![
623            IncomingBso::from_test_content(json!({
624                "id": guid1,
625                "title": "title",
626                "histUri": url.as_str(),
627                "visits": [ {"date": ServerVisitTimestamp::from(ts1), "type": 1}]
628            })),
629            IncomingBso::from_test_content(json!({
630                "id": guid2,
631                "title": "title",
632                "histUri": url.as_str(),
633                "sortindex": 0,
634                "ttl": 100,
635                "visits": [ {"date": ServerVisitTimestamp::from(ts2), "type": 1}]
636            })),
637        ];
638
639        let outgoing = apply_and_get_outgoing(&db, incoming);
640        assert_eq!(
641            outgoing.len(),
642            1,
643            "should have guid1 as outgoing with both visits."
644        );
645
646        // should have 1 URL with all visits locally, but with the first incoming guid.
647        let (page, visits) = fetch_visits(&db, &url, 3)?.expect("page exists");
648        assert_eq!(page.guid, guid1, "should have the expected guid");
649        assert_eq!(visits.len(), 3, "should have all visits");
650
651        Ok(())
652    }
653
654    #[test]
655    fn test_apply_plan_incoming_invalid_timestamp() -> Result<()> {
656        error_support::init_for_tests();
657        let json = json!({
658            "id": "aaaaaaaaaaaa",
659            "title": "title",
660            "histUri": "http://example.com",
661            "visits": [ {"date": 15_423_493_234_840_000_000u64, "type": 1}]
662        });
663        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
664        let outgoing = apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
665        assert_eq!(outgoing.len(), 0, "nothing outgoing");
666
667        let now: Timestamp = SystemTime::now().into();
668        let (_page, visits) =
669            fetch_visits(&db, &Url::parse("http://example.com").unwrap(), 2)?.expect("page exists");
670        assert_eq!(visits.len(), 1);
671        assert!(
672            visits[0].visit_date <= now,
673            "should have clamped the timestamp"
674        );
675        Ok(())
676    }
677
678    #[test]
679    fn test_apply_plan_incoming_invalid_negative_timestamp() -> Result<()> {
680        error_support::init_for_tests();
681        let json = json!({
682            "id": "aaaaaaaaaaaa",
683            "title": "title",
684            "histUri": "http://example.com",
685            "visits": [ {"date": -123, "type": 1}]
686        });
687        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
688        let outgoing = apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
689        assert_eq!(outgoing.len(), 0, "should skip the invalid entry");
690        Ok(())
691    }
692
693    #[test]
694    fn test_apply_plan_incoming_invalid_visit_type() -> Result<()> {
695        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
696        let visits = vec![HistoryRecordVisit {
697            date: SystemTime::now().into(),
698            transition: 99,
699            unknown_fields: UnknownFields::new(),
700        }];
701        let record = HistoryRecord {
702            id: "aaaaaaaaaaaa".into(),
703            title: "title".into(),
704            hist_uri: "http://example.com".into(),
705            visits,
706            unknown_fields: UnknownFields::new(),
707        };
708        let plan = plan_incoming_record(&db, record, 10);
709        // We expect "Reconciled" because after skipping the invalid visit
710        // we found nothing to apply.
711        assert!(matches!(plan, IncomingPlan::Reconciled));
712        Ok(())
713    }
714
715    #[test]
716    fn test_apply_plan_incoming_new() -> Result<()> {
717        error_support::init_for_tests();
718        let now: Timestamp = SystemTime::now().into();
719        let json = json!({
720            "id": "aaaaaaaaaaaa",
721            "title": "title",
722            "histUri": "http://example.com",
723            "visits": [ {"date": ServerVisitTimestamp::from(now), "type": 1}]
724        });
725        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
726        let outgoing = apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
727
728        // should have applied it locally.
729        let (page, visits) =
730            fetch_visits(&db, &Url::parse("http://example.com").unwrap(), 2)?.expect("page exists");
731        assert_eq!(page.title, "title");
732        assert_eq!(visits.len(), 1);
733        let visit = visits.into_iter().next().unwrap();
734        assert_eq!(visit.visit_date, now);
735
736        // page should have frecency (going through a public api to get this is a pain)
737        // XXX - FIXME - searching for "title" here fails to find a result?
738        // But above, we've checked title is in the record.
739        let found = search_frecent(
740            &db,
741            SearchParams {
742                search_string: "http://example.com".into(),
743                limit: 2,
744            },
745        )?;
746        assert_eq!(found.len(), 1);
747        let result = found.into_iter().next().unwrap();
748        assert!(result.frecency > 0, "should have frecency");
749
750        // and nothing outgoing.
751        assert_eq!(outgoing.len(), 0);
752        Ok(())
753    }
754
755    #[test]
756    fn test_apply_plan_outgoing_new() -> Result<()> {
757        error_support::init_for_tests();
758        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
759        let url = Url::parse("https://example.com")?;
760        let now = SystemTime::now();
761        let obs = VisitObservation::new(url)
762            .with_visit_type(VisitType::Link)
763            .with_at(Some(now.into()));
764        apply_observation(&db, obs)?;
765
766        let outgoing = apply_and_get_outgoing(&db, vec![]);
767
768        assert_eq!(outgoing.len(), 1);
769        Ok(())
770    }
771
772    #[test]
773    fn test_simple_visit_reconciliation() -> Result<()> {
774        error_support::init_for_tests();
775        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
776        let ts: Timestamp = (SystemTime::now() - Duration::new(5, 0)).into();
777        let url = Url::parse("https://example.com")?;
778
779        // First add a local visit with the timestamp.
780        let obs = VisitObservation::new(url.clone())
781            .with_visit_type(VisitType::Link)
782            .with_at(Some(ts));
783        apply_observation(&db, obs)?;
784        // Sync status should be "new" and have a change recorded.
785        assert_eq!(get_sync(&db, &url), (SyncStatus::New, 1));
786
787        let guid = get_existing_guid(&db, &url);
788
789        // and an incoming record with the same timestamp
790        let json = json!({
791            "id": guid,
792            "title": "title",
793            "histUri": url.as_str(),
794            "visits": [ {"date": ServerVisitTimestamp::from(ts), "type": 1}]
795        });
796
797        apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
798
799        // should still have only 1 visit and it should still be local.
800        let (_page, visits) = fetch_visits(&db, &url, 2)?.expect("page exists");
801        assert_eq!(visits.len(), 1);
802        assert!(visits[0].is_local);
803        // The item should have changed to Normal and have no change counter.
804        assert_eq!(get_sync(&db, &url), (SyncStatus::Normal, 0));
805        Ok(())
806    }
807
808    #[test]
809    fn test_simple_visit_incoming_and_outgoing() -> Result<()> {
810        error_support::init_for_tests();
811        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
812        let ts1: Timestamp = (SystemTime::now() - Duration::new(5, 0)).into();
813        let ts2: Timestamp = SystemTime::now().into();
814        let url = Url::parse("https://example.com")?;
815
816        // First add a local visit with ts1.
817        let obs = VisitObservation::new(url.clone())
818            .with_visit_type(VisitType::Link)
819            .with_at(Some(ts1));
820        apply_observation(&db, obs)?;
821
822        let guid = get_existing_guid(&db, &url);
823
824        // and an incoming record with ts2
825        let json = json!({
826            "id": guid,
827            "title": "title",
828            "histUri": url.as_str(),
829            "visits": [ {"date": ServerVisitTimestamp::from(ts2), "type": 1}]
830        });
831
832        let outgoing = apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
833
834        // should now have both visits locally.
835        let (_page, visits) = fetch_visits(&db, &url, 3)?.expect("page exists");
836        assert_eq!(visits.len(), 2);
837
838        // and the record should still be in outgoing due to our local change.
839        assert_eq!(outgoing.len(), 1);
840        let record = outgoing[0].to_test_incoming_t::<HistoryRecord>();
841        assert_eq!(record.id, guid);
842        assert_eq!(record.visits.len(), 2, "should have both visits outgoing");
843        assert_eq!(
844            record.visits[0].date,
845            ts2.into(),
846            "most recent timestamp should be first"
847        );
848        assert_eq!(
849            record.visits[1].date,
850            ts1.into(),
851            "both timestamps should appear"
852        );
853        Ok(())
854    }
855
856    #[test]
857    fn test_incoming_tombstone_local_new() -> Result<()> {
858        error_support::init_for_tests();
859        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
860        let url = Url::parse("https://example.com")?;
861        let obs = VisitObservation::new(url.clone())
862            .with_visit_type(VisitType::Link)
863            .with_at(Some(SystemTime::now().into()));
864        apply_observation(&db, obs)?;
865        assert_eq!(get_sync(&db, &url), (SyncStatus::New, 1));
866
867        let guid = get_existing_guid(&db, &url);
868
869        // and an incoming tombstone for that guid
870        let json = json!({
871            "id": guid,
872            "deleted": true,
873        });
874        let outgoing = apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
875        assert_eq!(outgoing.len(), 0, "should be nothing outgoing");
876        assert_eq!(get_tombstone_count(&db), 0, "should be no tombstones");
877        Ok(())
878    }
879
880    #[test]
881    fn test_incoming_tombstone_local_normal() -> Result<()> {
882        error_support::init_for_tests();
883        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
884        let url = Url::parse("https://example.com")?;
885        let obs = VisitObservation::new(url.clone())
886            .with_visit_type(VisitType::Link)
887            .with_at(Some(SystemTime::now().into()));
888        apply_observation(&db, obs)?;
889        let guid = get_existing_guid(&db, &url);
890
891        // Set the status to normal
892        apply_and_get_outgoing(&db, vec![]);
893        // It should have changed to normal but still have the initial counter.
894        assert_eq!(get_sync(&db, &url), (SyncStatus::Normal, 1));
895
896        // and an incoming tombstone for that guid
897        let json = json!({
898            "id": guid,
899            "deleted": true,
900        });
901
902        let outgoing = apply_and_get_outgoing(&db, vec![IncomingBso::from_test_content(json)]);
903        assert_eq!(outgoing.len(), 0, "should be nothing outgoing");
904        Ok(())
905    }
906
907    #[test]
908    fn test_outgoing_tombstone() -> Result<()> {
909        error_support::init_for_tests();
910        let db = PlacesDb::open_in_memory(ConnectionType::Sync)?;
911        let url = Url::parse("https://example.com")?;
912        let obs = VisitObservation::new(url.clone())
913            .with_visit_type(VisitType::Link)
914            .with_at(Some(SystemTime::now().into()));
915        apply_observation(&db, obs)?;
916        let guid = get_existing_guid(&db, &url);
917
918        // Set the status to normal
919        apply_and_get_outgoing(&db, vec![]);
920        // It should have changed to normal but still have the initial counter.
921        assert_eq!(get_sync(&db, &url), (SyncStatus::Normal, 1));
922
923        // Delete it.
924        delete_visits_for(&db, &guid)?;
925
926        // should be a local tombstone.
927        assert_eq!(get_tombstone_count(&db), 1);
928
929        let outgoing = apply_and_get_outgoing(&db, vec![]);
930        assert_eq!(outgoing.len(), 1, "tombstone should be uploaded");
931        finish_plan(&db)?;
932        // tombstone should be removed.
933        assert_eq!(get_tombstone_count(&db), 0);
934
935        Ok(())
936    }
937
938    #[test]
939    fn test_clamp_visit_date() {
940        let ts = Timestamp::from(727_747_199_999);
941        assert!(clamp_visit_date(ts).is_err());
942
943        let ts = Timestamp::now();
944        assert_eq!(clamp_visit_date(ts), Ok(ts));
945    }
946}
places/history_sync/plan.rs

places/history_sync/
plan.rs
