webext_storage/sync/
mod.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5pub(crate) mod bridge;
6mod incoming;
7mod outgoing;
8
9#[cfg(test)]
10mod sync_tests;
11
12use crate::api::{StorageChanges, StorageValueChange};
13use crate::db::StorageDb;
14use crate::error::*;
15use serde::Deserialize;
16use serde_derive::*;
17use sql_support::ConnExt;
18use sync_guid::Guid as SyncGuid;
19
20use incoming::IncomingAction;
21
22type JsonMap = serde_json::Map<String, serde_json::Value>;
23
24pub const STORAGE_VERSION: usize = 1;
25
26#[derive(Debug, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct WebextRecord {
29    #[serde(rename = "id")]
30    guid: SyncGuid,
31    #[serde(rename = "extId")]
32    ext_id: String,
33    data: String,
34}
35
36// Perform a 2-way or 3-way merge, where the incoming value wins on conflict.
37fn merge(
38    ext_id: String,
39    mut other: JsonMap,
40    mut ours: JsonMap,
41    parent: Option<JsonMap>,
42) -> IncomingAction {
43    if other == ours {
44        return IncomingAction::Same { ext_id };
45    }
46    let old_incoming = other.clone();
47    // worst case is keys in each are unique.
48    let mut changes = StorageChanges::with_capacity(other.len() + ours.len());
49    if let Some(parent) = parent {
50        // Perform 3-way merge. First, for every key in parent,
51        // compare the parent value with the incoming value to compute
52        // an implicit "diff".
53        for (key, parent_value) in parent.into_iter() {
54            if let Some(incoming_value) = other.remove(&key) {
55                if incoming_value != parent_value {
56                    trace!(
57                        "merge: key {} was updated in incoming - copying value locally",
58                        key
59                    );
60                    let old_value = ours.remove(&key);
61                    let new_value = Some(incoming_value.clone());
62                    if old_value != new_value {
63                        changes.push(StorageValueChange {
64                            key: key.clone(),
65                            old_value,
66                            new_value,
67                        });
68                    }
69                    ours.insert(key, incoming_value);
70                }
71            } else {
72                // Key was not present in incoming value.
73                // Another client must have deleted it.
74                trace!(
75                    "merge: key {} no longer present in incoming - removing it locally",
76                    key
77                );
78                if let Some(old_value) = ours.remove(&key) {
79                    changes.push(StorageValueChange {
80                        key,
81                        old_value: Some(old_value),
82                        new_value: None,
83                    });
84                }
85            }
86        }
87
88        // Then, go through every remaining key in incoming. These are
89        // the ones where a corresponding key does not exist in
90        // parent, so it is a new key, and we need to add it.
91        for (key, incoming_value) in other.into_iter() {
92            trace!(
93                "merge: key {} doesn't occur in parent - copying from incoming",
94                key
95            );
96            changes.push(StorageValueChange {
97                key: key.clone(),
98                old_value: None,
99                new_value: Some(incoming_value.clone()),
100            });
101            ours.insert(key, incoming_value);
102        }
103    } else {
104        // No parent. Server wins. Overwrite every key in ours with
105        // the corresponding value in other.
106        trace!("merge: no parent - copying all keys from incoming");
107        for (key, incoming_value) in other.into_iter() {
108            let old_value = ours.remove(&key);
109            let new_value = Some(incoming_value.clone());
110            if old_value != new_value {
111                changes.push(StorageValueChange {
112                    key: key.clone(),
113                    old_value,
114                    new_value,
115                });
116            }
117            ours.insert(key, incoming_value);
118        }
119    }
120
121    if ours == old_incoming {
122        IncomingAction::TakeRemote {
123            ext_id,
124            data: old_incoming,
125            changes,
126        }
127    } else {
128        IncomingAction::Merge {
129            ext_id,
130            data: ours,
131            changes,
132        }
133    }
134}
135
136fn remove_matching_keys(mut ours: JsonMap, keys_to_remove: &JsonMap) -> (JsonMap, StorageChanges) {
137    let mut changes = StorageChanges::with_capacity(keys_to_remove.len());
138    for key in keys_to_remove.keys() {
139        if let Some(old_value) = ours.remove(key) {
140            changes.push(StorageValueChange {
141                key: key.clone(),
142                old_value: Some(old_value),
143                new_value: None,
144            });
145        }
146    }
147    (ours, changes)
148}
149
150/// Holds a JSON-serialized map of all synced changes for an extension.
151#[derive(Clone, Debug, Eq, PartialEq)]
152pub struct SyncedExtensionChange {
153    /// The extension ID.
154    pub ext_id: String,
155    /// The contents of a `StorageChanges` struct, in JSON format. We don't
156    /// deserialize these because they need to be passed back to the browser
157    /// as strings anyway.
158    pub changes: String,
159}
160
161// Fetches the applied changes we stashed in the storage_sync_applied table.
162pub fn get_synced_changes(db: &StorageDb) -> Result<Vec<SyncedExtensionChange>> {
163    let signal = db.begin_interrupt_scope()?;
164    let sql = "SELECT ext_id, changes FROM temp.storage_sync_applied";
165    let conn = db.get_connection()?;
166    conn.query_rows_and_then(sql, [], |row| -> Result<_> {
167        signal.err_if_interrupted()?;
168        Ok(SyncedExtensionChange {
169            ext_id: row.get("ext_id")?,
170            changes: row.get("changes")?,
171        })
172    })
173}
174
175// Helpers for tests
176#[cfg(test)]
177pub mod test {
178    use crate::db::{test::new_mem_db, StorageDb};
179    use crate::schema::create_empty_sync_temp_tables;
180
181    pub fn new_syncable_mem_db() -> StorageDb {
182        error_support::init_for_tests();
183        let db = new_mem_db();
184        let conn = db.get_connection().expect("should retrieve connection");
185        create_empty_sync_temp_tables(conn).expect("should work");
186        db
187    }
188}
189
190#[cfg(test)]
191mod tests {
192    use super::test::new_syncable_mem_db;
193    use super::*;
194    use serde_json::json;
195
196    #[test]
197    fn test_serde_record_ser() {
198        assert_eq!(
199            serde_json::to_string(&WebextRecord {
200                guid: "guid".into(),
201                ext_id: "ext_id".to_string(),
202                data: "data".to_string()
203            })
204            .unwrap(),
205            r#"{"id":"guid","extId":"ext_id","data":"data"}"#
206        );
207    }
208
209    // a macro for these tests - constructs a serde_json::Value::Object
210    macro_rules! map {
211        ($($map:tt)+) => {
212            json!($($map)+).as_object().unwrap().clone()
213        };
214    }
215
216    macro_rules! change {
217        ($key:literal, None, None) => {
218            StorageValueChange {
219                key: $key.to_string(),
220                old_value: None,
221                new_value: None,
222            };
223        };
224        ($key:literal, $old:tt, None) => {
225            StorageValueChange {
226                key: $key.to_string(),
227                old_value: Some(json!($old)),
228                new_value: None,
229            }
230        };
231        ($key:literal, None, $new:tt) => {
232            StorageValueChange {
233                key: $key.to_string(),
234                old_value: None,
235                new_value: Some(json!($new)),
236            }
237        };
238        ($key:literal, $old:tt, $new:tt) => {
239            StorageValueChange {
240                key: $key.to_string(),
241                old_value: Some(json!($old)),
242                new_value: Some(json!($new)),
243            }
244        };
245    }
246    macro_rules! changes {
247        ( ) => {
248            StorageChanges::new()
249        };
250        ( $( $change:expr ),* ) => {
251            {
252                let mut changes = StorageChanges::new();
253                $(
254                    changes.push($change);
255                )*
256                changes
257            }
258        };
259    }
260
261    #[test]
262    fn test_3way_merging() {
263        // No conflict - identical local and remote.
264        assert_eq!(
265            merge(
266                "ext-id".to_string(),
267                map!({"one": "one", "two": "two"}),
268                map!({"two": "two", "one": "one"}),
269                Some(map!({"parent_only": "parent"})),
270            ),
271            IncomingAction::Same {
272                ext_id: "ext-id".to_string()
273            }
274        );
275        assert_eq!(
276            merge(
277                "ext-id".to_string(),
278                map!({"other_only": "other", "common": "common"}),
279                map!({"ours_only": "ours", "common": "common"}),
280                Some(map!({"parent_only": "parent", "common": "old_common"})),
281            ),
282            IncomingAction::Merge {
283                ext_id: "ext-id".to_string(),
284                data: map!({"other_only": "other", "ours_only": "ours", "common": "common"}),
285                changes: changes![change!("other_only", None, "other")],
286            }
287        );
288        // Simple conflict - parent value is neither local nor incoming. incoming wins.
289        assert_eq!(
290            merge(
291                "ext-id".to_string(),
292                map!({"other_only": "other", "common": "incoming"}),
293                map!({"ours_only": "ours", "common": "local"}),
294                Some(map!({"parent_only": "parent", "common": "parent"})),
295            ),
296            IncomingAction::Merge {
297                ext_id: "ext-id".to_string(),
298                data: map!({"other_only": "other", "ours_only": "ours", "common": "incoming"}),
299                changes: changes![
300                    change!("common", "local", "incoming"),
301                    change!("other_only", None, "other")
302                ],
303            }
304        );
305        // Local change, no conflict.
306        assert_eq!(
307            merge(
308                "ext-id".to_string(),
309                map!({"other_only": "other", "common": "old_value"}),
310                map!({"ours_only": "ours", "common": "new_value"}),
311                Some(map!({"parent_only": "parent", "common": "old_value"})),
312            ),
313            IncomingAction::Merge {
314                ext_id: "ext-id".to_string(),
315                data: map!({"other_only": "other", "ours_only": "ours", "common": "new_value"}),
316                changes: changes![change!("other_only", None, "other")],
317            }
318        );
319        // Field was removed remotely.
320        assert_eq!(
321            merge(
322                "ext-id".to_string(),
323                map!({"other_only": "other"}),
324                map!({"common": "old_value"}),
325                Some(map!({"common": "old_value"})),
326            ),
327            IncomingAction::TakeRemote {
328                ext_id: "ext-id".to_string(),
329                data: map!({"other_only": "other"}),
330                changes: changes![
331                    change!("common", "old_value", None),
332                    change!("other_only", None, "other")
333                ],
334            }
335        );
336        // Field was removed remotely but we added another one.
337        assert_eq!(
338            merge(
339                "ext-id".to_string(),
340                map!({"other_only": "other"}),
341                map!({"common": "old_value", "new_key": "new_value"}),
342                Some(map!({"common": "old_value"})),
343            ),
344            IncomingAction::Merge {
345                ext_id: "ext-id".to_string(),
346                data: map!({"other_only": "other", "new_key": "new_value"}),
347                changes: changes![
348                    change!("common", "old_value", None),
349                    change!("other_only", None, "other")
350                ],
351            }
352        );
353        // Field was removed both remotely and locally.
354        assert_eq!(
355            merge(
356                "ext-id".to_string(),
357                map!({}),
358                map!({"new_key": "new_value"}),
359                Some(map!({"common": "old_value"})),
360            ),
361            IncomingAction::Merge {
362                ext_id: "ext-id".to_string(),
363                data: map!({"new_key": "new_value"}),
364                changes: changes![],
365            }
366        );
367    }
368
369    #[test]
370    fn test_remove_matching_keys() {
371        assert_eq!(
372            remove_matching_keys(
373                map!({"key1": "value1", "key2": "value2"}),
374                &map!({"key1": "ignored", "key3": "ignored"})
375            ),
376            (
377                map!({"key2": "value2"}),
378                changes![change!("key1", "value1", None)]
379            )
380        );
381    }
382
383    #[test]
384    fn test_get_synced_changes() -> Result<()> {
385        let db = new_syncable_mem_db();
386        let conn = db.get_connection()?;
387        conn.execute_batch(&format!(
388            r#"INSERT INTO temp.storage_sync_applied (ext_id, changes)
389                VALUES
390                ('an-extension', '{change1}'),
391                ('ext"id', '{change2}')
392            "#,
393            change1 = serde_json::to_string(&changes![change!("key1", "old-val", None)])?,
394            change2 = serde_json::to_string(&changes![change!("key-for-second", None, "new-val")])?
395        ))?;
396        let changes = get_synced_changes(&db)?;
397        assert_eq!(changes[0].ext_id, "an-extension");
398        // sanity check it's valid!
399        let c1: JsonMap =
400            serde_json::from_str(&changes[0].changes).expect("changes must be an object");
401        assert_eq!(
402            c1.get("key1")
403                .expect("must exist")
404                .as_object()
405                .expect("must be an object")
406                .get("oldValue"),
407            Some(&json!("old-val"))
408        );
409
410        // phew - do it again to check the string got escaped.
411        assert_eq!(
412            changes[1],
413            SyncedExtensionChange {
414                ext_id: "ext\"id".into(),
415                changes: r#"{"key-for-second":{"newValue":"new-val"}}"#.into(),
416            }
417        );
418        assert_eq!(changes[1].ext_id, "ext\"id");
419        let c2: JsonMap =
420            serde_json::from_str(&changes[1].changes).expect("changes must be an object");
421        assert_eq!(
422            c2.get("key-for-second")
423                .expect("must exist")
424                .as_object()
425                .expect("must be an object")
426                .get("newValue"),
427            Some(&json!("new-val"))
428        );
429        Ok(())
430    }
431}