shada: Add support for merging everything like described in the doc

7 files changed, 862 insertions, 429 deletions

diff --git a/src/nvim/lib/khash.h b/src/nvim/lib/khash.h
index fd4f910dac..17f653f45e 100644
--- a/src/nvim/lib/khash.h
+++ b/src/nvim/lib/khash.h
@@ -196,6 +196,7 @@ static const double __ac_HASH_UPPER = 0.77;
 
 #define __KHASH_PROTOTYPES(name, khkey_t, khval_t)	 					\
 	extern kh_##name##_t *kh_init_##name(void);							\
+	extern void kh_dealloc_##name(kh_##name##_t *h);					\
 	extern void kh_destroy_##name(kh_##name##_t *h);					\
 	extern void kh_clear_##name(kh_##name##_t *h);						\
 	extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); 	\
@@ -204,16 +205,24 @@ static const double __ac_HASH_UPPER = 0.77;
 	extern void kh_del_##name(kh_##name##_t *h, khint_t x);
 
 #define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	SCOPE kh_##name##_t *kh_init_##name(void) \
+		REAL_FATTR_UNUSED; \
 	SCOPE kh_##name##_t *kh_init_##name(void) {							\
 		return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t));		\
 	}																	\
+	SCOPE void kh_dealloc_##name(kh_##name##_t *h)						\
+		REAL_FATTR_UNUSED;                                      \
+	SCOPE void kh_dealloc_##name(kh_##name##_t *h)						\
+	{																	\
+		kfree((void *)h->keys); kfree(h->flags);					\
+		kfree((void *)h->vals);										\
+	}																	\
 	SCOPE void kh_destroy_##name(kh_##name##_t *h)						\
 		REAL_FATTR_UNUSED;                                      \
 	SCOPE void kh_destroy_##name(kh_##name##_t *h)						\
 	{																	\
 		if (h) {														\
-			kfree((void *)h->keys); kfree(h->flags);					\
-			kfree((void *)h->vals);										\
+			kh_dealloc_##name(h); \
 			kfree(h);													\
 		}																\
 	}																	\
@@ -447,6 +456,13 @@ static kh_inline khint_t __ac_Wang_hash(khint_t key)
 #define kh_destroy(name, h) kh_destroy_##name(h)
 
 /*! @function
+  @abstract     Free memory referenced directly inside a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_dealloc(name, h) kh_dealloc_##name(h)
+
+/*! @function
   @abstract     Reset a hash table without deallocating memory.
   @param  name  Name of the hash table [symbol]
   @param  h     Pointer to the hash table [khash_t(name)*]
diff --git a/src/nvim/mark.c b/src/nvim/mark.c
index b7746f192f..2072483e1d 100644
--- a/src/nvim/mark.c
+++ b/src/nvim/mark.c
@@ -1350,12 +1350,8 @@ size_t mark_buffer_amount(const buf_T *const buf)
 ///                     later then existing one.
 void mark_set_global(const char name, const xfmark_T fm, const bool update)
 {
-  xfmark_T *fm_tgt = NULL;
-  if (ASCII_ISUPPER(name)) {
-    fm_tgt = &(namedfm[name - 'A']);
-  } else if (ascii_isdigit(name)) {
-    fm_tgt = &(namedfm[NMARKS + (name - '0')]);
-  } else {
+  xfmark_T *fm_tgt = &(namedfm[mark_global_index(name)]);
+  if (fm_tgt == &namedfm[-1]) {
     return;
   }
   if (update && fm.fmark.timestamp < fm_tgt->fmark.timestamp) {
diff --git a/src/nvim/mark.h b/src/nvim/mark.h
index 3f63e274bb..aff6e7273a 100644
--- a/src/nvim/mark.h
+++ b/src/nvim/mark.h
@@ -1,6 +1,8 @@
 #ifndef NVIM_MARK_H
 #define NVIM_MARK_H
 
+#include "nvim/macros.h"
+#include "nvim/ascii.h"
 #include "nvim/buffer_defs.h"
 #include "nvim/mark_defs.h"
 #include "nvim/memory.h"
@@ -46,6 +48,32 @@
       SET_FMARK(&(xfmarkp__->fmark), mark_, fnum_); \
     } while (0)
 
+/// Convert mark name to the offset
+static inline int mark_global_index(const char name)
+  FUNC_ATTR_CONST
+{
+  return (ASCII_ISUPPER(name)
+          ? (name - 'A')
+          : (ascii_isdigit(name)
+             ? (NMARKS + (name - '0'))
+             : -1));
+}
+
+/// Convert local mark name to the offset
+static inline int mark_local_index(const char name)
+  FUNC_ATTR_CONST
+{
+  return (ASCII_ISLOWER(name)
+          ? (name - 'a')
+          : (name == '"'
+             ? NMARKS
+             : (name == '^'
+                ? NMARKS + 1
+                : (name == '.'
+                   ? NMARKS + 2
+                   : -1))));
+}
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "mark.h.generated.h"
 #endif
diff --git a/src/nvim/mark_defs.h b/src/nvim/mark_defs.h
index 2f7c9b4ed4..af4727d634 100644
--- a/src/nvim/mark_defs.h
+++ b/src/nvim/mark_defs.h
@@ -19,6 +19,12 @@
 /// Total possible number of global marks
 #define NGLOBALMARKS    (NMARKS + EXTRA_MARKS)
 
+/// Total possible number of local marks
+///
+/// That are uppercase marks plus '"', '^' and '.'. There are other local marks, 
+/// but they are not saved in ShaDa files.
+#define NLOCALMARKS     (NMARKS + 3)
+
 /// Maximum number of marks in jump list
 #define JUMPLISTSIZE    100
 
diff --git a/src/nvim/ops.c b/src/nvim/ops.c
index 52064fae06..0d7c319fba 100644
--- a/src/nvim/ops.c
+++ b/src/nvim/ops.c
@@ -54,22 +54,6 @@
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
 
-/*
- * Registers:
- *      0 = register for latest (unnamed) yank
- *   1..9 = registers '1' to '9', for deletes
- * 10..35 = registers 'a' to 'z'
- *     36 = delete register '-'
- *     37 = selection register '*'
- *     38 = clipboard register '+'
- */
-#define DELETION_REGISTER 36
-#define NUM_SAVED_REGISTERS 37
-// The following registers should not be saved in ShaDa file:
-#define STAR_REGISTER 37
-#define PLUS_REGISTER 38
-#define NUM_REGISTERS 39
-
 static yankreg_T y_regs[NUM_REGISTERS];
 
 static yankreg_T *y_previous = NULL; /* ptr to last written yankreg */
@@ -749,31 +733,6 @@ typedef enum {
   YREG_PUT,
 } yreg_mode_t;
 
-/// Convert register name into register index
-///
-/// @param[in]  regname  Register name.
-///
-/// @return Index in y_regs array or -1 if register name was not recognized.
-static inline int reg_index(const int regname)
-  FUNC_ATTR_CONST
-{
-  if (ascii_isdigit(regname)) {
-    return regname - '0';
-  } else if (ASCII_ISLOWER(regname)) {
-    return CharOrdLow(regname) + 10;
-  } else if (ASCII_ISUPPER(regname)) {
-    return CharOrdUp(regname) + 10;
-  } else if (regname == '-') {
-    return DELETION_REGISTER;
-  } else if (regname == '*') {
-    return STAR_REGISTER;
-  } else if (regname == '+') {
-    return PLUS_REGISTER;
-  } else {
-    return -1;
-  }
-}
-
 /// Return yankreg_T to use, according to the value of `regname`.
 /// Cannot handle the '_' (black hole) register.
 /// Must only be called with a valid register name!
@@ -806,7 +765,7 @@ yankreg_T *get_yank_register(int regname, int mode)
     return y_previous;
   }
 
-  int i = reg_index(regname);
+  int i = op_reg_index(regname);
   // when not 0-9, a-z, A-Z or '-'/'+'/'*': use register 0
   if (i == -1) {
     i = 0;
@@ -5417,7 +5376,7 @@ size_t op_register_amount(void)
 /// Set register to a given value
 void register_set(const char name, const yankreg_T reg)
 {
-  int i = reg_index(name);
+  int i = op_reg_index(name);
   if (i == -1) {
     return;
   }
diff --git a/src/nvim/ops.h b/src/nvim/ops.h
index 4da5cfc93d..5565f1631f 100644
--- a/src/nvim/ops.h
+++ b/src/nvim/ops.h
@@ -3,6 +3,8 @@
 
 #include <stdbool.h>
 
+#include "nvim/macros.h"
+#include "nvim/ascii.h"
 #include "nvim/types.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/os/time.h"
@@ -18,6 +20,22 @@ typedef int (*Indenter)(void);
 #define PUT_LINE_FORWARD 32     /* put linewise register below Visual sel. */
 
 /*
+ * Registers:
+ *      0 = register for latest (unnamed) yank
+ *   1..9 = registers '1' to '9', for deletes
+ * 10..35 = registers 'a' to 'z'
+ *     36 = delete register '-'
+ *     37 = selection register '*'
+ *     38 = clipboard register '+'
+ */
+#define DELETION_REGISTER 36
+#define NUM_SAVED_REGISTERS 37
+// The following registers should not be saved in ShaDa file:
+#define STAR_REGISTER 37
+#define PLUS_REGISTER 38
+#define NUM_REGISTERS 39
+
+/*
  * Operator IDs; The order must correspond to opchars[] in ops.c!
  */
 #define OP_NOP          0       /* no pending operation */
@@ -66,6 +84,31 @@ typedef struct yankreg {
   Dictionary *additional_data;  ///< Additional data from ShaDa file.
 } yankreg_T;
 
+/// Convert register name into register index
+///
+/// @param[in]  regname  Register name.
+///
+/// @return Index in y_regs array or -1 if register name was not recognized.
+static inline int op_reg_index(const int regname)
+  FUNC_ATTR_CONST
+{
+  if (ascii_isdigit(regname)) {
+    return regname - '0';
+  } else if (ASCII_ISLOWER(regname)) {
+    return CharOrdLow(regname) + 10;
+  } else if (ASCII_ISUPPER(regname)) {
+    return CharOrdUp(regname) + 10;
+  } else if (regname == '-') {
+    return DELETION_REGISTER;
+  } else if (regname == '*') {
+    return STAR_REGISTER;
+  } else if (regname == '+') {
+    return PLUS_REGISTER;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ops.h.generated.h"
 #endif
diff --git a/src/nvim/shada.c b/src/nvim/shada.c
index 6920115735..a7bde92032 100644
--- a/src/nvim/shada.c
+++ b/src/nvim/shada.c
@@ -1,3 +1,4 @@
+#include <stdlib.h>
 #include <stddef.h>
 #include <stdbool.h>
 #include <string.h>
@@ -154,7 +155,6 @@ enum SRNIFlags {
   kSDReadUndisableableData = (
     (1 << kSDItemSearchPattern)
     | (1 << kSDItemSubString)
-    | (1 << kSDItemGlobalMark)
     | (1 << kSDItemJump)
   ), ///< Data reading which cannot be disabled by &shada or other options 
      ///< except for disabling reading ShaDa as a whole.
@@ -175,6 +175,11 @@ enum SRNIFlags {
   kSDReadUnknown = (1 << (SHADA_LAST_ENTRY + 1)),  ///< Determines whether 
                                                    ///< unknown items should be 
                                                    ///< read (usually disabled).
+  kSDReadGlobalMarks = (1 << kSDItemGlobalMark),  ///< Determines whether global 
+                                                  ///< marks should be read. Can 
+                                                  ///< only be disabled by 
+                                                  ///< having f0 in &shada when 
+                                                  ///< writing.
   kSDReadLocalMarks = (1 << kSDItemLocalMark),  ///< Determines whether local 
                                                 ///< marks should be read. Can 
                                                 ///< only be disabled by 
@@ -286,6 +291,41 @@ typedef struct {
   uint8_t history_type;
 } HistoryMergerState;
 
+/// ShadaEntry structure that knows whether it should be freed
+typedef struct {
+  ShadaEntry data;      ///< ShadaEntry data.
+  bool can_free_entry;  ///< True if entry can be freed.
+} PossiblyFreedShadaEntry;
+
+/// Structure that holds one file marks.
+typedef struct {
+  PossiblyFreedShadaEntry marks[NLOCALMARKS];  ///< All file marks.
+  PossiblyFreedShadaEntry changes[JUMPLISTSIZE];  ///< All file changes.
+  size_t changes_size;  ///< Number of changes occupied.
+  ShadaEntry *additional_marks;  ///< All marks with unknown names.
+  size_t additional_marks_size;  ///< Size of the additional_marks array.
+  Timestamp greatest_timestamp;  ///< Greatest timestamp among marks.
+  bool is_local_entry;  ///< True if structure comes from the current session.
+} FileMarks;
+
+KHASH_MAP_INIT_STR(file_marks, FileMarks)
+
+/// State structure used by shada_write
+///
+/// Before actually writing most of the data is read to this structure.
+typedef struct {
+  HistoryMergerState hms[HIST_COUNT];  ///< Structures for history merging.
+  PossiblyFreedShadaEntry global_marks[NGLOBALMARKS];  ///< All global marks.
+  PossiblyFreedShadaEntry registers[NUM_SAVED_REGISTERS];  ///< All registers.
+  PossiblyFreedShadaEntry jumps[JUMPLISTSIZE];  ///< All dumped jumps.
+  size_t jumps_size;  ///< Number of jumps occupied.
+  PossiblyFreedShadaEntry search_pattern;  ///< Last search pattern.
+  PossiblyFreedShadaEntry sub_search_pattern;  ///< Last s/ search pattern.
+  PossiblyFreedShadaEntry replacement;  ///< Last s// replacement string.
+  khash_t(strset) dumped_variables;  ///< Names of already dumped variables.
+  khash_t(file_marks) file_marks;  ///< All file marks.
+} WriteMergerState;
+
 struct sd_read_def;
 
 /// Function used to read ShaDa files
@@ -457,18 +497,6 @@ static inline void hmll_dealloc(HMLList *const hmll)
   xfree(hmll->free_entries);
 }
 
-/// Free linked list and all entries
-///
-/// @param[in]  hmll  List to free.
-static inline void hmll_free(HMLList *const hmll)
-  FUNC_ATTR_NONNULL_ALL
-{
-  HMLL_FORALL(hmll, cur_entry) {
-    shada_free_shada_entry(&cur_entry->data);
-  }
-  hmll_dealloc(hmll);
-}
-
 /// Wrapper for reading from file descriptors
 ///
 /// @return true if read was successfull, false otherwise.
@@ -897,14 +925,7 @@ static inline void hms_to_he_array(const HistoryMergerState *const hms_p,
 static inline void hms_dealloc(HistoryMergerState *const hms_p)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (hms_p->reading) {
-    // Free only the linked list if reading because all of the allocated memory 
-    // was either already freed or saved in internal NeoVim history.
-    hmll_dealloc(&hms_p->hmll);
-  } else {
-    // Free everything because when writing data is only used once to write it.
-    hmll_free(&hms_p->hmll);
-  }
+  hmll_dealloc(&hms_p->hmll);
 }
 
 /// Iterate over all history entries in history merger, in order
@@ -957,30 +978,28 @@ static inline bool marks_equal(const pos_T a, const pos_T b)
 static void shada_read(ShaDaReadDef *const sd_reader, const int flags)
   FUNC_ATTR_NONNULL_ALL
 {
-  unsigned srni_flags = 0;
   const bool force = flags & kShaDaForceit;
   const bool get_old_files = flags & (kShaDaGetOldfiles | kShaDaForceit);
   const bool want_marks = flags & kShaDaWantMarks;
-  if (flags & kShaDaWantInfo) {
-    srni_flags |= kSDReadUndisableableData | kSDReadRegisters;
-    if (p_hi) {
-      srni_flags |= kSDReadHistory;
-    }
-    if (find_shada_parameter('!') != NULL) {
-      srni_flags |= kSDReadVariables;
-    }
-    if (find_shada_parameter('%') != NULL && ARGCOUNT == 0) {
-      srni_flags |= kSDReadBufferList;
-    }
-  }
-  if (want_marks) {
-    if (get_shada_parameter('\'') > 0) {
-      srni_flags |= kSDReadLocalMarks | kSDReadChanges;
-    }
-  }
-  if (get_old_files) {
-    srni_flags |= kSDReadLocalMarks;
-  }
+  const unsigned srni_flags = ((flags & kShaDaWantInfo
+                                ? (kSDReadUndisableableData
+                                   | kSDReadRegisters
+                                   | kSDReadGlobalMarks
+                                   | (p_hi ? kSDReadHistory : 0)
+                                   | (find_shada_parameter('!') != NULL
+                                      ? kSDReadVariables
+                                      : 0)
+                                   | (find_shada_parameter('%') != NULL
+                                      && ARGCOUNT == 0
+                                      ? kSDReadBufferList
+                                      : 0))
+                                : 0)
+                               | (want_marks && get_shada_parameter('\'') > 0
+                                  ? kSDReadLocalMarks | kSDReadChanges
+                                  : 0)
+                               | (get_old_files
+                                  ? kSDReadLocalMarks
+                                  : 0));
   if (srni_flags == 0) {
     // Nothing to do.
     return;
@@ -1009,7 +1028,8 @@ static void shada_read(ShaDaReadDef *const sd_reader, const int flags)
     oldfiles_list = list_alloc();
     set_vim_var_list(VV_OLDFILES, oldfiles_list);
   }
-  while (shada_read_next_item(sd_reader, &cur_entry, srni_flags) == NOTDONE) {
+  while (shada_read_next_item(sd_reader, &cur_entry, srni_flags, 0)
+         == NOTDONE) {
     switch (cur_entry.type) {
       case kSDItemMissing: {
         assert(false);
@@ -1689,6 +1709,140 @@ static void shada_pack_entry(msgpack_packer *const packer,
   msgpack_sbuffer_destroy(&sbuf);
 }
 
+/// Write single ShaDa entry, converting it if needed
+///
+/// @warning Frees entry after packing.
+///
+/// @param[in]  packer     Packer used to write entry.
+/// @param[in]  sd_conv    Conversion definitions.
+/// @param[in]  entry      Entry written. If entry.can_free_entry is false then 
+///                        it assumes that entry was not converted, otherwise it 
+///                        is assumed that entry was already converted.
+/// @param[in]  max_kbyte  Maximum size of an item in KiB. Zero means no 
+///                        restrictions.
+static void shada_pack_encoded_entry(msgpack_packer *const packer,
+                                     const vimconv_T *const sd_conv,
+                                     PossiblyFreedShadaEntry entry,
+                                     const size_t max_kbyte)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (entry.can_free_entry) {
+    shada_pack_entry(packer, entry.data, max_kbyte);
+    shada_free_shada_entry(&entry.data);
+    return;
+  }
+#define RUN_WITH_CONVERTED_STRING(cstr, code) \
+  do { \
+    bool did_convert = false; \
+    if (sd_conv->vc_type != CONV_NONE && has_non_ascii((cstr))) { \
+      char *const converted_string = string_convert(sd_conv, (cstr), NULL); \
+      if (converted_string != NULL) { \
+        (cstr) = converted_string; \
+        did_convert = true; \
+      } \
+    } \
+    code \
+    if (did_convert) { \
+      xfree((cstr)); \
+    } \
+  } while (0)
+  switch (entry.data.type) {
+    case kSDItemUnknown:
+    case kSDItemMissing: {
+      assert(false);
+    }
+    case kSDItemSearchPattern: {
+      RUN_WITH_CONVERTED_STRING(entry.data.data.search_pattern.pat, {
+        shada_pack_entry(packer, entry.data, max_kbyte);
+      });
+      break;
+    }
+    case kSDItemHistoryEntry: {
+      RUN_WITH_CONVERTED_STRING(entry.data.data.history_item.string, {
+        shada_pack_entry(packer, entry.data, max_kbyte);
+      });
+      break;
+    }
+    case kSDItemSubString: {
+      RUN_WITH_CONVERTED_STRING(entry.data.data.sub_string.sub, {
+        shada_pack_entry(packer, entry.data, max_kbyte);
+      });
+      break;
+    }
+    case kSDItemVariable: {
+      if (sd_conv->vc_type != CONV_NONE) {
+        convert_object(sd_conv, &entry.data.data.global_var.value);
+      }
+      shada_pack_entry(packer, entry.data, max_kbyte);
+      break;
+    }
+    case kSDItemRegister: {
+      bool did_convert = false;
+      if (sd_conv->vc_type != CONV_NONE) {
+        size_t first_non_ascii = 0;
+        for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
+          if (has_non_ascii(entry.data.data.reg.contents[i])) {
+            first_non_ascii = i;
+            did_convert = true;
+            break;
+          }
+        }
+        if (did_convert) {
+          entry.data.data.reg.contents =
+              xmemdup(entry.data.data.reg.contents,
+                      (entry.data.data.reg.contents_size
+                       * sizeof(entry.data.data.reg.contents)));
+          for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
+            if (i >= first_non_ascii) {
+              entry.data.data.reg.contents[i] = get_converted_string(
+                  sd_conv,
+                  entry.data.data.reg.contents[i],
+                  strlen(entry.data.data.reg.contents[i]));
+            } else {
+              entry.data.data.reg.contents[i] =
+                  xstrdup(entry.data.data.reg.contents[i]);
+            }
+          }
+        }
+      }
+      shada_pack_entry(packer, entry.data, max_kbyte);
+      if (did_convert) {
+        for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
+          xfree(entry.data.data.reg.contents[i]);
+        }
+        xfree(entry.data.data.reg.contents);
+      }
+      break;
+    }
+    case kSDItemHeader:
+    case kSDItemGlobalMark:
+    case kSDItemJump:
+    case kSDItemBufferList:
+    case kSDItemLocalMark:
+    case kSDItemChange: {
+      shada_pack_entry(packer, entry.data, max_kbyte);
+      break;
+    }
+  }
+#undef RUN_WITH_CONVERTED_STRING
+}
+
+/// Compare two FileMarks structure to order them by greatest_timestamp
+///
+/// Order is reversed: structure with greatest greatest_timestamp comes first. 
+/// Function signature is compatible with qsort.
+static int compare_file_marks(const void *a, const void *b)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE
+{
+  const FileMarks *const *const a_fms = a;
+  const FileMarks *const *const b_fms = b;
+  return ((*a_fms)->greatest_timestamp == (*b_fms)->greatest_timestamp
+          ? 0
+          : ((*a_fms)->greatest_timestamp > (*b_fms)->greatest_timestamp
+             ? -1
+             : 1));
+}
+
 /// Write ShaDa file
 ///
 /// @param[in]  sd_writer  Structure containing file writer definition.
@@ -1699,11 +1853,6 @@ static void shada_write(ShaDaWriteDef *const sd_writer,
                         ShaDaReadDef *const sd_reader)
   FUNC_ATTR_NONNULL_ARG(1)
 {
-  // TODO(ZyX-I): Write only one search pattern, substitute search pattern and 
-  // substitute replacement string, which has the greatest timestamp. Make sure 
-  // that this also applies if ShaDa file contains more then one replacement 
-  // string.
-  khash_t(bufset) *const removable_bufs = kh_init(bufset);
   int max_kbyte_i = get_shada_parameter('s');
   if (max_kbyte_i < 0) {
     max_kbyte_i = 10;
@@ -1711,10 +1860,49 @@ static void shada_write(ShaDaWriteDef *const sd_writer,
   if (max_kbyte_i == 0) {
     return;
   }
+
+  WriteMergerState *const wms = xcalloc(1, sizeof(*wms));
+  bool dump_one_history[HIST_COUNT];
+  const bool dump_global_vars = (find_shada_parameter('!') != NULL);
+  int max_reg_lines = get_shada_parameter('<');
+  if (max_reg_lines < 0) {
+    max_reg_lines = get_shada_parameter('"');
+  }
+  const bool limit_reg_lines = max_reg_lines >= 0;
+  const bool dump_registers = (max_reg_lines != 0);
+  khash_t(bufset) *const removable_bufs = kh_init(bufset);
   const size_t max_kbyte = (size_t) max_kbyte_i;
+  const size_t num_marked_files = (size_t) get_shada_parameter('\'');
+  const bool dump_global_marks = get_shada_parameter('f') != 0;
+  bool dump_history = false;
 
-  msgpack_packer *packer = msgpack_packer_new(sd_writer,
-                                              &msgpack_sd_writer_write);
+  // Initialize history merger
+  for (uint8_t i = 0; i < HIST_COUNT; i++) {
+    long num_saved = get_shada_parameter(hist_type2char(i));
+    if (num_saved == -1) {
+      num_saved = p_hi;
+    }
+    if (num_saved > 0) {
+      dump_history = true;
+      dump_one_history[i] = true;
+      hms_init(&wms->hms[i], i, (size_t) num_saved, sd_reader != NULL, false);
+    } else {
+      dump_one_history[i] = false;
+    }
+  }
+
+  const unsigned srni_flags = (
+    kSDReadUndisableableData
+    | kSDReadUnknown
+    | (dump_history ? kSDReadHistory : 0)
+    | (dump_registers ? kSDReadRegisters : 0)
+    | (dump_global_vars ? kSDReadVariables : 0)
+    | (dump_global_marks ? kSDReadGlobalMarks : 0)
+    | (num_marked_files ? kSDReadLocalMarks | kSDReadChanges : 0)
+  );
+
+  msgpack_packer *const packer = msgpack_packer_new(sd_writer,
+                                                    &msgpack_sd_writer_write);
 
   FOR_ALL_BUFFERS(buf) {
     if (buf->b_ffname != NULL && shada_removable((char *) buf->b_ffname)) {
@@ -1723,10 +1911,7 @@ static void shada_write(ShaDaWriteDef *const sd_writer,
     }
   }
 
-  // TODO(ZyX-I): Iterate over sd_reader, keeping “replaced” values in a set.
-
-  // First write values that do not require merging
-  // 1. Header
+  // Write header
   shada_pack_entry(packer, (ShadaEntry) {
     .type = kSDItemHeader,
     .timestamp = os_time(),
@@ -1748,7 +1933,7 @@ static void shada_write(ShaDaWriteDef *const sd_writer,
     }
   }, 0);
 
-  // 2. Buffer list
+  // Write buffer list
   if (find_shada_parameter('%') != NULL) {
     size_t buf_count = 0;
     FOR_ALL_BUFFERS(buf) {
@@ -1784,129 +1969,94 @@ static void shada_write(ShaDaWriteDef *const sd_writer,
     xfree(buflist_entry.data.buffer_list.buffers);
   }
 
-  // 3. Jump list
-  const void *jump_iter = NULL;
-  do {
-    xfmark_T fm;
-    cleanup_jumplist();
-    jump_iter = mark_jumplist_iter(jump_iter, curwin, &fm);
-    const buf_T *const buf = (fm.fmark.fnum == 0
-                              ? NULL
-                              : buflist_findnr(fm.fmark.fnum));
-    if (buf != NULL
-        ? SHADA_REMOVABLE(buf)
-        : fm.fmark.fnum != 0) {
-      continue;
-    }
-    const char *const fname = (char *) (fm.fmark.fnum == 0
-                                        ? (fm.fname == NULL
-                                           ? NULL
-                                           : fm.fname)
-                                        : buf->b_ffname);
-    shada_pack_entry(packer, (ShadaEntry) {
-      .type = kSDItemJump,
-      .timestamp = fm.fmark.timestamp,
-      .data = {
-        .filemark = {
-          .name = NUL,
-          .mark = fm.fmark.mark,
-          .fname = (char *) fname,
-          .additional_data = fm.fmark.additional_data,
-        }
+  // Write some of the variables
+  if (dump_global_vars) {
+    const void *var_iter = NULL;
+    const Timestamp cur_timestamp = os_time();
+    do {
+      typval_T vartv;
+      const char *name;
+      var_iter = var_shada_iter(var_iter, &name, &vartv);
+      if (var_iter == NULL && vartv.v_type == VAR_UNKNOWN) {
+        break;
       }
-    }, max_kbyte);
-  } while (jump_iter != NULL);
-
-  // FIXME No merging currently
-
-#define RUN_WITH_CONVERTED_STRING(cstr, code) \
-  do { \
-    bool did_convert = false; \
-    if (sd_writer->sd_conv.vc_type != CONV_NONE && has_non_ascii((cstr))) { \
-      char *const converted_string = string_convert(&sd_writer->sd_conv, \
-                                                    (cstr), NULL); \
-      if (converted_string != NULL) { \
-        (cstr) = converted_string; \
-        did_convert = true; \
-      } \
-    } \
-    code \
-    if (did_convert) { \
-      xfree((cstr)); \
-    } \
-  } while (0)
-  // 4. History
-  HistoryMergerState hms[HIST_COUNT];
-  for (uint8_t i = 0; i < HIST_COUNT; i++) {
-    long num_saved = get_shada_parameter(hist_type2char(i));
-    if (num_saved == -1) {
-      num_saved = p_hi;
-    }
-    if (num_saved > 0) {
-      hms_init(&hms[i], i, (size_t) num_saved, false, false);
-      hms_insert_whole_neovim_history(&hms[i]);
-      HMS_ITER(&hms[i], cur_entry) {
-        RUN_WITH_CONVERTED_STRING(cur_entry->data.data.history_item.string, {
-          shada_pack_entry(packer, cur_entry->data, max_kbyte);
-        });
+      Object obj = vim_to_object(&vartv);
+      if (sd_writer->sd_conv.vc_type != CONV_NONE) {
+        convert_object(&sd_writer->sd_conv, &obj);
       }
-      hms_dealloc(&hms[i]);
-    }
+      shada_pack_entry(packer, (ShadaEntry) {
+        .type = kSDItemVariable,
+        .timestamp = cur_timestamp,
+        .data = {
+          .global_var = {
+            .name = (char *) name,
+            .value = obj,
+            .additional_elements = NULL,
+          }
+        }
+      }, max_kbyte);
+      api_free_object(obj);
+      clear_tv(&vartv);
+      int kh_ret;
+      (void) kh_put(strset, &wms->dumped_variables, name, &kh_ret);
+    } while (var_iter != NULL);
   }
 
-  // 5. Search patterns
+  // Initialize search pattern
   {
     SearchPattern pat;
     get_search_pattern(&pat);
-    ShadaEntry sp_entry = (ShadaEntry) {
-      .type = kSDItemSearchPattern,
-      .timestamp = pat.timestamp,
+    wms->search_pattern = (PossiblyFreedShadaEntry) {
+      .can_free_entry = false,
       .data = {
-        .search_pattern = {
-          .magic = pat.magic,
-          .smartcase = !pat.no_scs,
-          .has_line_offset = pat.off.line,
-          .place_cursor_at_end = pat.off.end,
-          .offset = pat.off.off,
-          .is_last_used = search_was_last_used(),
-          .is_substitute_pattern = false,
-          .highlighted = (!no_hlsearch && find_shada_parameter('h') != NULL),
-          .pat = (char *) pat.pat,
-          .additional_data = pat.additional_data,
+        .type = kSDItemSearchPattern,
+        .timestamp = pat.timestamp,
+        .data = {
+          .search_pattern = {
+            .magic = pat.magic,
+            .smartcase = !pat.no_scs,
+            .has_line_offset = pat.off.line,
+            .place_cursor_at_end = pat.off.end,
+            .offset = pat.off.off,
+            .is_last_used = search_was_last_used(),
+            .is_substitute_pattern = false,
+            .highlighted = (!no_hlsearch && find_shada_parameter('h') != NULL),
+            .pat = (char *) pat.pat,
+            .additional_data = pat.additional_data,
+          }
         }
       }
     };
-    RUN_WITH_CONVERTED_STRING(sp_entry.data.search_pattern.pat, {
-      shada_pack_entry(packer, sp_entry, max_kbyte);
-    });
   }
+
+  // Initialize substitute search pattern
   {
     SearchPattern pat;
     get_substitute_pattern(&pat);
-    ShadaEntry sp_entry = (ShadaEntry) {
-      .type = kSDItemSearchPattern,
-      .timestamp = pat.timestamp,
+    wms->sub_search_pattern = (PossiblyFreedShadaEntry) {
+      .can_free_entry = false,
       .data = {
-        .search_pattern = {
-          .magic = pat.magic,
-          .smartcase = !pat.no_scs,
-          .has_line_offset = false,
-          .place_cursor_at_end = false,
-          .offset = 0,
-          .is_last_used = !search_was_last_used(),
-          .is_substitute_pattern = true,
-          .highlighted = false,
-          .pat = (char *) pat.pat,
-          .additional_data = pat.additional_data,
+        .type = kSDItemSearchPattern,
+        .timestamp = pat.timestamp,
+        .data = {
+          .search_pattern = {
+            .magic = pat.magic,
+            .smartcase = !pat.no_scs,
+            .has_line_offset = false,
+            .place_cursor_at_end = false,
+            .offset = 0,
+            .is_last_used = !search_was_last_used(),
+            .is_substitute_pattern = true,
+            .highlighted = false,
+            .pat = (char *) pat.pat,
+            .additional_data = pat.additional_data,
+          }
         }
       }
     };
-    RUN_WITH_CONVERTED_STRING(sp_entry.data.search_pattern.pat, {
-      shada_pack_entry(packer, sp_entry, max_kbyte);
-    });
   }
 
-  // 6. Substitute string
+  // Initialize substitute replacement string
   {
     SubReplacementString sub;
     sub_get_replacement(&sub);
@@ -1923,119 +2073,482 @@ static void shada_write(ShaDaWriteDef *const sd_writer,
         }
       }
     };
-    RUN_WITH_CONVERTED_STRING(sub_entry.data.sub_string.sub, {
-      shada_pack_entry(packer, sub_entry, max_kbyte);
-    });
   }
 
-  // 7. Global marks
-  if (get_shada_parameter('f') != 0) {
-    ShadaEntry *const global_marks = list_global_marks(removable_bufs);
-    for (ShadaEntry *mark = global_marks; mark->type != kSDItemMissing;
-         mark++) {
-      shada_pack_entry(packer, *mark, max_kbyte);
-    }
-    xfree(global_marks);
-  }
-
-  // 8. Buffer marks and buffer change list
-  FOR_ALL_BUFFERS(buf) {
-    if (buf->b_ffname == NULL || SHADA_REMOVABLE(buf)) {
+  // Initialize jump list
+  const void *jump_iter = NULL;
+  do {
+    xfmark_T fm;
+    cleanup_jumplist();
+    jump_iter = mark_jumplist_iter(jump_iter, curwin, &fm);
+    const buf_T *const buf = (fm.fmark.fnum == 0
+                              ? NULL
+                              : buflist_findnr(fm.fmark.fnum));
+    if (buf != NULL
+        ? SHADA_REMOVABLE(buf)
+        : fm.fmark.fnum != 0) {
       continue;
     }
-    ShadaEntry *const buffer_marks = list_buffer_marks(buf);
-    for (ShadaEntry *mark = buffer_marks; mark->type != kSDItemMissing;
-         mark++) {
-      shada_pack_entry(packer, *mark, max_kbyte);
+    const char *const fname = (char *) (fm.fmark.fnum == 0
+                                        ? (fm.fname == NULL
+                                           ? NULL
+                                           : fm.fname)
+                                        : buf->b_ffname);
+    if (fname == NULL) {
+      continue;
     }
-    xfree(buffer_marks);
-
-    for (int i = 0; i < buf->b_changelistlen; i++) {
-      const fmark_T fm = buf->b_changelist[i];
-      shada_pack_entry(packer, (ShadaEntry) {
-        .type = kSDItemChange,
-        .timestamp = fm.timestamp,
+    wms->jumps[wms->jumps_size++] = (PossiblyFreedShadaEntry) {
+      .can_free_entry = false,
+      .data = {
+        .type = kSDItemJump,
+        .timestamp = fm.fmark.timestamp,
         .data = {
           .filemark = {
-            .mark = fm.mark,
-            .fname = (char *) buf->b_ffname,
-            .additional_data = fm.additional_data,
+            .name = NUL,
+            .mark = fm.fmark.mark,
+            .fname = (char *) fname,
+            .additional_data = fm.fmark.additional_data,
           }
         }
-      }, max_kbyte);
-    }
-  }
-  // FIXME: Copy previous marks, up to num_marked_files
-  // size_t num_marked_files = get_shada_parameter('\'');
+      }
+    };
+  } while (jump_iter != NULL);
 
-  // 9. Registers
-  int max_num_lines_i = get_shada_parameter('<');