From 36e66edb809a1140aa9d4b116669da4a4697f854 Mon Sep 17 00:00:00 2001 From: Edward Miller Date: Mon, 26 Oct 2026 16:55:20 -0500 Subject: [PATCH] Allow typed deserialization to reuse a tape Fixes simd-lite/simd-json#494. Add the missing from_tape constructor, document padded typed parsing, and cover borrowing, strict fields and allocation recovery. --- src/lib.rs | 39 ++++++++++++++++++ src/value/tape.rs | 5 +-- tests/fill_tape_padded.rs | 83 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a5e95b22..0d390a01 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -188,6 +188,35 @@ pub const INPUT_PADDING: usize = SIMDINPUT_LENGTH; /// `s[..len]`. The padding is unchanged on return, so it can hold the next document of a /// buffer that packs several back to back. /// +/// With Serde enabled, the tape can deserialize directly into a typed value, including +/// strings borrowed from the input. [`Deserializer::from_tape`] lets the caller recover +/// the tape allocation afterward; `Tape::deserialize` consumes it instead: +/// +/// ``` +/// # #[cfg(feature = "serde_impl")] +/// # { +/// use serde::Deserialize; +/// use simd_json::{Buffers, Deserializer, INPUT_PADDING, Tape, fill_tape_padded}; +/// +/// #[derive(Deserialize)] +/// struct Message<'a> { +/// content: &'a str, +/// } +/// +/// let mut input = br#"{"content":"hello\nworld"}"#.to_vec(); +/// let len = input.len(); +/// input.resize(len + INPUT_PADDING, 0); +/// let mut buffers = Buffers::new(len); +/// let mut tape = Tape::null(); +/// fill_tape_padded(&mut input, len, &mut buffers, &mut tape).unwrap(); +/// let mut deserializer = Deserializer::from_tape(tape); +/// let message = Message::deserialize(&mut deserializer).unwrap(); +/// let tape = deserializer.into_tape().reset(); +/// assert_eq!(message.content, "hello\nworld"); +/// assert!(tape.0.is_empty()); // Ready for another input, retaining its allocation. +/// # } +/// ``` +/// /// # Errors /// /// Will return `Err` if `s[..len]` is invalid JSON or `s` is shorter than @@ -882,6 +911,16 @@ impl Deserializer<'_> { } impl<'de> Deserializer<'de> { + /// Creates a deserializer from an existing tape without copying or reparsing it. + /// The allocation can be recovered with [`Self::into_tape`] after deserialization. + #[must_use] + pub fn from_tape(tape: Tape<'de>) -> Self { + Self { + tape: tape.0, + idx: 0, + } + } + /// Extracts the tape from the Deserializer #[must_use] pub fn into_tape(self) -> Tape<'de> { diff --git a/src/value/tape.rs b/src/value/tape.rs index daab7933..304bed48 100644 --- a/src/value/tape.rs +++ b/src/value/tape.rs @@ -46,10 +46,7 @@ impl<'input> Tape<'input> { { use crate::Deserializer; - let mut deserializer = Deserializer { - tape: self.0, - idx: 0, - }; + let mut deserializer = Deserializer::from_tape(self); T::deserialize(&mut deserializer) } diff --git a/tests/fill_tape_padded.rs b/tests/fill_tape_padded.rs index 6e395994..ebc0a552 100644 --- a/tests/fill_tape_padded.rs +++ b/tests/fill_tape_padded.rs @@ -229,6 +229,89 @@ fn invalid_utf8_matches_fill_tape() { ]); } +#[cfg(feature = "serde_impl")] +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Message<'a> { + content: &'a str, +} + +#[cfg(feature = "serde_impl")] +#[test] +fn padded_tape_deserializes_borrowed_strings() { + let mut buffers = Buffers::new(256); + for offset in [0, 15, 16, 31, 32, 63, 64] { + let prefix = "a".repeat(offset); + let doc = format!(r#"{{"content":"{prefix}\n\ud83c\udf33"}}"#); + let mut input = doc.into_bytes(); + let len = input.len(); + input.resize(len + INPUT_PADDING, PAD); + let start = input.as_ptr(); + let end = start.wrapping_add(len); + let mut tape = Tape::null(); + fill_tape_padded(&mut input, len, &mut buffers, &mut tape).unwrap(); + let message: Message<'_> = tape.deserialize().unwrap(); + assert_eq!(message.content, format!("{prefix}\n🌳")); + assert!((start..end).contains(&message.content.as_ptr())); + assert!(input[len..].iter().all(|&byte| byte == PAD)); + } +} + +#[cfg(feature = "serde_impl")] +#[test] +fn padded_tape_preserves_struct_field_rejections() { + let mut buffers = Buffers::new(256); + for doc in [ + br#"{"content":"a","content":"b"}"#.as_slice(), + br#"{"content":"a","\u0063ontent":"b"}"#, + br#"{"content":"a","extra":1}"#, + ] { + let mut reference = doc.to_vec(); + assert!(simd_json::serde::from_slice::>(&mut reference).is_err()); + let mut input = doc.to_vec(); + input.resize(doc.len() + INPUT_PADDING, PAD); + let mut tape = Tape::null(); + fill_tape_padded(&mut input, doc.len(), &mut buffers, &mut tape).unwrap(); + assert!(tape.deserialize::>().is_err()); + assert!(input[doc.len()..].iter().all(|&byte| byte == PAD)); + } +} + +#[cfg(feature = "serde_impl")] +#[test] +fn padded_tape_allocation_survives_typed_success_and_failure() { + use serde::Deserialize as _; + use simd_json::Deserializer; + + let mut buffers = Buffers::new(256); + let mut tape = Tape::null(); + tape.0.reserve(16); + let allocation = tape.0.as_ptr(); + let capacity = tape.0.capacity(); + let mut input = br#"{"content":"first\nmessage"}"#.to_vec(); + let len = input.len(); + input.resize(len + INPUT_PADDING, PAD); + fill_tape_padded(&mut input, len, &mut buffers, &mut tape).unwrap(); + let mut deserializer = Deserializer::from_tape(tape); + let message = Message::deserialize(&mut deserializer).unwrap(); + let mut tape = deserializer.into_tape().reset(); + assert_eq!(tape.0.as_ptr(), allocation); + assert_eq!(tape.0.capacity(), capacity); + assert!(tape.0.is_empty()); + + let mut other = br#"{"content":"second","extra":true}"#.to_vec(); + let len = other.len(); + other.resize(len + INPUT_PADDING, PAD); + fill_tape_padded(&mut other, len, &mut buffers, &mut tape).unwrap(); + let mut deserializer = Deserializer::from_tape(tape); + assert!(Message::deserialize(&mut deserializer).is_err()); + let tape = deserializer.into_tape().reset(); + assert_eq!(tape.0.as_ptr(), allocation); + assert_eq!(tape.0.capacity(), capacity); + assert!(tape.0.is_empty()); + assert_eq!(message.content, "first\nmessage"); +} + /// Deterministic xorshift so the corpus is reproducible (no rand dev-dependency). struct Rng(u64);