diff --git a/Cargo.toml b/Cargo.toml index ef2183a8..c332a29c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,8 @@ serde_json = "1.0.59" [dependencies] +indexmap = "~2.13" +smallvec = "1" hashbrown = "0.13.2" dashmap = { version = "5.4", features = ["raw-api"] } lazy_static = "1.4.0" diff --git a/fuzz/fuzz_targets/fuzz_json_de.rs b/fuzz/fuzz_targets/fuzz_json_de.rs index b5a35cff..92192774 100644 --- a/fuzz/fuzz_targets/fuzz_json_de.rs +++ b/fuzz/fuzz_targets/fuzz_json_de.rs @@ -1,12 +1,19 @@ #![no_main] use arbitrary_json::ArbitraryValue; -use ijson::IValue; +use ijson::{IValue, IValueDeserSeed}; use libfuzzer_sys::fuzz_target; use serde::Deserialize; fuzz_target!(|value: ArbitraryValue| { let json_string = value.to_string(); let mut deserializer = serde_json::Deserializer::from_str(&json_string); - let _ = IValue::deserialize(&mut deserializer); + let ordinary = IValue::deserialize(&mut deserializer); + let mut deserializer = serde_json::Deserializer::from_str(&json_string); + let buffered = IValueDeserSeed::new(None).deserialize_compact_objects(&mut deserializer); + match (ordinary, buffered) { + (Ok(ordinary), Ok(buffered)) => assert_eq!(ordinary, buffered), + (Err(_), Err(_)) => {} + _ => panic!("buffered objects changed whether JSON parsing succeeds"), + } }); diff --git a/src/de.rs b/src/de.rs index 2270aae5..86acc10f 100644 --- a/src/de.rs +++ b/src/de.rs @@ -1,5 +1,6 @@ use std::convert::TryFrom; use std::fmt::{self, Formatter}; +use std::marker::PhantomData; use serde::de::{ DeserializeSeed, EnumAccess, Error as SError, Expected, IntoDeserializer, MapAccess, SeqAccess, @@ -36,8 +37,33 @@ impl IValueDeserSeed { pub fn new(fpha_config: Option) -> Self { IValueDeserSeed { fpha_config } } + + /// Deserializes using temporary object buffers, then moves each object's + /// unique fields into exactly sized storage. Arrays keep normal growth. + /// The caller must still check for trailing input with the deserializer. + pub fn deserialize_compact_objects<'de, D>(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let mut buffers = Vec::new(); + deserializer.deserialize_any(ValueVisitor { + fpha_config: self.fpha_config, + buffers: Some(&mut buffers), + }) + } } +/// Temporary parser storage; independent of the final object's table threshold. +pub(crate) const OBJECT_BUFFER_INLINE_CAPACITY: usize = 16; + +pub(crate) type InlineObjectEntries = + smallvec::SmallVec<[(IString, IValue); OBJECT_BUFFER_INLINE_CAPACITY]>; + +// IndexMap preserves insertion order and replaces duplicate values. Reuse the +// existing hash builder and retain drained maps for later sibling objects. +pub(crate) type ObjectBuffer = + indexmap::IndexMap; + impl<'de> DeserializeSeed<'de> for IValueDeserSeed { type Value = IValue; @@ -82,7 +108,10 @@ impl<'de> Deserialize<'de> for IArray { where D: Deserializer<'de>, { - deserializer.deserialize_seq(ArrayVisitor { fpha_config: None }) + deserializer.deserialize_seq(ArrayVisitor { + fpha_config: None, + buffers: None, + }) } } @@ -91,21 +120,43 @@ impl<'de> Deserialize<'de> for IObject { where D: Deserializer<'de>, { - deserializer.deserialize_map(ObjectVisitor { fpha_config: None }) + deserializer.deserialize_map(ObjectVisitor { + fpha_config: None, + buffers: None, + }) } } -struct ValueVisitor { +struct ValueVisitor<'a> { fpha_config: Option, + // Pool of drained maps that retain capacity for reuse while parsing objects. + // Maps are popped when needed and returned after use; indices do not identify + // JSON objects or nesting levels. None selects ordinary deserialization. + buffers: Option<&'a mut Vec>, } -impl ValueVisitor { +impl ValueVisitor<'_> { fn new(fpha_config: Option) -> Self { - ValueVisitor { fpha_config } + ValueVisitor { + fpha_config, + buffers: None, + } + } +} + +// Nested visitors share the pool of reusable object buffers. +impl<'de> DeserializeSeed<'de> for ValueVisitor<'_> { + type Value = IValue; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(self) } } -impl<'de> Visitor<'de> for ValueVisitor { +impl<'de> Visitor<'de> for ValueVisitor<'_> { type Value = IValue; fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { @@ -152,7 +203,7 @@ impl<'de> Visitor<'de> for ValueVisitor { where D: Deserializer<'de>, { - IValueDeserSeed::new(self.fpha_config).deserialize(deserializer) + self.deserialize(deserializer) } #[inline] @@ -167,6 +218,7 @@ impl<'de> Visitor<'de> for ValueVisitor { { ArrayVisitor { fpha_config: self.fpha_config, + buffers: self.buffers, } .visit_seq(visitor) .map(Into::into) @@ -178,6 +230,7 @@ impl<'de> Visitor<'de> for ValueVisitor { { ObjectVisitor { fpha_config: self.fpha_config, + buffers: self.buffers, } .visit_map(visitor) .map(Into::into) @@ -248,11 +301,12 @@ impl<'de> Visitor<'de> for StringVisitor { } } -struct ArrayVisitor { +struct ArrayVisitor<'a> { fpha_config: Option, + buffers: Option<&'a mut Vec>, } -impl<'de> Visitor<'de> for ArrayVisitor { +impl<'de> Visitor<'de> for ArrayVisitor<'_> { type Value = IArray; fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { @@ -260,13 +314,16 @@ impl<'de> Visitor<'de> for ArrayVisitor { } #[inline] - fn visit_seq(self, mut visitor: V) -> Result + fn visit_seq(mut self, mut visitor: V) -> Result where V: SeqAccess<'de>, { let mut arr = IArray::with_capacity(visitor.size_hint().unwrap_or(0)) .map_err(|_| SError::custom("Failed to allocate array"))?; - while let Some(v) = visitor.next_element_seed(IValueDeserSeed::new(self.fpha_config))? { + while let Some(v) = visitor.next_element_seed(ValueVisitor { + fpha_config: self.fpha_config, + buffers: self.buffers.as_deref_mut(), + })? { match self.fpha_config { Some(FPHAConfig { fpha_type }) => arr.push_with_fp_type(v, fpha_type), None => arr.push(v).map_err(Into::into), @@ -277,29 +334,96 @@ impl<'de> Visitor<'de> for ArrayVisitor { } } -struct ObjectVisitor { +struct ObjectVisitor<'a> { fpha_config: Option, + buffers: Option<&'a mut Vec>, } -impl<'de> Visitor<'de> for ObjectVisitor { +impl<'de> Visitor<'de> for ObjectVisitor<'_> { type Value = IObject; fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { formatter.write_str("JSON object") } - fn visit_map(self, mut visitor: V) -> Result + fn visit_map(self, mut map_access: V) -> Result where V: MapAccess<'de>, { - let mut obj = IObject::with_capacity(visitor.size_hint().unwrap_or(0)) + if let Some(buffer_pool) = self.buffers { + // Keep up to 16 unique fields inline while parsing. Final objects + // still use their own eight-field threshold for hash tables. + let mut inline_entries = InlineObjectEntries::new(); + let mut indexed_entries: Option = None; + // Nested values reuse the same pool, but each active object owns its entries. + while let Some((key, value)) = map_access.next_entry_seed( + PhantomData::, + ValueVisitor { + fpha_config: self.fpha_config, + buffers: Some(&mut *buffer_pool), + }, + )? { + let entries_map = match &mut indexed_entries { + Some(entries_map) => entries_map, + None => { + // Duplicate keys replace their value without consuming another slot. + if let Some((_, existing_value)) = inline_entries + .iter_mut() + .find(|(existing_key, _)| *existing_key == key) + { + *existing_value = value; + continue; + } + if inline_entries.len() < OBJECT_BUFFER_INLINE_CAPACITY { + inline_entries.push((key, value)); + continue; + } + // The next unique field exceeds inline storage: move to a pooled map. + let mut entries_map = buffer_pool.pop().unwrap_or_default(); + entries_map + .try_reserve(inline_entries.len() + 1) + .map_err(SError::custom)?; + entries_map.extend(inline_entries.drain(..)); + indexed_entries.insert(entries_map) + } + }; + // Reserve only for new keys; inserting a duplicate replaces its value. + if entries_map.len() == entries_map.capacity() && !entries_map.contains_key(&key) { + entries_map.try_reserve(1).map_err(SError::custom)?; + } + entries_map.insert(key, value); + } + // Allocate only for validated, unique fields; move their values. + if let Some(mut entries_map) = indexed_entries { + let object = IObject::from_unique_entries(&mut entries_map) + .map_err(|_| SError::custom("Failed to allocate object"))?; + // Construction drains the map; retain its capacity for another object. + // Pooling is optional: a failed reservation must not reject + // an object that was already parsed successfully. + if buffer_pool.try_reserve(1).is_ok() { + buffer_pool.push(entries_map); + } + return Ok(object); + } + return IObject::from_unique_inline_entries(inline_entries) + .map_err(|_| SError::custom("Failed to allocate object")); + } + + // Ordinary deserialization inserts directly into a growing object. + let mut object = IObject::with_capacity(map_access.size_hint().unwrap_or(0)) .map_err(|_| SError::custom("Failed to allocate object"))?; - while let Some(k) = visitor.next_key::()? { - let v = visitor.next_value_seed(IValueDeserSeed::new(self.fpha_config))?; - obj.insert(k, v) + while let Some((key, value)) = map_access.next_entry_seed( + PhantomData::, + ValueVisitor { + fpha_config: self.fpha_config, + buffers: None, + }, + )? { + object + .insert(key, value) .map_err(|e| SError::custom(e.to_string()))?; } - Ok(obj) + Ok(object) } } @@ -1072,6 +1196,33 @@ mod tests { use crate::array::ArraySliceRef; use serde::de::DeserializeSeed; + #[test] + #[cfg(not(miri))] + fn compact_objects_do_not_reserve_for_unvalidated_or_duplicate_fields() { + for (input, valid) in [ + (format!("{{{}", ":".repeat(100_000)), false), + (r#"{"a":{"x":1},"b":[{"y":2},"#.to_owned(), false), + ( + format!("{{{}\"same\":2}}", "\"same\":1,".repeat(100_000)), + true, + ), + ] { + // Build the input outside the measured scope. Repetition must not + // translate into a proportional temporary or final object allocation. + let allocations = mockalloc::record_allocs(|| { + let mut de = serde_json::Deserializer::from_str(&input); + let result = IValueDeserSeed::new(None).deserialize_compact_objects(&mut de); + assert_eq!(result.is_ok(), valid); + if let Ok(value) = result { + assert_eq!(value.as_object().unwrap().capacity(), 1); + de.end().unwrap(); + } + }); + assert!(allocations.peak_mem() < 64 * 1024); + allocations.result().unwrap(); + } + } + #[test] fn test_deserialize_with_f64_fp() { let json = r#"[1.5, 2.5, 3.5]"#; diff --git a/src/object.rs b/src/object.rs index 403266bd..1efc6155 100644 --- a/src/object.rs +++ b/src/object.rs @@ -96,7 +96,15 @@ struct SplitHeader<'a> { impl<'a> SplitHeader<'a> { // Returns the *item index* (Ok) in no-table mode, or the *table bucket* (Ok) in table mode. fn find_bucket(&self, key: &IString) -> Result { + self.find_bucket_impl::(key) + } + + // Unique entries still need collision handling, but no equality checks. + fn find_bucket_impl(&self, key: &IString) -> Result { if !has_table(self.cap) { + if !CHECK_DUPLICATES { + return Err(u32::MAX); + } // Small object: linear scan the items array. // No table bucket to report; insertion just appends. return self @@ -121,7 +129,7 @@ impl<'a> SplitHeader<'a> { // If the bucket contains our key, we found the bucket let k = &self.items.get_unchecked(index as usize).key; - if k == key { + if CHECK_DUPLICATES && k == key { return Ok(bucket as u32); } @@ -332,6 +340,16 @@ trait HeaderMut<'a>: ThinMutExt<'a, Header> { self.set_len(res.checked_add(1).expect("object length overflow")); res } + + // SAFETY: there must be capacity for one more entry, and key must be absent. + unsafe fn push_unique(&mut self, key: IString, value: IValue) { + let bucket = self + .split() + .find_bucket_impl::(&key) + .expect_err("unique insertion only finds vacant buckets"); + let index = self.push(key, value); + self.reborrow().split_mut().shift(bucket, index); + } fn clear(&mut self) { // Clear the table for item in self.reborrow().split_mut().table { @@ -683,6 +701,44 @@ impl IObject { } } + /// Moves already-deduplicated inline fields into their final storage. + pub(crate) fn from_unique_inline_entries( + entries: crate::de::InlineObjectEntries, + ) -> Result { + let mut obj = Self::with_capacity(entries.len())?; + if !entries.is_empty() { + // SAFETY: the object is non-static and has space for every entry. + // The parser has already removed duplicate keys. Objects above + // the storage threshold still need their hash table initialized. + unsafe { + let mut header = obj.header_mut(); + for (key, value) in entries { + header.push_unique(key, value); + } + } + } + Ok(obj) + } + + /// Moves fields whose uniqueness is guaranteed by the source map. + pub(crate) fn from_unique_entries( + entries: &mut crate::de::ObjectBuffer, + ) -> Result { + let mut obj = Self::with_capacity(entries.len())?; + if !entries.is_empty() { + // SAFETY: allocation covers every drained entry; the source map + // guarantees unique keys. Existing probing and shifting preserve + // the final table's collision and lookup invariants. + unsafe { + let mut header = obj.header_mut(); + for (key, value) in entries.drain(..) { + header.push_unique(key, value); + } + } + } + Ok(obj) + } + fn header(&self) -> ThinRef<'_, Header> { unsafe { ThinRef::new(self.0.ptr().cast()) } } @@ -740,10 +796,19 @@ impl IObject { if current_capacity >= desired_capacity { return Ok(()); } - self.resize_internal(cmp::max( - current_capacity.checked_mul(2).ok_or(AllocError)?, + let grown = cmp::max( + current_capacity + .checked_add(current_capacity.div_ceil(4)) + .ok_or(AllocError)?, desired_capacity.max(4), - )) + ); + // Exact-size parsed objects can have capacities such as 5 or 7. Keep + // growth table-free while all requested fields still fit a small object. + self.resize_internal(if desired_capacity <= SMALL_OBJECT_THRESHOLD { + grown.min(SMALL_OBJECT_THRESHOLD) + } else { + grown + }) } /// Returns a view of an entry within this object. @@ -751,9 +816,20 @@ impl IObject { /// # Errors /// Returns an `AllocError` if reserving space for the entry fails. pub fn entry(&mut self, key: impl Into) -> Result, IJsonError> { + let key = key.into(); + // A full object only needs to grow if the key is absent. + if self.len() == self.capacity() { + if let Ok(bucket) = self.header().split().find_bucket(&key) { + return Ok(Entry::Occupied(OccupiedEntry { + // SAFETY: finding an occupied entry proves the object is non-static. + header: unsafe { self.header_mut() }, + bucket, + })); + } + } self.reserve(1)?; // Safety: cannot be static after reserving space - Ok(unsafe { self.header_mut().entry(key.into()) }) + Ok(unsafe { self.header_mut().entry(key) }) } /// Returns a view of an entry within this object, whilst avoiding /// cloning the key if the entry is already occupied. @@ -761,6 +837,16 @@ impl IObject { /// # Errors /// Returns an `AllocError` if reserving space for the entry fails. pub fn entry_or_clone(&mut self, key: &IString) -> Result, IJsonError> { + // Reuse the occupied bucket instead of reserving or probing a second time. + if self.len() == self.capacity() { + if let Ok(bucket) = self.header().split().find_bucket(key) { + return Ok(Entry::Occupied(OccupiedEntry { + // SAFETY: finding an occupied entry proves the object is non-static. + header: unsafe { self.header_mut() }, + bucket, + })); + } + } self.reserve(1)?; // Safety: cannot be static after reserving space Ok(unsafe { self.header_mut().entry_or_clone(key) }) @@ -1277,6 +1363,68 @@ mod tests { use super::*; use crate::convert::TryCollect; + #[test] + fn unique_entries_preserve_colliding_keys() { + let mut entries = indexmap::IndexMap::default(); + let cap = hash_capacity(20); + // Force collisions across the end of the table to exercise wrapping. + for i in 0.. { + let key = IString::intern(&format!("collision_field_{i}")); + if hash_bucket(&key, cap) >= cap - 2 { + entries.insert(key, IValue::NULL); + } + if entries.len() == 20 { + break; + } + } + let keys: Vec<_> = entries.keys().cloned().collect(); + entries.insert(keys[0].clone(), true.into()); + let mut object = IObject::from_unique_entries(&mut entries).unwrap(); + assert!(entries.is_empty()); + assert_eq!(object.capacity(), 20); + assert_eq!(object.keys().cloned().collect::>(), keys); + for (i, key) in keys.iter().enumerate() { + let expected = if i == 0 { true.into() } else { IValue::NULL }; + assert_eq!(object.get(key.as_str()), Some(&expected)); + } + for key in keys { + assert!(object.remove(key.as_str()).is_some()); + } + assert!(object.is_empty()); + object.insert("another_long_field", IValue::NULL).unwrap(); + assert_eq!(object.get("another_long_field"), Some(&IValue::NULL)); + } + + #[test] + fn unique_inline_entries_move_and_grow() { + // Heap keys exercise string allocation and ownership under Miri. + for len in 0..=crate::de::OBJECT_BUFFER_INLINE_CAPACITY { + let entries = (0..len) + .map(|i| { + ( + IString::intern(&format!("long_field_name_{i}")), + IValue::NULL, + ) + }) + .collect(); + let mut object = IObject::from_unique_inline_entries(entries).unwrap(); + assert_eq!(object.len() as usize, len); + assert_eq!(object.capacity() as usize, len); + for i in len..17 { + object + .insert(format!("long_field_name_{i}"), IValue::NULL) + .unwrap(); + } + assert_eq!(object.len(), 17); + for i in 0..17 { + assert_eq!( + object.get(format!("long_field_name_{i}").as_str()), + Some(&IValue::NULL) + ); + } + } + } + #[mockalloc::test] fn can_create() { let x = IObject::new(); diff --git a/tests/precount.rs b/tests/precount.rs new file mode 100644 index 00000000..0051f7f1 --- /dev/null +++ b/tests/precount.rs @@ -0,0 +1,268 @@ +use ijson::{array::ArraySliceRef, FPHAConfig, FloatType, IValue, IValueDeserSeed}; +use serde::de::DeserializeSeed; + +fn parse(s: &str, compact: bool) -> Result { + let mut de = serde_json::Deserializer::from_str(s); + let value = if compact { + IValueDeserSeed::new(None).deserialize_compact_objects(&mut de)? + } else { + IValueDeserSeed::new(None).deserialize(&mut de)? + }; + de.end()?; + Ok(value) +} + +#[test] +fn replacing_compact_object_fields_does_not_grow_capacity() { + for count in [0, 1, 5, 8, 9, 16, 17, 100] { + let fields = (0..count) + .map(|i| format!("\"field{i}\":{i}")) + .collect::>() + .join(","); + for borrowed_key in [false, true] { + let mut value = parse(&format!("{{{fields}}}"), true).unwrap(); + let object = value.as_object_mut().unwrap(); + assert_eq!(object.capacity(), count); + for i in 0..count { + let key = ijson::IString::from(format!("field{i}").as_str()); + let entry = if borrowed_key { + object.entry_or_clone(&key) + } else { + object.entry(key) + }; + entry + .unwrap() + .and_modify(|value| *value = IValue::from(i + 1000)) + .or_insert(IValue::NULL); + assert_eq!(object.len(), count); + assert_eq!(object.capacity(), count); + } + + // A genuinely new field must still reserve space, including on {}. + let key = ijson::IString::from("new_field"); + let entry = if borrowed_key { + object.entry_or_clone(&key) + } else { + object.entry(key) + }; + entry.unwrap().or_insert(IValue::from(true)); + assert_eq!(object.len(), count + 1); + assert!(object.capacity() > count); + assert_eq!(object.get("new_field"), Some(&IValue::from(true))); + for i in 0..count { + assert_eq!( + object.get(format!("field{i}").as_str()), + Some(&IValue::from(i + 1000)) + ); + } + } + } +} + +#[test] +fn buffered_objects_preserve_values_errors_and_capacity() { + for input in [ + r#"{"a":1,"b":2,"c":3,"d":4,"e":5}"#, + r#"{"x":[{}, {"nested":{"a":null,"b":true}}],"str":"{[colon:]}"}"#, + r#"{"escaped\"key":"slash\\\" colon: } {","unicode":"שלום","n":-1.25e4}"#, + r#"[{"a":1,"a":{"b":2}},{"z":[]},[1,2,3]]"#, + "{}", + "[]", + "null", + "123", + "\"hello\"", + ] { + let compact = parse(input, true).unwrap(); + let ordinary = parse(input, false).unwrap(); + assert_eq!( + serde_json::to_value(&compact).unwrap(), + serde_json::to_value(&ordinary).unwrap() + ); + } + let input = r#"{"a":1,"b":2,"c":3,"d":4,"e":5}"#; + let mut value = parse(input, true).unwrap(); + assert_eq!(value.as_object().unwrap().capacity(), 5); + assert_eq!( + parse(input, false).unwrap().as_object().unwrap().capacity(), + 5 + ); + value + .as_object_mut() + .unwrap() + .insert("f", IValue::NULL) + .unwrap(); + assert_eq!(value.as_object().unwrap().len(), 6); + let nested = parse(r#"{"x":[{"a":1,"b":2,"c":3}] }"#, true).unwrap(); + assert_eq!(nested.as_object().unwrap().capacity(), 1); + assert_eq!(nested["x"][0].as_object().unwrap().capacity(), 3); + for input in [ + "", + "{", + "[", + "{\"a\":}", + "{\"a\":1,}", + "[1,]", + "{} {}", + "{\"a\":\"\\q\"}", + "{\"a\":1]", + "\"unterminated", + ] { + assert!(parse(input, true).is_err(), "{input}"); + assert!(parse(input, false).is_err(), "{input}"); + } + let deep = format!("{}0{}", "[".repeat(150), "]".repeat(150)); + assert!(parse(&deep, true).is_err()); +} + +#[test] +fn repeated_keys_use_exact_storage_and_preserve_last_value() { + // Escaped names compare equal after decoding. Replaced values may themselves + // contain objects, whose buffers must not interfere with the next field. + let input = r#"{"a":{"x":1},"\u0061":{"y":2,"z":3},"next":{"p":4}}"#; + let value = parse(input, true).unwrap(); + assert_eq!(value, parse(input, false).unwrap()); + assert_eq!( + serde_json::to_string(&value).unwrap(), + serde_json::to_string(&parse(input, false).unwrap()).unwrap() + ); + assert_eq!(value.as_object().unwrap().capacity(), 2); + assert_eq!(value["a"].as_object().unwrap().capacity(), 2); + assert_eq!(value["next"].as_object().unwrap().capacity(), 1); + + let input = format!("{{{}\"same\":2}}", "\"same\":1,".repeat(100_000)); + let value = parse(&input, true).unwrap(); + assert_eq!(value.as_object().unwrap().capacity(), 1); + assert_eq!(value, parse(r#"{"same":2}"#, false).unwrap()); +} + +#[test] +fn buffered_objects_preserve_typed_arrays() { + let input = r#"{"values":[0.5,1.0,1.5],"nested":[{"x":1,"y":2}]}"#; + for fp_type in [ + FloatType::F16, + FloatType::BF16, + FloatType::F32, + FloatType::F64, + ] { + let seed = || IValueDeserSeed::new(Some(FPHAConfig::new_with_type(fp_type))); + let mut de = serde_json::Deserializer::from_str(input); + let value = seed().deserialize_compact_objects(&mut de).unwrap(); + de.end().unwrap(); + let ordinary = seed() + .deserialize(&mut serde_json::Deserializer::from_str(input)) + .unwrap(); + assert_eq!(value, ordinary); + assert_eq!(value["nested"][0].as_object().unwrap().capacity(), 2); + let array = value["values"].as_array().unwrap().as_slice(); + assert!(matches!( + (fp_type, array), + (FloatType::F16, ArraySliceRef::F16(_)) + | (FloatType::BF16, ArraySliceRef::BF16(_)) + | (FloatType::F32, ArraySliceRef::F32(_)) + | (FloatType::F64, ArraySliceRef::F64(_)) + )); + } +} + +#[test] +fn buffered_objects_preserve_escaped_strings() { + // Exercise escaped keys and values at varied lengths. + for padding in [0, 1, 15, 16, 31, 32, 63, 64, 65, 256, 4096] { + for slashes in 0..8 { + for suffix in ["", "\"", "\" :{}[] שלום"] { + let text = format!("{}{}{}", "x".repeat(padding), "\\".repeat(slashes), suffix); + let text = serde_json::to_string(&text).unwrap(); + let input = format!("{{{text}:{text},\"next\":{{\"a\":1,\"b\":2}}}}"); + let value = parse(&input, true).unwrap(); + assert_eq!(value, parse(&input, false).unwrap()); + assert_eq!(value.as_object().unwrap().capacity(), 2); + assert_eq!(value["next"].as_object().unwrap().capacity(), 2); + } + } + } + for input in ["\"", "\"abc\\", "{\"key\":\"abc\\", "\"\\q\"", "\"\\u12\""] { + assert!(parse(input, true).is_err(), "{input}"); + assert!(parse(input, false).is_err(), "{input}"); + } +} + +#[test] +fn small_objects_promote_without_changing_order_or_duplicate_values() { + let widths = [0, 1, 8, 9, 12, 16, 17, 20, 31, 32, 33, 8, 16, 17]; + let inputs: Vec<_> = widths + .into_iter() + .map(|n| { + let mut fields: Vec<_> = (0..n).map(|i| format!("\"k{i}\":{i}")).collect(); + if n > 0 { + fields.push(r#""\u006b0":{"replacement":true}"#.to_owned()); + } + format!("{{{}}}", fields.join(",")) + }) + .collect(); + let input = format!("[{}]", inputs.join(",")); + let value = parse(&input, true).unwrap(); + assert_eq!( + serde_json::to_string(&value).unwrap(), + serde_json::to_string(&parse(&input, false).unwrap()).unwrap() + ); + for (i, n) in widths.into_iter().enumerate() { + assert_eq!(value[i].as_object().unwrap().capacity(), n); + } +} + +#[test] +fn exact_small_objects_grow_without_a_premature_hash_table() { + let mut value = parse(r#"{"a":1,"b":2,"c":3,"d":4,"e":5}"#, true).unwrap(); + let object = value.as_object_mut().unwrap(); + for (key, capacity) in [("f", 7), ("g", 7), ("h", 8)] { + object.insert(key, IValue::NULL).unwrap(); + assert_eq!(object.capacity(), capacity); + } + object.insert("i", IValue::NULL).unwrap(); + assert_eq!(object.capacity(), 10); + assert_eq!(object.len(), 9); + assert_eq!(object["a"], IValue::from(1)); + assert!(object["i"].is_null()); +} + +#[test] +fn compact_objects_reuse_large_buffers_and_preserve_errors() { + let fields = (0..33) + .map(|i| format!("\"long_field_{i}\":null")) + .collect::>() + .join(","); + let wide = format!("{{{fields}}}"); + // Replacing a nested value drops its object; its drained buffer can still + // be reused by siblings without retaining any of the old fields. + let input = format!(r#"[{wide},{{"child":{wide},"\u0063hild":{{}},"tail":{wide}}},{wide}]"#); + let value = parse(&input, true).unwrap(); + assert_eq!( + serde_json::to_string(&value).unwrap(), + serde_json::to_string(&parse(&input, false).unwrap()).unwrap() + ); + assert_eq!(value[0].as_object().unwrap().capacity(), 33); + assert_eq!(value[1].as_object().unwrap().capacity(), 2); + assert!(value[1]["child"].as_object().unwrap().is_empty()); + assert_eq!(value[1]["tail"].as_object().unwrap().capacity(), 33); + assert_eq!(value[2].as_object().unwrap().capacity(), 33); + + for suffix in ["", ",}", ",\"bad\":]", r#", "bad":"\uD800"}"#] { + let malformed = format!("[{wide},{{{fields}{suffix}"); + let compact = parse(&malformed, true).unwrap_err(); + let ordinary = parse(&malformed, false).unwrap_err(); + assert_eq!(compact.classify(), ordinary.classify()); + assert_eq!( + (compact.line(), compact.column()), + (ordinary.line(), ordinary.column()) + ); + } + + let input = r#"{"":0,"\u0000":1,"":2,"😀":3,"\ud83d\ude00":4}"#; + let mut de = serde_json::Deserializer::from_reader(input.as_bytes()); + let value = IValueDeserSeed::new(None) + .deserialize_compact_objects(&mut de) + .unwrap(); + de.end().unwrap(); + assert_eq!(value, parse(input, false).unwrap()); + assert_eq!(value.as_object().unwrap().capacity(), 3); +}