diff --git a/.github/actions/build_and_test/action.yml b/.github/actions/build_and_test/action.yml index 4c770c78..864b77ed 100644 --- a/.github/actions/build_and_test/action.yml +++ b/.github/actions/build_and_test/action.yml @@ -26,3 +26,9 @@ runs: command: test args: -- --test-threads=1 + - name: Test (thread_safe) + uses: actions-rs/cargo@v1 + with: + command: test + args: --features thread_safe -- --test-threads=1 + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dced2c88..80f50c14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,7 @@ jobs: - fuzz_json_de - fuzz_cbor_decode - fuzz_cbor_roundtrip + - fuzz_cbor_stream_decode - fuzz_object_ops steps: - name: Checkout repository diff --git a/README.md b/README.md index 1d3415f5..8ecd0db0 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,133 @@ case that is important in the original JSON. Removing from an `IObject` will disrupt the insertion order. +### String streams + +`StreamEncoder` shares strings across the documents of one ordered stream. A +string with at least two owners and at least 8 bytes is +written inline once, wrapped in CBOR tag 6 (a definition), and every later +occurrence is written as tag 7 plus an ID (a reference). IDs are implicit: the +n-th definition of the stream is ID n, so `StreamDecoder` rebuilds the table by +decoding the documents in the order they were encoded. A reference costs 2, 3, 4 +or 6 bytes for IDs below 24, 256, 65,536 and 2^32. Typed numeric arrays keep their +existing CBOR tags, and `encode`/`decode` and the compressed APIs keep their +standalone formats. + +```rust +use ijson::{IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig}; + +let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: 64 << 20, +}); +let first: IValue = serde_json::from_str(r#"{"k":"a string stored in two documents"}"#).unwrap(); +let second: IValue = serde_json::from_str(r#"{"k":"a string stored in two documents"}"#).unwrap(); + +let (mut first_bytes, mut second_bytes) = (Vec::new(), Vec::new()); +encoder.encode_into(&mut first_bytes, &first); // defines the string +encoder.encode_into(&mut second_bytes, &second); // references it + +let mut decoder = StreamDecoder::new(); +assert_eq!(decoder.decode(&first_bytes, true).unwrap(), first); +assert_eq!(decoder.decode(&second_bytes, true).unwrap(), second); +``` + +The encoder recognizes a string by the address of its shared allocation, so it +neither hashes nor copies string bytes and only reads them. The caller must keep +all defined string allocations alive and at the same address until the stream +ends. The encoder holds no owned handles. A freed string whose address another +string of the same length takes over goes unnoticed, and the reader gets the old +text; only a string of another length at a registered address is noticed, which +poisons the encoder: that string and every later one is written inline. The table budget bounds the +encoder's memory: when one more definition +would exceed it, new strings stay inline while earlier ones keep being +referenced. If a document that defined strings is not written, call `poison()`; +every later document is then plain CBOR. `StreamEncoder::next_id` before a +document and `StreamDecoder::len` before reading it must agree, which lets a +framing layer detect skipped, repeated or reordered documents. + +To reuse a full table, call `StreamEncoder::reset_if_needed()` between documents +that may define strings. It clears the table only after an eligible definition +missed the table budget, preserving its allocation. Signal that reset in your +framing and call `StreamDecoder::reset()` before decoding the next document +that introduces definitions. Inline documents can precede that marker; they do +not use the dictionary. Both dictionaries then assign IDs from 0 again. + +With resets, use cumulative `stats().defs` for framing positions rather than the +current table length, and require a reset-bearing record to introduce a definition. +Otherwise, skipping a reset-only record could silently reuse old IDs. Statistics +stay cumulative across the entire stream. Reader resets release old dictionary handles while loaded +values keep their own. Neither reset API clears a poisoned or failed state. + +### Standalone documents + +A fresh `StreamEncoder` and a fresh `StreamDecoder` per document make that document stand +alone: ids start at 0, nothing else is needed to read it, and strings that repeat inside the +document are defined once. A string only counts as repeated here if it has two or more owners, +so a document that holds a shared string once also gets a definition that nothing references. +Write the plain bytes when the definitions do not pay: each definition costs 1 byte and +`stats().saved_bytes` is what the references saved, so keep the stream form only when +`saved_bytes > defs`. + +```rust +use ijson::{IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig}; + +let value: IValue = serde_json::from_str( + r#"[{"a repeated object key": 1}, {"a repeated object key": 2}]"#, +).unwrap(); +let mut bytes = Vec::new(); +let mut encoder = StreamEncoder::new(StreamEncoderConfig { table_budget_bytes: 1 << 20 }); +encoder.encode_into(&mut bytes, &value); +let stats = encoder.stats(); +assert!(stats.saved_bytes > stats.defs); +assert_eq!(bytes.len() as u64, ijson::encode(&value).len() as u64 - stats.saved_bytes + stats.defs); +assert_eq!(StreamDecoder::new().decode(&bytes, true).unwrap(), value); +``` + +#### Other value representations + +`StreamEncoder` accepts any implementation of `StringDictionary`, defaulting +to `AddressDictionary`. `ContentDictionary` matches ordinary `&str` values by +content and owns copies, so sources can be freed after each document. Its budget +includes copied UTF-8 bytes, table storage and simultaneous growth allocations. +It admits all strings of at least 8 bytes, including strings that occur only once. + +```rust +use ijson::{ContentDictionary, StreamDecoder, StreamEncoder, StreamEncoderConfig}; + +let mut encoder = StreamEncoder::with_dictionary( + StreamEncoderConfig { table_budget_bytes: 4096 }, + ContentDictionary::default(), +); +let mut bytes = Vec::new(); +encoder.encode_with(&mut bytes, |strings, out| { + ciborium::into_writer(&strings.string("text from another value representation"), out) +}).unwrap(); +assert_eq!( + StreamDecoder::new().decode(&bytes, true).unwrap().as_string().unwrap().as_str(), + "text from another value representation", +); +``` + +A custom Serde serializer wraps each participating string value and object key +with `strings.string(...)`. Wrapping does not register a definition; serialization +does. Serialize one complete CBOR document directly in wire order, without +pre-serializing, discarding or reordering encoded pieces. Propagate errors: +`encode_with` discards appended bytes and poisons the encoder on failure. +Unwrapped strings remain ordinary inline CBOR. + +A custom dictionary supplies identity, eligibility, storage and allocation +accounting. Its input is a `StreamString`: text plus an optional interned allocation +identity and owner count. Ordinary string slices have no allocation identity. +A dictionary that borrows its strings, like `AddressDictionary`, needs the caller to keep the +source storage alive; one that owns copies, like `ContentDictionary`, does not. +Choose one dictionary for the stream. Pass a fresh dictionary, or ensure retained +allocations already fit the budget. `Box` supports choosing +the implementation at initialization without runtime type checks. + +All dictionaries use the same DEF/REF format and existing `StreamDecoder`. +The decoder reconstructs its own `IString` table from definitions; it does not +need the writer's cache or value representation. + ## Technical details ### `IValue` diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 823dde09..9f2673de 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -42,6 +42,13 @@ test = false doc = false bench = false +[[bin]] +name = "fuzz_cbor_stream_decode" +path = "fuzz_targets/fuzz_cbor_stream_decode.rs" +test = false +doc = false +bench = false + [[bin]] name = "fuzz_object_ops" path = "fuzz_targets/fuzz_object_ops.rs" diff --git a/fuzz/fuzz_targets/fuzz_cbor_stream_decode.rs b/fuzz/fuzz_targets/fuzz_cbor_stream_decode.rs new file mode 100644 index 00000000..a123f868 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_cbor_stream_decode.rs @@ -0,0 +1,46 @@ +#![no_main] + +use arbitrary::Arbitrary; +use ijson::StreamDecoder; +use libfuzzer_sys::fuzz_target; + +// One document of the stream, decoded with or without the nesting limit. +#[derive(Arbitrary, Debug)] +struct Document { + limit_depth: bool, + bytes: Vec, +} + +// A stream document that defines three strings, so that the input can also +// use the references 0 to 2. +fn decoder_with_definitions() -> StreamDecoder { + let mut document = vec![0x83]; + for text in ["aaaaaaaa", "bbbbbbbb", "cccccccc"] { + document.extend_from_slice(&[0xC6, 0x68]); + document.extend_from_slice(text.as_bytes()); + } + let mut decoder = StreamDecoder::new(); + decoder + .decode(&document, true) + .expect("the definitions are a valid stream document"); + decoder +} + +// Reads the documents in order with one decoder, as a loader does: the table +// grows from document to document, and after a document that failed the decoder +// must refuse every other one. +fuzz_target!(|documents: Vec| { + let mut decoder = decoder_with_definitions(); + let mut failed = false; + for document in documents { + let known = decoder.len(); + let result = decoder.decode(&document.bytes, document.limit_depth); + assert!( + !failed || result.is_err(), + "a decoder that failed decoded another document" + ); + failed |= result.is_err(); + assert!(decoder.len() >= known, "the table shrank"); + assert_eq!(decoder.len() as u64, decoder.stats().defs); + } +}); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 50b3f5d4..e9cbc71a 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,3 @@ [toolchain] channel = "1.92" +components = ["rust-analyzer"] diff --git a/src/cbor.rs b/src/cbor.rs index 78c23684..db1e22fb 100644 --- a/src/cbor.rs +++ b/src/cbor.rs @@ -5,13 +5,21 @@ //! //! Use [`encode`] / [`decode`] for raw CBOR, or [`encode_compressed`] / //! [`decode_compressed`] for zstd-compressed CBOR. +//! +//! [`StreamEncoder`] and [`StreamDecoder`] share strings across the documents +//! of one ordered stream: the first use of a string defines it inline and later +//! uses refer to it by an implicit ID. use std::fmt; use bytemuck::Pod; -use ciborium::value::{Integer, Value}; +#[cfg(test)] +use ciborium::value::Integer; +use ciborium::value::Value; use ciborium_ll::tag; use half::{bf16, f16}; +use serde::de::DeserializeSeed; +use serde::Serialize; /// Converts a typed slice to/from little-endian bytes using each type's own /// `to_le_bytes` / `from_le_bytes`, which are correct on every host endianness. @@ -54,6 +62,13 @@ impl_le_bytes!( use crate::array::ArraySliceRef; use crate::{DestructuredRef, IArray, INumber, IObject, IString, IValue}; +mod stream; +pub use stream::{ + AddressDictionary, ContentDictionary, DocUse, StreamDecoder, StreamDecoderStats, StreamEncoder, + StreamEncoderConfig, StreamEncoderStats, StreamString, StreamStrings, StringDictionary, +}; +use stream::{DecoderCell, EncoderCell}; + use tag::{ TYPED_F16_LE as TAG_F16_LE, TYPED_F32_LE as TAG_F32_LE, TYPED_F64_LE as TAG_F64_LE, TYPED_I16_LE as TAG_I16_LE, TYPED_I32_LE as TAG_I32_LE, TYPED_I64_LE as TAG_I64_LE, @@ -64,6 +79,12 @@ use tag::{ /// Private CBOR tag for BF16 arrays (no RFC 8746 standard tag exists for BF16). const TAG_BF16_LE: u64 = 0x10000; +/// CBOR tag that defines a stream string: it wraps the text, which takes the next ID. +pub(crate) const TAG_DEF: u64 = 6; + +/// CBOR tag that references an earlier stream definition by its ID. +pub(crate) const TAG_REF: u64 = 7; + const MAX_DEPTH: u32 = 128; /// Error returned when CBOR decoding fails. @@ -83,6 +104,8 @@ pub enum CborDecodeError { CastError, /// Zstd decompression failed. DecompressError, + /// A stream string reference had an invalid payload or an out-of-range ID. + InvalidStringReference, } impl fmt::Display for CborDecodeError { @@ -95,6 +118,7 @@ impl fmt::Display for CborDecodeError { CborDecodeError::DepthLimitExceeded => write!(f, "nesting depth limit exceeded"), CborDecodeError::CastError => write!(f, "failed to cast byte slice"), CborDecodeError::DecompressError => write!(f, "zstd decompression failed"), + CborDecodeError::InvalidStringReference => write!(f, "invalid string reference"), } } } @@ -106,12 +130,20 @@ impl std::error::Error for CborDecodeError {} /// Encodes an [`IValue`] tree into CBOR bytes, preserving typed array tags via /// RFC 8746. pub fn encode(value: &IValue) -> Vec { - let cbor = ivalue_to_cbor(value); let mut out = Vec::new(); - ciborium::into_writer(&cbor, &mut out).expect("write to Vec never fails"); + encode_into(value, &mut out); out } +/// Appends CBOR bytes to an existing buffer, preserving its prefix and typed array tags. +pub fn encode_into(value: &IValue, out: &mut Vec) { + let cbor = EncodedValue { + value, + strings: Inline, + }; + ciborium::into_writer(&cbor, out).expect("write to Vec never fails"); +} + /// Encodes an [`IValue`] tree as CBOR and then compresses it with zstd (level 3). /// /// Use [`decode_compressed`] to decode the output. @@ -122,17 +154,24 @@ pub fn encode_compressed(value: &IValue) -> Vec { .expect("zstd compress") } +#[cfg(test)] fn ivalue_to_cbor(value: &IValue) -> Value { match value.destructure_ref() { DestructuredRef::Null => Value::Null, DestructuredRef::Bool(b) => Value::Bool(b), DestructuredRef::Number(n) => number_to_cbor(n), - DestructuredRef::String(s) => Value::Text(s.as_str().to_owned()), + DestructuredRef::String(s) => string_to_cbor(s), DestructuredRef::Array(a) => array_to_cbor(a), DestructuredRef::Object(o) => object_to_cbor(o), } } +#[cfg(test)] +fn string_to_cbor(value: &IString) -> Value { + Value::Text(value.as_str().to_owned()) +} + +#[cfg(test)] fn number_to_cbor(n: &INumber) -> Value { if n.has_decimal_point() { Value::Float(n.to_f64().unwrap()) @@ -143,6 +182,7 @@ fn number_to_cbor(n: &INumber) -> Value { } } +#[cfg(test)] fn array_to_cbor(a: &IArray) -> Value { match a.as_slice() { ArraySliceRef::Heterogeneous(s) => Value::Array(s.iter().map(ivalue_to_cbor).collect()), @@ -161,18 +201,557 @@ fn array_to_cbor(a: &IArray) -> Value { } } +#[cfg(test)] fn object_to_cbor(o: &IObject) -> Value { Value::Map( o.iter() - .map(|(k, v)| (Value::Text(k.as_str().to_owned()), ivalue_to_cbor(v))) + .map(|(k, v)| (string_to_cbor(k), ivalue_to_cbor(v))) .collect(), ) } +#[cfg(test)] fn typed_le_tag(tag: u64, s: &[T]) -> Value { Value::Tag(tag, Box::new(Value::Bytes(T::slice_to_le_bytes(s)))) } +/// How the strings of a document are written. It is a type parameter, not a value checked +/// per string, so that a document encoded without a stream runs code with no trace of the +/// stream: a per-string branch alone made the plain encoder about 40% slower on an array of +/// eight million small values. +trait WriteStrings: Copy { + fn write_string( + self, + string: &IString, + serializer: S, + ) -> Result; +} + +/// Every string is text. +#[derive(Clone, Copy)] +struct Inline; + +impl WriteStrings for Inline { + #[inline(always)] + fn write_string( + self, + string: &IString, + serializer: S, + ) -> Result { + serializer.serialize_str(string.as_str()) + } +} + +/// A string is text, a definition or a reference, as the stream encoder decides. +impl WriteStrings for &EncoderCell<'_, D> { + fn write_string( + self, + string: &IString, + serializer: S, + ) -> Result { + stream::serialize_string(self, string, serializer) + } +} + +struct EncodedValue<'a, W> { + value: &'a IValue, + strings: W, +} + +struct EncodedString<'a, W>(&'a IString, W); + +impl serde::Serialize for EncodedString<'_, W> { + fn serialize(&self, serializer: S) -> Result { + self.1.write_string(self.0, serializer) + } +} + +struct EncodedBytes<'a>(&'a [u8]); + +impl serde::Serialize for EncodedBytes<'_> { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(self.0) + } +} + +fn serialize_typed( + tag: u64, + values: &[T], + serializer: S, +) -> Result { + let converted; + let bytes = if cfg!(target_endian = "little") { + bytemuck::cast_slice(values) + } else { + converted = T::slice_to_le_bytes(values); + &converted + }; + ciborium::tag::RequireAny(tag, EncodedBytes(bytes)).serialize(serializer) +} + +impl serde::Serialize for EncodedValue<'_, W> { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::{SerializeMap, SerializeSeq}; + match self.value.destructure_ref() { + DestructuredRef::Null => serializer.serialize_unit(), + DestructuredRef::Bool(value) => serializer.serialize_bool(value), + DestructuredRef::Number(value) => { + if value.has_decimal_point() { + serializer.serialize_f64(value.to_f64().unwrap()) + } else if let Some(value) = value.to_i64() { + serializer.serialize_i64(value) + } else { + serializer.serialize_u64(value.to_u64().unwrap()) + } + } + DestructuredRef::String(value) => { + EncodedString(value, self.strings).serialize(serializer) + } + DestructuredRef::Object(value) => { + let mut map = serializer.serialize_map(Some(value.len() as usize))?; + for (key, value) in value.iter() { + map.serialize_entry( + &EncodedString(key, self.strings), + &EncodedValue { + value, + strings: self.strings, + }, + )?; + } + map.end() + } + DestructuredRef::Array(value) => match value.as_slice() { + ArraySliceRef::Heterogeneous(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&EncodedValue { + value, + strings: self.strings, + })?; + } + seq.end() + } + ArraySliceRef::I8(values) => serialize_typed(TAG_I8, values, serializer), + ArraySliceRef::U8(values) => serialize_typed(TAG_U8, values, serializer), + ArraySliceRef::I16(values) => serialize_typed(TAG_I16_LE, values, serializer), + ArraySliceRef::U16(values) => serialize_typed(TAG_U16_LE, values, serializer), + ArraySliceRef::F16(values) => serialize_typed(TAG_F16_LE, values, serializer), + ArraySliceRef::BF16(values) => serialize_typed(TAG_BF16_LE, values, serializer), + ArraySliceRef::I32(values) => serialize_typed(TAG_I32_LE, values, serializer), + ArraySliceRef::U32(values) => serialize_typed(TAG_U32_LE, values, serializer), + ArraySliceRef::F32(values) => serialize_typed(TAG_F32_LE, values, serializer), + ArraySliceRef::I64(values) => serialize_typed(TAG_I64_LE, values, serializer), + ArraySliceRef::U64(values) => serialize_typed(TAG_U64_LE, values, serializer), + ArraySliceRef::F64(values) => serialize_typed(TAG_F64_LE, values, serializer), + }, + } + } +} + +/// How the tagged string references of a document resolve. +#[derive(Clone, Copy)] +enum StringRefs<'a> { + /// No tag is a string reference. + None, + /// Definitions and references of a stream use tags 6 and 7. + Stream(&'a DecoderCell<'a>), +} + +struct DecodeContext<'a> { + refs: StringRefs<'a>, + limit_depth: bool, + error: std::cell::Cell>, +} + +impl DecodeContext<'_> { + fn fail(&self, error: CborDecodeError) -> E { + self.error.set(Some(error.clone())); + E::custom(error) + } +} + +#[derive(Clone, Copy)] +struct ValueSeed<'a, 'd> { + context: &'a DecodeContext<'d>, + depth: u32, +} + +impl ValueSeed<'_, '_> { + fn convert(self, value: Value) -> Result { + leaf_to_ivalue(value).map_err(|error| self.context.fail(error)) + } + + fn child(self) -> Self { + Self { + depth: self.depth + 1, + ..self + } + } +} + +impl<'de> serde::de::DeserializeSeed<'de> for ValueSeed<'_, '_> { + type Value = IValue; + + fn deserialize>(self, deserializer: D) -> Result { + if self.context.limit_depth && self.depth >= MAX_DEPTH { + return Err(self.context.fail(CborDecodeError::DepthLimitExceeded)); + } + deserializer.deserialize_any(self) + } +} + +impl<'de> serde::de::Visitor<'de> for ValueSeed<'_, '_> { + type Value = IValue; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a JSON value with CBOR typed arrays or stream string references") + } + + fn visit_unit(self) -> Result { + Ok(IValue::NULL) + } + fn visit_none(self) -> Result { + Ok(IValue::NULL) + } + fn visit_bool(self, value: bool) -> Result { + Ok(value.into()) + } + fn visit_i64(self, value: i64) -> Result { + self.convert(value.into()) + } + fn visit_u64(self, value: u64) -> Result { + self.convert(value.into()) + } + fn visit_i128(self, value: i128) -> Result { + self.convert(value.into()) + } + fn visit_u128(self, value: u128) -> Result { + self.convert(value.into()) + } + fn visit_f64(self, value: f64) -> Result { + self.convert(value.into()) + } + fn visit_str(self, value: &str) -> Result { + Ok(IString::from(value).into()) + } + fn visit_bytes(self, _: &[u8]) -> Result { + Err(self.context.fail(CborDecodeError::InvalidValue)) + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut array = IArray::with_capacity(access.size_hint().unwrap_or(0).min(1024)) + .map_err(|_| self.context.fail(CborDecodeError::AllocError))?; + while let Some(value) = access.next_element_seed(self.child())? { + array + .push(value) + .map_err(|_| self.context.fail(CborDecodeError::AllocError))?; + } + Ok(array.into()) + } + + fn visit_map>(self, mut access: A) -> Result { + let mut object = IObject::with_capacity(access.size_hint().unwrap_or(0).min(1024)) + .map_err(|_| self.context.fail(CborDecodeError::AllocError))?; + while let Some(key) = access.next_key_seed(KeySeed { + context: self.context, + })? { + let value = access.next_value_seed(self.child())?; + object + .insert(key, value) + .map_err(|_| self.context.fail(CborDecodeError::AllocError))?; + } + Ok(object.into()) + } + + fn visit_enum>(self, access: A) -> Result { + use serde::de::VariantAccess; + // Keep ciborium's tag and bignum normalization; valid tags hold one byte string or ID. + // Stream definitions hold one text string. + // The variant name is always "@@TAGGED@@"; skip it without allocating a String. + let (_, data): (serde::de::IgnoredAny, _) = access.variant()?; + // The payload is a 2-tuple: element 1 is the tag number, element 2 its content. + data.tuple_variant(2, TaggedValue { seed: self }) + } +} + +/// Reads a stream string tag (6 or 7) directly from the CBOR reader, without a `Value`. +/// Returns `None`, with nothing consumed after the tag, when `tag` is not a stream string tag. +fn stream_string<'c, 'de, A: serde::de::SeqAccess<'de>>( + context: &DecodeContext<'c>, + tag: u64, + access: &mut A, +) -> Result, A::Error> { + // Without a stream decoder no tag is a string tag. + let StringRefs::Stream(cell) = context.refs else { + return Ok(None); + }; + let missing = || ::custom("expected val"); + let result = match tag { + // Definition: the text is interned and takes the next ID. + TAG_DEF => access + .next_element_seed(DefText { cell })? + .ok_or_else(missing)?, + // Reference: an ID below the number of definitions, resolved to the shared handle. + TAG_REF => { + let id = access.next_element_seed(RefId)?.ok_or_else(missing)?; + id.and_then(|id| cell.borrow_mut().reference_id(id)) + } + _ => return Ok(None), + }; + // A refused definition or reference fails the whole decode with its own error kind. + result.map(Some).map_err(|error| context.fail(error)) +} + +struct TaggedValue<'a, 'd> { + seed: ValueSeed<'a, 'd>, +} + +impl<'de> serde::de::Visitor<'de> for TaggedValue<'_, '_> { + type Value = IValue; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a valid CBOR item") + } + + fn visit_seq>(self, mut access: A) -> Result { + // Element 1 of the tag pair: the tag number. + let tag: u64 = access + .next_element()? + .ok_or_else(|| ::custom("expected tag"))?; + // Tags 6 and 7 are read here, with no intermediate `Value`. + if let Some(string) = stream_string(self.seed.context, tag, &mut access)? { + return Ok(string.into()); + } + // Typed arrays keep ciborium's value path. + let inner: Value = access + .next_element()? + .ok_or_else(|| ::custom("expected val"))?; + self.seed.convert(Value::Tag(tag, Box::new(inner))) + } +} + +/// Implements the scalar `Visitor` methods a seed refuses, each as `self.reject()`. +macro_rules! reject_scalars { + ($($name:ident: $ty:ty),* $(,)?) => {$( + fn $name(self, _: $ty) -> Result { + self.reject() + } + )*}; +} + +/// Implements the argument-less `Visitor` methods a seed refuses. +macro_rules! reject_empty { + ($($name:ident),* $(,)?) => {$( + fn $name(self) -> Result { + self.reject() + } + )*}; +} + +/// Implements the `seq`, `map` and `enum` `Visitor` methods a seed refuses. +macro_rules! reject_compound { + () => {}; + (seq $($rest:tt)*) => { + fn visit_seq>(self, _: A) -> Result { + self.reject() + } + reject_compound!($($rest)*); + }; + (map $($rest:tt)*) => { + fn visit_map>(self, _: A) -> Result { + self.reject() + } + reject_compound!($($rest)*); + }; + (enum $($rest:tt)*) => { + fn visit_enum>(self, _: A) -> Result { + self.reject() + } + reject_compound!($($rest)*); + }; +} + +/// The text of a definition, borrowed from the reader and interned once. +/// Non-text content yields `Err(InvalidValue)` inside `Ok`, so the caller gets a +/// `CborDecodeError` and not a serde error. +struct DefText<'c> { + cell: &'c DecoderCell<'c>, +} + +impl DefText<'_> { + fn reject(self) -> Result, E> { + Ok(Err(CborDecodeError::InvalidValue)) + } +} + +impl<'de> serde::de::DeserializeSeed<'de> for DefText<'_> { + type Value = Result; + + fn deserialize>( + self, + deserializer: D, + ) -> Result { + deserializer.deserialize_any(self) + } +} + +impl<'de> serde::de::Visitor<'de> for DefText<'_> { + type Value = Result; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a stream definition text") + } + + fn visit_str(self, text: &str) -> Result { + Ok(self.cell.borrow_mut().define_str(text)) + } + + reject_scalars!( + visit_u64: u64, + visit_i64: i64, + visit_i128: i128, + visit_u128: u128, + visit_f64: f64, + visit_bool: bool, + visit_bytes: &[u8], + ); + reject_empty!(visit_unit, visit_none); + reject_compound!(seq map enum); +} + +/// The ID of a reference. Anything but an unsigned integer is `InvalidStringReference`. +struct RefId; + +impl RefId { + fn reject(self) -> Result, E> { + Ok(Err(CborDecodeError::InvalidStringReference)) + } +} + +impl<'de> serde::de::DeserializeSeed<'de> for RefId { + type Value = Result; + + fn deserialize>( + self, + deserializer: D, + ) -> Result { + deserializer.deserialize_any(self) + } +} + +impl<'de> serde::de::Visitor<'de> for RefId { + type Value = Result; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a stream reference id") + } + + fn visit_u64(self, id: u64) -> Result { + Ok(usize::try_from(id).map_err(|_| CborDecodeError::InvalidStringReference)) + } + + fn visit_i128(self, id: i128) -> Result { + Ok(usize::try_from(id).map_err(|_| CborDecodeError::InvalidStringReference)) + } + fn visit_u128(self, id: u128) -> Result { + Ok(usize::try_from(id).map_err(|_| CborDecodeError::InvalidStringReference)) + } + + reject_scalars!( + visit_i64: i64, + visit_f64: f64, + visit_bool: bool, + visit_str: &str, + visit_bytes: &[u8], + ); + reject_empty!(visit_unit, visit_none); + reject_compound!(seq map enum); +} + +/// An object key: text, or a stream string tag. Inline text is interned directly from the reader. +struct KeySeed<'a, 'd> { + context: &'a DecodeContext<'d>, +} + +impl KeySeed<'_, '_> { + fn reject(self) -> Result { + Err(self.context.fail(CborDecodeError::InvalidValue)) + } +} + +impl<'de> serde::de::DeserializeSeed<'de> for KeySeed<'_, '_> { + type Value = IString; + + fn deserialize>( + self, + deserializer: D, + ) -> Result { + deserializer.deserialize_any(self) + } +} + +impl<'de> serde::de::Visitor<'de> for KeySeed<'_, '_> { + type Value = IString; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an object key") + } + + fn visit_str(self, key: &str) -> Result { + Ok(IString::from(key)) + } + + fn visit_enum>(self, access: A) -> Result { + use serde::de::VariantAccess; + // A tagged key arrives as the same ("@@TAGGED@@", (tag, content)) enum as a value: + // skip the name, then read the 2-tuple with `TaggedKey`. A plain array reaches + // `visit_seq` of this visitor instead, which refuses it. + let (_, data): (serde::de::IgnoredAny, _) = access.variant()?; + data.tuple_variant( + 2, + TaggedKey { + context: self.context, + }, + ) + } + + reject_scalars!( + visit_u64: u64, + visit_i64: i64, + visit_i128: i128, + visit_u128: u128, + visit_f64: f64, + visit_bool: bool, + visit_bytes: &[u8], + ); + reject_empty!(visit_unit, visit_none); + reject_compound!(seq map); +} + +/// The `(tag, content)` pair of a tagged object key. +struct TaggedKey<'a, 'd> { + context: &'a DecodeContext<'d>, +} + +impl<'de> serde::de::Visitor<'de> for TaggedKey<'_, '_> { + type Value = IString; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a tagged object key") + } + + fn visit_seq>(self, mut access: A) -> Result { + let tag: u64 = access + .next_element()? + .ok_or_else(|| ::custom("expected tag"))?; + // Only stream string tags can be keys; any other tag is invalid. + match stream_string(self.context, tag, &mut access)? { + Some(string) => Ok(string), + None => Err(self.context.fail(CborDecodeError::InvalidValue)), + } + } +} + // ── Decode ──────────────────────────────────────────────────────────────────── /// Decodes an [`IValue`] tree from CBOR bytes produced by [`encode`]. @@ -182,17 +761,40 @@ pub fn decode(bytes: &[u8]) -> Result { /// Decodes an [`IValue`] tree, optionally enforcing the standard nesting limit. pub fn decode_with_depth_limit(bytes: &[u8], limit_depth: bool) -> Result { + decode_one_value(&mut &*bytes, limit_depth, StringRefs::None) +} + +/// Decodes the first value of `bytes` and leaves the bytes after it in `bytes`. +fn decode_one_value( + bytes: &mut &[u8], + limit_depth: bool, + refs: StringRefs<'_>, +) -> Result { let recursion_limit = if limit_depth { MAX_DEPTH as usize } else { usize::MAX }; - let cbor: Value = ciborium::de::from_reader_with_recursion_limit(bytes, recursion_limit) - .map_err(|err| match err { - ciborium::de::Error::RecursionLimitExceeded => CborDecodeError::DepthLimitExceeded, - _ => CborDecodeError::DecodeError, - })?; - cbor_to_ivalue(cbor, 0, limit_depth) + let mut scratch = [0u8; 4096]; + let mut deserializer = ciborium::de::deserializer_from_reader_with_buffer_and_recursion_limit( + bytes, + &mut scratch, + recursion_limit, + ); + let context = DecodeContext { + refs, + limit_depth, + error: std::cell::Cell::new(None), + }; + ValueSeed { + context: &context, + depth: 0, + } + .deserialize(&mut deserializer) + .map_err(|error| match error { + ciborium::de::Error::RecursionLimitExceeded => CborDecodeError::DepthLimitExceeded, + _ => context.error.take().unwrap_or(CborDecodeError::DecodeError), + }) } /// Decodes an [`IValue`] tree from bytes produced by [`encode_compressed`]. @@ -201,13 +803,10 @@ pub fn decode_compressed(bytes: &[u8]) -> Result { decode(&raw) } -fn cbor_to_ivalue(val: Value, depth: u32, limit_depth: bool) -> Result { - if limit_depth && depth >= MAX_DEPTH { - return Err(CborDecodeError::DepthLimitExceeded); - } +/// Converts what the visitor leaves to ciborium: numbers (ciborium has already turned bignums into +/// integers) and tags. The nesting limit was checked before the visitor ran. +fn leaf_to_ivalue(val: Value) -> Result { match val { - Value::Null => Ok(IValue::NULL), - Value::Bool(b) => Ok(b.into()), Value::Float(f) => Ok(INumber::try_from(f).map(Into::into).unwrap_or(IValue::NULL)), Value::Integer(i) => { if let Ok(v) = i64::try_from(i.clone()) { @@ -218,32 +817,7 @@ fn cbor_to_ivalue(val: Value, depth: u32, limit_depth: bool) -> Result Ok(IString::from(s.as_str()).into()), - Value::Array(arr) => { - let hint = arr.len().min(1024); - let mut out = IArray::with_capacity(hint).map_err(|_| CborDecodeError::AllocError)?; - for v in arr { - let iv = cbor_to_ivalue(v, depth + 1, limit_depth)?; - out.push(iv).map_err(|_| CborDecodeError::AllocError)?; - } - Ok(out.into()) - } - Value::Map(entries) => { - let mut obj = - IObject::with_capacity(entries.len()).map_err(|_| CborDecodeError::AllocError)?; - for (k, v) in entries { - let key = match k { - Value::Text(s) => s, - _ => return Err(CborDecodeError::InvalidValue), - }; - let val = cbor_to_ivalue(v, depth + 1, limit_depth)?; - obj.insert(&key, val) - .map_err(|_| CborDecodeError::AllocError)?; - } - Ok(obj.into()) - } Value::Tag(tag, inner) => decode_typed_array(tag, *inner), - Value::Bytes(_) => Err(CborDecodeError::InvalidValue), _ => Err(CborDecodeError::InvalidValue), } } @@ -289,6 +863,94 @@ mod tests { use crate::IValueDeserSeed; use serde::de::DeserializeSeed; + #[test] + fn plain_encoding_appends_without_changing_prefix() { + let value = crate::IValue::from("hello"); + let mut out = vec![0, 42]; + super::encode_into(&value, &mut out); + assert_eq!(out, [0, 42, 0x65, b'h', b'e', b'l', b'l', b'o']); + } + + #[test] + fn compatibility_encoder_matches_tree_bytes() { + let mut cases: Vec = vec![ + IValue::NULL, + true.into(), + false.into(), + i64::MIN.into(), + u64::MAX.into(), + (-0.0f64).into(), + 1.5f64.into(), + f64::MIN_POSITIVE.into(), + f64::MAX.into(), + IString::from("shared value").into(), + serde_json::from_str(r#"{"shared field":["shared value",null,42,{},[],"literal"]}"#) + .unwrap(), + ]; + macro_rules! typed { + ($($values:expr),+ $(,)?) => { $(cases.push(IArray::try_from($values).unwrap().into());)+ }; + } + typed!( + vec![i8::MIN, i8::MAX], + vec![0u8, u8::MAX], + vec![i16::MIN, i16::MAX], + vec![0u16, u16::MAX], + vec![i32::MIN, i32::MAX], + vec![0u32, u32::MAX], + vec![i64::MIN, i64::MAX], + vec![0u64, u64::MAX], + vec![f16::from_f32(1.5), f16::NEG_INFINITY], + vec![bf16::from_f32(1.5), bf16::NAN], + vec![1.5f32, f32::INFINITY], + vec![1.5f64, f64::NAN] + ); + for value in cases { + let mut expected = Vec::new(); + ciborium::into_writer(&ivalue_to_cbor(&value), &mut expected).unwrap(); + let actual = encode(&value); + assert_eq!(actual, expected); + let decoded = decode(&actual).unwrap(); + assert_eq!(encode(&decoded), expected); + } + } + + #[test] + fn compatibility_decoder_matches_legacy_inputs() { + let json = |text: &str| serde_json::from_str::(text).unwrap(); + let cases: Vec<(Vec, IValue)> = vec![ + (vec![0x9f, 0xf6, 0xf5, 0xff], json("[null,true]")), + (vec![0x7f, 0x61, b'a', 0x61, b'b', 0xff], json("\"ab\"")), + ( + vec![0xbf, 0x61, b'a', 1, 0x61, b'a', 2, 0xff], + json("{\"a\":2}"), + ), + ( + vec![0xd8, TAG_U16_LE as u8, 0x5f, 0x41, 1, 0x41, 0, 0xff], + IArray::try_from(vec![1u16]).unwrap().into(), + ), + (vec![0xc2, 0x41, 0], json("0")), + (vec![0xc3, 0x41, 0], json("-1")), + (vec![0xfb, 0x7f, 0xf0, 0, 0, 0, 0, 0, 0], IValue::NULL), + (vec![0xf6, 0xf5], IValue::NULL), + ]; + for (bytes, expected) in cases { + for limit in [true, false] { + assert_eq!( + decode_with_depth_limit(&bytes, limit), + Ok(expected.clone()), + "{bytes:?}" + ); + } + } + // A tag that no decoder defines is no string tag. + for limit in [true, false] { + assert_eq!( + decode_with_depth_limit(&[0xda, 0, 1, 0, 1, 0xc2, 0x41, 0], limit), + Err(CborDecodeError::InvalidValue) + ); + } + } + fn round_trip(value: &IValue) -> IValue { let bytes = encode(value); decode(&bytes).expect("decode should succeed") diff --git a/src/cbor/stream.rs b/src/cbor/stream.rs new file mode 100644 index 00000000..bc7903c6 --- /dev/null +++ b/src/cbor/stream.rs @@ -0,0 +1,647 @@ +//! First-use string streams: strings shared between the documents of one ordered +//! stream. +//! +//! [`StreamEncoder`] writes a string that several values own inline once, wrapped +//! in tag 6 (a definition), and every later occurrence as tag 7 followed by the ID +//! the definition received (a reference). IDs are implicit: the n-th definition of +//! a stream has ID n, so [`StreamDecoder`] rebuilds the same table by reading the +//! documents in the order they were written. Documents with neither tag are plain +//! CBOR. +//! +//! The encoder recognizes a string by the address of its shared allocation, so it +//! never hashes or copies string bytes to find repeats. An address identifies a +//! string only while that string is alive. Callers must keep every defined +//! allocation alive and at the same address until the stream ends. +//! +//! Callers may explicitly reset both dictionaries between documents. IDs then restart, +//! while statistics and the definition-byte allowance remain cumulative. +//! +//! The address-based behavior above is the default. [`ContentDictionary`] accepts +//! ordinary string slices and owns their contents for serializers outside ijson. + +use std::cell::RefCell; +use std::fmt; + +use ciborium::tag::RequireAny; +use hashbrown::HashMap; +use serde::Serialize; + +use super::{EncodedValue, StringRefs, TAG_DEF, TAG_REF}; +use crate::{CborDecodeError, IString, IValue}; + +mod dictionary; +pub use dictionary::{AddressDictionary, ContentDictionary, StreamString, StringDictionary}; + +/// The encoder as the serializers see it, borrowed for one string at a time. +pub(super) type EncoderCell<'a, D = AddressDictionary> = RefCell<&'a mut StreamEncoder>; + +/// The decoder as the deserializers see it, borrowed for one string at a time. +pub(super) type DecoderCell<'a> = RefCell<&'a mut StreamDecoder>; + +/// Shortest string, in UTF-8 bytes, that is defined and referenced. Strings of up to 7 bytes are +/// stored inside their handles and have no address to recognize them by, so every string that +/// has an address qualifies. +const MIN_DEFINED_LEN: usize = 8; + +/// Settings for a [`StreamEncoder`]. +#[derive(Debug, Clone, Copy)] +pub struct StreamEncoderConfig { + /// Upper bound on the heap the encoder uses, in bytes: its ID table and the + /// old and new allocations while it grows. When one more definition would + /// exceed it no more strings are defined: strings already defined keep being + /// referenced and everything else is written inline. Zero defines nothing. + /// An explicit reset allows new definitions to reuse the table allocation. + /// With [`ContentDictionary`], this also includes owned string bytes. + /// Fixed dictionary-object storage and the output buffer are outside this budget. + pub table_budget_bytes: usize, +} + +/// The definitions and references in one encoded document. A document with +/// neither is plain CBOR. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DocUse { + /// Strings the document defined. Their IDs follow the encoder's + /// [`StreamEncoder::next_id`] from before the document. + pub defs: u64, + /// References to strings defined earlier, in this document or before it. + pub refs: u64, +} + +/// Counters of a [`StreamEncoder`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StreamEncoderStats { + /// Strings defined. + pub defs: u64, + /// UTF-8 bytes of the strings defined. + pub def_bytes: u64, + /// References written. + pub refs: u64, + /// Bytes the references saved compared with writing their text. Each definition adds one + /// byte, its tag, so the stream is `saved_bytes - defs` bytes smaller than plain CBOR. + pub saved_bytes: u64, +} + +/// Bytes of a CBOR head that carries `value`. +fn cbor_head_len(value: u64) -> u64 { + match value { + 0..=23 => 1, + 24..=0xff => 2, + 0x100..=0xffff => 3, + 0x1_0000..=0xffff_ffff => 5, + _ => 9, + } +} + +/// Counters of a [`StreamDecoder`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StreamDecoderStats { + /// Definitions read, equal to the length of the ID table. + /// After resets this remains cumulative and can exceed the current table length. + pub defs: u64, + /// UTF-8 bytes of the strings defined. + pub def_bytes: u64, + /// References resolved. + pub refs: u64, +} + +/// What the table remembers of a registered string. +#[derive(Debug, Clone, Copy)] +struct Slot { + id: u32, + /// Length of the string when it was registered, to notice a recycled address. + len: u32, +} + +type AddressTable = HashMap; + +fn buckets_for_capacity(capacity: usize) -> usize { + match capacity { + 0 => 0, + 1..=7 => capacity + 1, + _ => capacity / 7 * 8, + } +} + +/// Buckets of the table that replaces one of `capacity` entries when it is full. +fn next_buckets(capacity: usize) -> usize { + match capacity { + 0 => 4, + _ => 2 * buckets_for_capacity(capacity), + } +} + +/// Bytes allocated for `buckets` buckets: 16 for the entry and 1 control byte +/// each, and a trailing group of control bytes of at most 16. +fn table_alloc_bytes(buckets: usize) -> usize { + buckets * 17 + 16 +} + +fn table_bytes(capacity: usize) -> usize { + match buckets_for_capacity(capacity) { + 0 => 0, + buckets => table_alloc_bytes(buckets), + } +} + +/// How one string occurrence is written. +enum Choice { + Text, + Define, + Reference(u32), +} + +/// Encodes documents into a stream in which shared strings are defined once. +/// +/// A string is defined at its first use when at least two values own it and it +/// has at least 8 bytes, and is referenced at every use after that. Write the +/// documents of a stream in the order they are encoded, and have the reader +/// decode them in that order: a reference is only valid once the document with +/// the definition has been read. +/// +/// A document that was encoded but not written must be followed by +/// [`poison`](Self::poison), because its definitions would shift the IDs of every +/// later document. +/// +/// The default [`AddressDictionary`] requires stable allocations; a +/// [`ContentDictionary`] owns copies and admits strings without owner counts. +pub struct StreamEncoder { + /// Heap budget for the address table, including growth allocations. + budget: usize, + /// Maps string allocation addresses to IDs and lengths; does not own the strings. + // The default dictionary has the address-only behavior described above. + table: D, + reset_pending: bool, + /// Sticky failure state: encode later strings inline, without DEF or REF. + poisoned: bool, + /// Stream counters for reporting. + /// Definition counts provide cumulative stream positions across resets. + stats: StreamEncoderStats, +} + +impl StreamEncoder { + /// Creates an encoder that has defined nothing yet. + pub fn new(config: StreamEncoderConfig) -> Self { + Self::with_dictionary(config, AddressDictionary::default()) + } + + /// Creates an encoder with the supplied dictionary, clearing any old entries. + pub fn with_dictionary( + config: StreamEncoderConfig, + mut dictionary: D, + ) -> StreamEncoder { + dictionary.clear(); + StreamEncoder { + budget: config.table_budget_bytes, + table: dictionary, + reset_pending: false, + poisoned: false, + stats: StreamEncoderStats::default(), + } + } +} + +impl StreamEncoder { + /// Appends the CBOR encoding of `value` to `out`, after whatever `out` + /// already holds, and reports what the document used. + /// + /// When the returned [`DocUse`] is zero the bytes are exactly those of + /// [`encode`](crate::encode). If an address turns out to be reused by a string of + /// another length, the encoder is poisoned and that string and the rest of the + /// document are written inline. + pub fn encode_into(&mut self, out: &mut Vec, value: &IValue) -> DocUse { + let (defs, refs) = (self.stats.defs, self.stats.refs); + let encoder = RefCell::new(&mut *self); + let cbor = EncodedValue { + value, + strings: &encoder, + }; + ciborium::into_writer(&cbor, out).expect("write to Vec never fails"); + DocUse { + defs: self.stats.defs - defs, + refs: self.stats.refs - refs, + } + } + + /// The ID the next definition receives, which is the number of strings + /// defined so far. Read it before encoding a document to learn the ID of its + /// first definition. A poisoned encoder defines nothing more, so what it + /// reports no longer matters. + /// Counts the current dictionary; `stats().defs` counts the entire stream. + pub fn next_id(&self) -> u32 { + // IDs stop below `u32::MAX`, see `has_room`. + // Strings in the ID table. + self.table.len() as u32 + } + + /// Reuses the table after a budget miss. Call only between definition-capable documents, + /// and signal the reader before the next document that introduces definitions. + pub fn reset_if_needed(&mut self) -> bool { + if !self.reset_pending || self.poisoned { + return false; + } + self.table.clear(); + self.reset_pending = false; + true + } + + /// Stops all sharing: every later string is written inline, with no + /// definitions and no references. The encoder does this itself when it finds an + /// address that was reused by a string of another length. + pub fn poison(&mut self) { + self.poisoned = true; + } + + /// Whether [`poison`](Self::poison) was called or the encoder poisoned itself. + pub fn is_poisoned(&self) -> bool { + self.poisoned + } + + /// Serializes one complete CBOR document using [`StreamStrings::string`] for + /// each string occurrence, including object keys. The callback must serialize + /// directly in wire order, without pre-serializing or discarding definitions. + /// + /// On error, appended bytes are discarded and the encoder is poisoned, so later + /// documents cannot reference definitions that were never written. + pub fn encode_with( + &mut self, + out: &mut Vec, + encode: impl FnOnce(&StreamStrings<'_>, &mut Vec) -> Result<(), E>, + ) -> Result { + let (defs, refs) = (self.stats.defs, self.stats.refs); + let start = out.len(); + let mut choose = |string: &StreamString<'_>| self.choose(string); + let strings = StreamStrings { + choose: RefCell::new(&mut choose), + }; + if let Err(error) = encode(&strings, out) { + out.truncate(start); + self.poison(); + return Err(error); + } + Ok(DocUse { + defs: self.stats.defs - defs, + refs: self.stats.refs - refs, + }) + } + + /// Counters and the current size of the ID table. + pub fn stats(&self) -> StreamEncoderStats { + self.stats + } + + fn choose(&mut self, string: &StreamString<'_>) -> Choice { + if self.poisoned { + return Choice::Text; + } + // Occurrences left inline because the string was shorter than 8 bytes, + // which includes the empty string and every string kept inside its handle. + let len = string.as_str().len(); + if len < MIN_DEFINED_LEN { + // The empty string and strings kept inside their handle, all below `MIN_DEFINED_LEN`. + return Choice::Text; + } + if let Some((id, defined_len)) = self.table.get(string) { + // Times a registered address belonged to a string of another length, which + // means its string was freed and the address reused. Each one poisons the + // encoder: that string and every later one is written inline. Zero proves + // nothing about lifetime safety: a string of the same length + // at a reused address goes unnoticed. + // only a length change shows a recycled address; an equal length is + // prevented by the caller keeping the allocation alive, never by this check. + if defined_len != len { + self.poisoned = true; + return Choice::Text; + } + self.stats.refs += 1; + // Text of 8 or more bytes takes at least 9; a reference takes its tag and at most 5. + self.stats.saved_bytes += + cbor_head_len(len as u64) + len as u64 - 1 - cbor_head_len(u64::from(id)); + return Choice::Reference(id); + } + // Cumulative definition positions must remain representable in a u32 stream header. + if self.stats.defs >= u64::from(u32::MAX) { + return Choice::Text; + } + if !self.table.can_define(string) { + return Choice::Text; + } + // Occurrences left inline because the table had no room or could not represent the string. + if u32::try_from(len).is_err() { + return Choice::Text; + } + let id = self.next_id(); + if !self.table.insert(string, id, self.budget) { + self.reset_pending = !self.table.is_empty(); + return Choice::Text; + } + self.stats.defs += 1; + self.stats.def_bytes += len as u64; + Choice::Define + } +} + +impl fmt::Debug for StreamEncoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StreamEncoder") + .field("next_id", &self.next_id()) + .field("poisoned", &self.poisoned) + .field("stats", &self.stats()) + .finish_non_exhaustive() + } +} + +/// String wrappers for a single document, borrowed from its stream encoder. +pub struct StreamStrings<'a> { + choose: RefCell<&'a mut dyn FnMut(&StreamString<'_>) -> Choice>, +} + +impl fmt::Debug for StreamStrings<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StreamStrings").finish_non_exhaustive() + } +} + +impl<'e> StreamStrings<'e> { + /// Wraps a string without registering it until the wrapper is serialized. + pub fn string<'a, T: Into> + 'a>( + &'a self, + string: T, + ) -> impl Serialize + 'a + use<'a, 'e, T> { + struct StringValue<'a, 'e>(&'a StreamStrings<'e>, StreamString<'a>); + impl Serialize for StringValue<'_, '_> { + fn serialize(&self, serializer: S) -> Result { + let choice = (self.0.choose.borrow_mut())(&self.1); + serialize_choice(choice, self.1.as_str(), serializer) + } + } + StringValue(self, string.into()) + } +} + +/// Writes one string occurrence of a document as text, a definition or a +/// reference. +pub(super) fn serialize_string( + encoder: &EncoderCell<'_, D>, + string: &IString, + serializer: S, +) -> Result { + // The choice is made, and a definition registered, right before the bytes are + // written, and no borrow is held while they are. + let choice = encoder.borrow_mut().choose(&string.into()); + serialize_choice(choice, string.as_str(), serializer) +} + +fn serialize_choice( + choice: Choice, + string: &str, + serializer: S, +) -> Result { + match choice { + Choice::Text => serializer.serialize_str(string), + Choice::Define => RequireAny(TAG_DEF, string).serialize(serializer), + Choice::Reference(id) => RequireAny(TAG_REF, id).serialize(serializer), + } +} + +/// Decodes the documents of a stream written by a [`StreamEncoder`]. +/// +/// Documents must be decoded in the order they were encoded. Each definition +/// adds a string to the table, and a reference to an ID the table does not hold +/// is an error. After any error the decoder refuses further documents, since the +/// failed one may have held definitions that later references depend on. +#[derive(Default)] +pub struct StreamDecoder { + entries: Vec, + stats: StreamDecoderStats, + failed: bool, +} + +impl StreamDecoder { + /// Creates a decoder with an empty table. + pub fn new() -> Self { + Self::default() + } + + /// The number of strings defined so far, which is the ID the next definition + /// receives. A document whose encoder-side [`next_id`](StreamEncoder::next_id) + /// differs from this was not decoded in order. + /// Counts only the current dictionary after a reset. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether nothing was defined yet. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Releases this dictionary's handles without discarding statistics or clearing failures. + /// Already decoded values retain their own string handles. + pub fn reset(&mut self) { + self.entries.clear(); + } + + /// Decodes one document, which must be the whole of `payload`. + /// + /// Tags 6 and 7 are definitions and references here, and no other tag is a + /// string tag. A definition must hold text of at least 8 bytes; a reference + /// must hold a nonnegative integer below [`len`](Self::len). Typed arrays and + /// the nesting limit behave as in + /// [`decode_with_depth_limit`](crate::decode_with_depth_limit). + /// Trailing bytes are an error. + pub fn decode(&mut self, payload: &[u8], limit_depth: bool) -> Result { + if self.failed { + return Err(CborDecodeError::DecodeError); + } + let mut rest = payload; + let decoder = RefCell::new(&mut *self); + let decoded = super::decode_one_value(&mut rest, limit_depth, StringRefs::Stream(&decoder)); + let result = match decoded { + Ok(_) if !rest.is_empty() => Err(CborDecodeError::DecodeError), + other => other, + }; + self.failed |= result.is_err(); + result + } + + /// Counters of the documents decoded so far. + pub fn stats(&self) -> StreamDecoderStats { + self.stats + } + + pub(super) fn define_str(&mut self, text: &str) -> Result { + if text.len() < MIN_DEFINED_LEN { + return Err(CborDecodeError::InvalidValue); + } + let string = IString::from(text); + self.stats.defs += 1; + self.stats.def_bytes += string.len() as u64; + self.entries.push(string.clone()); + Ok(string) + } + + pub(super) fn reference_id(&mut self, id: usize) -> Result { + let string = self + .entries + .get(id) + .cloned() + .ok_or(CborDecodeError::InvalidStringReference)?; + self.stats.refs += 1; + Ok(string) + } +} + +impl fmt::Debug for StreamDecoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StreamDecoder") + .field("len", &self.len()) + .field("failed", &self.failed) + .field("stats", &self.stats) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::IArray; + use mockalloc::record_allocs; + + fn config() -> StreamEncoderConfig { + StreamEncoderConfig { + table_budget_bytes: 1 << 20, + } + } + + /// An array that holds every string twice, so that each has two owners. + fn shared_array(texts: &[&str]) -> IValue { + let strings: Vec = texts + .iter() + .chain(texts) + .map(|text| IValue::from(IString::from(*text))) + .collect(); + IArray::try_from(strings).unwrap().into() + } + + #[test] + fn budget_model_covers_what_the_allocator_sees() { + let mut model_peak = 0; + let mut final_bytes = 0; + let allocations = record_allocs(|| { + let mut table = AddressTable::default(); + for index in 1..=3_000usize { + if table.len() == table.capacity() { + let old = table_bytes(table.capacity()); + model_peak = + model_peak.max(old + table_alloc_bytes(next_buckets(table.capacity()))); + } + table.insert(index * 8, Slot { id: 0, len: 0 }); + } + final_bytes = table_bytes(table.capacity()); + }); + let peak = allocations.peak_mem() as usize; + assert!(peak <= model_peak, "model {model_peak} below actual {peak}"); + assert!( + model_peak - peak <= 64, + "model {model_peak} far above actual {peak}" + ); + assert!(final_bytes <= peak); + } + + #[test] + fn repeated_resets_keep_writer_allocations_within_budget() { + let texts: Vec<_> = (0..40) + .map(|i| format!("reset allocation string {i:04}")) + .collect(); + let refs: Vec<_> = texts.iter().map(String::as_str).collect(); + let value = shared_array(&refs); + let mut bytes = Vec::with_capacity(4096); + let mut resets = 0; + let allocations = record_allocs(|| { + let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: 700, + ..config() + }); + for _ in 0..64 { + resets += usize::from(encoder.reset_if_needed()); + bytes.clear(); + encoder.encode_into(&mut bytes, &value); + assert!(encoder.table.used_bytes() <= 700); + } + assert!(encoder.stats().defs > 40); + }); + assert!(resets > 0); + assert!(allocations.peak_mem() <= 700); + } + + #[test] + fn cumulative_definition_positions_do_not_wrap_after_resets() { + let first = shared_array(&["a previously defined string"]); + let second = shared_array(&["a string beyond the position limit"]); + let mut encoder = StreamEncoder::new(config()); + encoder.encode_into(&mut Vec::new(), &first); + encoder.stats.defs = u64::from(u32::MAX); + assert_eq!( + encoder.encode_into(&mut Vec::new(), &second), + DocUse::default() + ); + assert!(!encoder.reset_if_needed()); + assert_eq!(encoder.encode_into(&mut Vec::new(), &first).refs, 2); + assert_eq!(encoder.stats().defs, u64::from(u32::MAX)); + } + + #[test] + fn saved_bytes_are_the_exact_difference_from_the_plain_encoding() { + // Lengths and ids on both sides of each CBOR head size. + let texts: Vec = [8, 23, 24, 255, 256, 65_535, 65_536] + .iter() + .map(|&len| "s".repeat(len)) + .chain((0..30).map(|i| format!("one of many ids {i:02}"))) + .collect(); + let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); + let value = shared_array(&refs); + let mut encoder = StreamEncoder::new(config()); + let mut shared = Vec::new(); + encoder.encode_into(&mut shared, &value); + let stats = encoder.stats(); + assert_eq!(stats.refs, texts.len() as u64); + assert_eq!( + crate::encode(&value).len() as u64, + shared.len() as u64 + stats.saved_bytes - stats.defs + ); + } + + #[test] + fn table_growth_stays_within_the_budget_at_scale() { + const STRINGS: usize = 60_000; + let strings: Vec = (0..STRINGS) + .map(|index| IString::from(format!("budget shared string {index:06}"))) + .collect(); + let doc: IValue = IArray::try_from( + strings + .iter() + .cloned() + .map(IValue::from) + .collect::>(), + ) + .unwrap() + .into(); + // Returns how many strings were defined and the bytes the table holds. + let encode_with_budget = |budget| { + let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: budget, + }); + let mut bytes = Vec::new(); + let used = encoder.encode_into(&mut bytes, &doc); + assert_eq!(used.defs, encoder.table.len() as u64); + assert!(encoder.table.used_bytes() <= budget, "{encoder:?}"); + assert_eq!(StreamDecoder::new().decode(&bytes, true).unwrap(), doc); + (used.defs, encoder.table.used_bytes()) + }; + let (all, needed) = encode_with_budget(1 << 30); + assert_eq!(all, STRINGS as u64); + // Half of what the whole table needs leaves no room for it to grow to that size. + let (defined, _) = encode_with_budget(needed / 2); + assert!(defined > 0 && defined < STRINGS as u64, "{defined}"); + } +} diff --git a/src/cbor/stream/dictionary.rs b/src/cbor/stream/dictionary.rs new file mode 100644 index 00000000..53da5f1a --- /dev/null +++ b/src/cbor/stream/dictionary.rs @@ -0,0 +1,226 @@ +use std::num::NonZeroUsize; + +use hashbrown::HashMap; + +use super::{next_buckets, table_alloc_bytes, table_bytes, AddressTable, Slot}; +use crate::IString; + +/// Text supplied to a dictionary, with an optional interned allocation identity. +#[derive(Debug, Clone, Copy)] +pub struct StreamString<'a> { + text: &'a str, + identity: Option<(NonZeroUsize, usize)>, +} + +impl<'a> StreamString<'a> { + /// The string's UTF-8 contents. + pub fn as_str(&self) -> &'a str { + self.text + } + + /// An interned allocation's address and owner count, when available. + pub fn identity(&self) -> Option<(NonZeroUsize, usize)> { + self.identity + } +} + +impl<'a> From<&'a str> for StreamString<'a> { + fn from(text: &'a str) -> Self { + Self { + text, + identity: None, + } + } +} + +impl<'a> From<&'a String> for StreamString<'a> { + fn from(text: &'a String) -> Self { + text.as_str().into() + } +} + +impl<'a> From<&'a IString> for StreamString<'a> { + fn from(text: &'a IString) -> Self { + Self { + text: text.as_str(), + identity: text + .heap_identity() + .map(|(address, _, owners)| (address, owners)), + } + } +} + +/// String identity and storage used by a stream encoder. +/// +/// Entries must retain their identity until `clear`. `insert` must account for all +/// entry allocations, including simultaneous old/new tables during growth. +/// IDs are assigned by the encoder; dictionaries must return them unchanged. +pub trait StringDictionary { + /// Number of definitions in the current dictionary. + fn len(&self) -> usize; + + /// Whether this dictionary has no definitions. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Releases definitions; implementations may retain their table allocation. + fn clear(&mut self); + + /// Returns a known string's ID and its length when registered. + fn get(&self, string: &StreamString<'_>) -> Option<(u32, usize)>; + + /// Whether a previously unseen string is eligible for a definition. + fn can_define(&self, string: &StreamString<'_>) -> bool; + + /// Inserts an absent string if its peak allocation fits `budget`. + /// A refusal must leave the dictionary unchanged. + fn insert(&mut self, string: &StreamString<'_>, id: u32, budget: usize) -> bool; +} + +impl StringDictionary for Box { + fn len(&self) -> usize { + (**self).len() + } + fn clear(&mut self) { + (**self).clear(); + } + fn get(&self, string: &StreamString<'_>) -> Option<(u32, usize)> { + (**self).get(string) + } + fn can_define(&self, string: &StreamString<'_>) -> bool { + (**self).can_define(string) + } + fn insert(&mut self, string: &StreamString<'_>, id: u32, budget: usize) -> bool { + (**self).insert(string, id, budget) + } +} + +/// Recognizes interned strings without copying their bytes or retaining handles. +/// +/// The caller must keep every defined string allocation alive and at the same +/// address until the stream ends. The encoder stores addresses, not owned handles; +/// dropping or moving those strings can make later references resolve to wrong text. +#[derive(Debug, Default)] +pub struct AddressDictionary { + pub(super) entries: AddressTable, +} + +impl AddressDictionary { + /// Heap bytes held by the ID table. + pub(super) fn used_bytes(&self) -> usize { + table_bytes(self.entries.capacity()) + } + + /// Whether one more definition fits the budget, counting the old and the new + /// allocation while the table grows. No allocation can fail gracefully, so the + /// budget is checked in advance. A table that is full and cannot grow stays so, + /// because nothing is added to it from then on. + /// An explicit reset reuses its existing allocation for a new dictionary. + fn has_room(&self, budget: usize) -> bool { + let table_growth = if self.entries.len() == self.entries.capacity() { + table_alloc_bytes(next_buckets(self.entries.capacity())) + } else { + 0 + }; + (table_growth == 0 || self.used_bytes() + table_growth <= budget) + && self.entries.len() < u32::MAX as usize + } +} + +impl StringDictionary for AddressDictionary { + fn len(&self) -> usize { + self.entries.len() + } + + fn clear(&mut self) { + self.entries.clear(); + } + + fn get(&self, string: &StreamString<'_>) -> Option<(u32, usize)> { + let (address, _) = string.identity()?; + self.entries + .get(&address.get()) + .map(|slot| (slot.id, slot.len as usize)) + } + + fn can_define(&self, string: &StreamString<'_>) -> bool { + // Occurrences left inline because the string had a single owner. + string.identity().is_some_and(|(_, owners)| owners >= 2) + } + + fn insert(&mut self, string: &StreamString<'_>, id: u32, budget: usize) -> bool { + let Some((address, _)) = string.identity() else { + return false; + }; + let Ok(len) = u32::try_from(string.as_str().len()) else { + return false; + }; + if !self.has_room(budget) { + return false; + } + self.entries.insert(address.get(), Slot { id, len }); + true + } +} + +/// Matches strings by contents and owns a copy of every definition. +/// +/// Source strings may be dropped immediately after encoding a document. The +/// budget includes these copies as well as hash-table storage and growth. +#[derive(Debug, Default)] +pub struct ContentDictionary { + entries: HashMap, u32>, + string_bytes: usize, +} + +impl ContentDictionary { + fn allocation(buckets: usize) -> usize { + buckets * (std::mem::size_of::<(Box, u32)>() + 1) + 16 + } +} + +impl StringDictionary for ContentDictionary { + fn len(&self) -> usize { + self.entries.len() + } + + fn clear(&mut self) { + self.entries.clear(); + self.string_bytes = 0; + } + + fn get(&self, string: &StreamString<'_>) -> Option<(u32, usize)> { + let string = string.as_str(); + self.entries.get(string).map(|&id| (id, string.len())) + } + + fn can_define(&self, _string: &StreamString<'_>) -> bool { + true + } + + fn insert(&mut self, string: &StreamString<'_>, id: u32, budget: usize) -> bool { + let string = string.as_str(); + let capacity = self.entries.capacity(); + let current = if capacity == 0 { + 0 + } else { + Self::allocation(super::buckets_for_capacity(capacity)) + }; + let growth = if self.entries.len() == capacity { + Self::allocation(next_buckets(capacity)) + } else { + 0 + }; + let required = current + .saturating_add(growth) + .saturating_add(self.string_bytes) + .saturating_add(string.len()); + if required > budget { + return false; + } + self.entries.insert(Box::from(string), id); + self.string_bytes += string.len(); + true + } +} diff --git a/src/lib.rs b/src/lib.rs index f805ad05..e990d8af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,7 +56,10 @@ pub mod cbor; mod de; mod ser; pub use cbor::{ - decode, decode_compressed, decode_with_depth_limit, encode, encode_compressed, CborDecodeError, + decode, decode_compressed, decode_with_depth_limit, encode, encode_compressed, encode_into, + AddressDictionary, CborDecodeError, ContentDictionary, DocUse, StreamDecoder, + StreamDecoderStats, StreamEncoder, StreamEncoderConfig, StreamEncoderStats, StreamString, + StreamStrings, StringDictionary, }; pub use de::{from_value, FPHAConfig, IValueDeserSeed}; pub use ser::to_value; @@ -83,14 +86,27 @@ pub trait Defrag { /// Any json that still uses a shared string will continue using it. /// But new strings will be reinitialized instead of reused the old ones. pub fn reinit_shared_string_cache() { + #[cfg(not(feature = "thread_safe"))] unsafe_string::reinit_cache(); + #[cfg(feature = "thread_safe")] + string::reinit_cache(); } /// Initialized shared string cache. Either thread safe or thread unsafe (in case /// the user know the string cache is protected by other means or the application is /// single threaded). Return `Ok(())` on succeed and error if the cache is already initialized. +/// With the `thread_safe` feature, initialization is idempotent and always concurrent. pub fn init_shared_string_cache(thread_safe: bool) -> Result<(), String> { - unsafe_string::init_cache(thread_safe) + #[cfg(not(feature = "thread_safe"))] + { + unsafe_string::init_cache(thread_safe) + } + #[cfg(feature = "thread_safe")] + { + let _ = thread_safe; + string::init_cache(); + Ok(()) + } } #[cfg(all(test, not(miri)))] diff --git a/src/string.rs b/src/string.rs index 9ac0e2ee..fdb3ca23 100644 --- a/src/string.rs +++ b/src/string.rs @@ -5,6 +5,7 @@ use std::borrow::Borrow; use std::cmp::Ordering; use std::fmt::{self, Debug, Formatter}; use std::hash::Hash; +use std::num::NonZeroUsize; use std::ops::Deref; use std::ptr::{copy_nonoverlapping, NonNull}; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; @@ -14,7 +15,8 @@ use lazy_static::lazy_static; use crate::{ thin::{ThinMut, ThinMutExt, ThinRef, ThinRefExt}, - Defrag, DefragAllocator, IValue, TypeTag, + value::TypeTag, + Defrag, DefragAllocator, IValue, }; #[repr(C)] @@ -75,6 +77,10 @@ pub fn init_cache() { lazy_static::initialize(&STRING_CACHE); } +pub(crate) fn reinit_cache() { + STRING_CACHE.clear(); +} + struct WeakIString { ptr: NonNull
, } @@ -275,7 +281,13 @@ impl IString { let mut guard = shard.write(); if hd.rc.fetch_sub(1, AtomicOrdering::Relaxed) == 1 { // Reference count reached zero, free the string - assert!(guard.remove(hd.str()).is_some()); + // A cache reset can replace this allocation while old handles remain alive. + if guard + .get_key_value(hd.str()) + .is_some_and(|(entry, _)| entry.ptr.as_ptr().cast_const() == hd.ptr()) + { + guard.remove(hd.str()); + } // Shrink the shard if it's mostly empty. // The second condition is necessary because `HashMap` sometimes @@ -299,6 +311,24 @@ impl IString { / self.header().rc.load(AtomicOrdering::Relaxed) as f64 } } + + /// Identity of this string's heap allocation: the untagged header address, + /// the length in bytes and the number of live owners. + /// + /// `None` for the empty string, which shares a static header. Every other + /// string is allocated, and all clones of it report the same address. Use + /// this instead of `as_ptr()`, which points at the bytes rather than the + /// header. Reads the header without writing it or touching the cache. The + /// owner count is a relaxed snapshot. + pub(crate) fn heap_identity(&self) -> Option<(NonZeroUsize, usize, usize)> { + let header = self.header(); + let len = header.len(); + if len == 0 { + return None; + } + let owners = header.rc.load(AtomicOrdering::Relaxed); + NonZeroUsize::new(header.ptr() as usize).map(|address| (address, len, owners)) + } } impl Deref for IString { @@ -352,7 +382,9 @@ impl From for String { impl PartialEq for IString { fn eq(&self, other: &Self) -> bool { - self.0.raw_eq(&other.0) + // After a cache reset the same text can live in two allocations, so compare the text when + // the pointers differ. + self.0.raw_eq(&other.0) || self.as_str() == other.as_str() } } @@ -403,7 +435,7 @@ impl PartialOrd for IString { } impl Hash for IString { fn hash(&self, state: &mut H) { - self.0.raw_hash(state); + self.as_str().hash(state); } } @@ -423,6 +455,67 @@ impl Defrag for IString { mod tests { use super::*; + #[test] + fn stream_encoding_never_waits_for_the_cache() { + use crate::{IArray, IValue, StreamEncoder, StreamEncoderConfig}; + use std::sync::mpsc::channel; + use std::time::Duration; + + let shared = IString::from("a string shared by two array slots"); + let doc: IValue = IArray::try_from(vec![ + IValue::from(shared.clone()), + IValue::from(shared.clone()), + ]) + .unwrap() + .into(); + + let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: 1 << 20, + }); + // With every shard locked, interning or freeing a string would block. + let guards: Vec<_> = STRING_CACHE.shards().iter().map(|s| s.write()).collect(); + let (sender, receiver) = channel(); + let used = std::thread::scope(|scope| { + scope.spawn(|| { + let used = encoder.encode_into(&mut Vec::new(), &doc); + sender.send(used).unwrap(); + }); + let used = receiver.recv_timeout(Duration::from_secs(10)); + // Release the locks first, so that a blocked encoder can finish. + drop(guards); + used + }); + let used = used.expect("encoding waited for the string cache"); + assert_eq!((used.defs, used.refs), (1, 1)); + } + + #[mockalloc::test] + fn heap_identity_names_the_shared_allocation() { + let first = IString::intern("heap identity of a long string"); + let (address, len, owners) = first.heap_identity().unwrap(); + assert_eq!((len, owners), (first.len(), 1)); + assert_eq!(address.get() % 8, 0); + assert_eq!( + first.as_ptr() as usize, + address.get() + std::mem::size_of::
() + ); + let second = first.clone(); + assert_eq!(second.heap_identity(), Some((address, len, 2))); + drop(second); + assert_eq!(first.heap_identity(), Some((address, len, 1))); + let other = IString::intern("another long string with other bytes"); + let (other_address, other_len, _) = other.heap_identity().unwrap(); + assert_ne!(other_address, address); + assert_eq!(other_len, other.len()); + } + + #[mockalloc::test] + fn heap_identity_ignores_only_the_empty_string() { + assert_eq!(IString::intern("").heap_identity(), None); + let short = IString::intern("a"); + assert_eq!(short.heap_identity().unwrap().1, 1); + } + #[mockalloc::test] fn can_intern() { let x = IString::intern("foo"); diff --git a/src/unsafe_string.rs b/src/unsafe_string.rs index 54d74792..41dd4668 100644 --- a/src/unsafe_string.rs +++ b/src/unsafe_string.rs @@ -7,6 +7,7 @@ use std::cmp::Ordering; use std::fmt::{self, Debug, Formatter}; use std::hash::Hash; use std::mem::{self, transmute}; +use std::num::NonZeroUsize; use std::ops::Deref; use std::ptr::{addr_of_mut, copy_nonoverlapping, NonNull}; use std::sync::{atomic::AtomicU32, Mutex, MutexGuard, OnceLock}; @@ -433,6 +434,30 @@ impl IString { / self.header().rc.load(std::sync::atomic::Ordering::Relaxed) as f64 } } + + /// Identity of this string's heap allocation: the untagged header address, + /// the length in bytes and the number of live owners. + /// + /// `None` for the empty string and for strings of 7 bytes or fewer, which are + /// stored inside the handle and have no allocation. All clones of a string + /// report the same address. Use this instead of `as_ptr()`, which for an + /// inline string points into the handle. Reads the header without writing + /// it or touching the cache. The owner count is a relaxed snapshot. + pub(crate) fn heap_identity(&self) -> Option<(NonZeroUsize, usize, usize)> { + if self.is_inline() { + return None; + } + let header = self.header(); + let len = header.len(); + if len == 0 { + return None; + } + // SAFETY: the handle is neither inline nor empty, so it is tagged + // `StringOrNull` and `ptr()` strips the tag to reach the header. + let address = unsafe { self.0.ptr() } as usize; + let owners = header.rc.load(std::sync::atomic::Ordering::Relaxed) as usize; + NonZeroUsize::new(address).map(|address| (address, len, owners)) + } } impl Deref for IString { @@ -573,6 +598,97 @@ mod tests { use super::*; use mockalloc::record_allocs; + // The unit tests share one cache, which an earlier test may have initialized + // without a lock, so this test runs again in a process of its own. + #[test] + fn stream_encoding_never_waits_for_the_cache_lock() { + use crate::{IArray, IValue, StreamEncoder, StreamEncoderConfig}; + use std::sync::mpsc::channel; + use std::time::Duration; + + const NAME: &str = "unsafe_string::tests::stream_encoding_never_waits_for_the_cache_lock"; + const CHILD: &str = "IJSON_UNIT_TEST_LOCKED_CACHE"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", NAME, "--test-threads=1"]) + .env(CHILD, "1") + .output() + .unwrap(); + let report = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.status.success() && report.contains("1 passed"), + "{report}" + ); + return; + } + + init_cache(true).unwrap(); + let StringCache::ThreadSafe(cache) = &*get_cache_mut() else { + unreachable!(); + }; + let shared = IString::intern("a string shared by two array slots"); + let doc: IValue = IArray::try_from(vec![ + IValue::from(shared.clone()), + IValue::from(shared.clone()), + ]) + .unwrap() + .into(); + + let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: 1 << 20, + }); + // With the cache locked, interning or freeing a string would block. + let guard = cache.lock().unwrap(); + let (sender, receiver) = channel(); + let used = std::thread::scope(|scope| { + scope.spawn(|| { + let used = encoder.encode_into(&mut Vec::new(), &doc); + sender.send(used).unwrap(); + }); + let used = receiver.recv_timeout(Duration::from_secs(10)); + // Release the lock first, so that a blocked encoder can finish. + drop(guard); + used + }); + let used = used.expect("encoding waited for the string cache"); + assert_eq!((used.defs, used.refs), (1, 1)); + } + + #[mockalloc::test] + fn heap_identity_names_the_shared_allocation() { + let first = IString::intern("heap identity of a long string"); + let (address, len, owners) = first.heap_identity().unwrap(); + assert_eq!((len, owners), (first.len(), 1)); + assert_eq!(address.get() % ALIGNMENT, 0); + assert_eq!( + first.as_ptr() as usize, + address.get() + mem::size_of::
() + ); + let second = first.clone(); + assert_eq!(second.heap_identity(), Some((address, len, 2))); + drop(second); + assert_eq!(first.heap_identity(), Some((address, len, 1))); + let other = IString::intern("another long string with other bytes"); + let (other_address, other_len, _) = other.heap_identity().unwrap(); + assert_ne!(other_address, address); + assert_eq!(other_len, other.len()); + } + + #[test] + fn heap_identity_ignores_the_empty_and_inline_strings() { + assert_no_allocs(|| { + for text in ["", "a", "hello", "1234567"] { + assert_eq!(IString::intern(text).heap_identity(), None, "{text:?}"); + } + }); + let eight = IString::intern("12345678"); + assert_eq!(eight.heap_identity().unwrap().1, 8); + } + fn assert_no_allocs(f: F) { let alloc_info = record_allocs(f); assert_eq!( diff --git a/src/value.rs b/src/value.rs index ef14aca8..e20e9018 100644 --- a/src/value.rs +++ b/src/value.rs @@ -238,7 +238,7 @@ impl IValue { // Safety: Pointer must be non-null and aligned to at least ALIGNMENT pub(crate) unsafe fn new_ptr(p: *mut u8, tag: TypeTag) -> Self { Self { - ptr: NonNull::new_unchecked(p.add(tag as usize)), + ptr: NonNull::new_unchecked(p.wrapping_add(tag as usize)), } } // Safety: Reference must be aligned to at least ALIGNMENT @@ -816,7 +816,11 @@ impl Hash for IValue { use ValueType::*; match self.type_() { // Inline and interned types can be trivially hashed - Null | Bool | String => self.ptr.hash(state), + Null | Bool => self.ptr.hash(state), + // Hash the text, as equality does: one string can have two allocations after a + // cache reset. + // Safety: We checked the type + String => unsafe { self.as_string_unchecked() }.hash(state), // Safety: We checked the type Array => unsafe { self.as_array_unchecked() }.hash(state), // Safety: We checked the type @@ -1227,7 +1231,14 @@ mod tests { assert!(matches!(x.clone().destructure(), Destructured::String(u) if u == s)); assert!(matches!(x.clone().destructure_ref(), DestructuredRef::String(u) if *u == s)); assert!(matches!(x.clone().destructure_mut(), DestructuredMut::String(u) if *u == s)); - assert_eq!(x.mem_allocated(), 0.0); + assert_eq!( + x.mem_allocated(), + if cfg!(feature = "thread_safe") { + 12.0 + } else { + 0.0 + } + ); } let s = String::from("foofoofoo"); @@ -1239,7 +1250,14 @@ mod tests { assert!(matches!(x.clone().destructure(), Destructured::String(u) if u == s)); assert!(matches!(x.clone().destructure_ref(), DestructuredRef::String(u) if *u == s)); assert!(matches!(x.clone().destructure_mut(), DestructuredMut::String(u) if *u == s)); - assert_eq!(x.mem_allocated(), 24.0); + assert_eq!( + x.mem_allocated(), + if cfg!(feature = "thread_safe") { + 32.0 + } else { + 24.0 + } + ); } #[mockalloc::test] diff --git a/tests/cbor_content_stream.rs b/tests/cbor_content_stream.rs new file mode 100644 index 00000000..186b979e --- /dev/null +++ b/tests/cbor_content_stream.rs @@ -0,0 +1,211 @@ +use ijson::{ + ContentDictionary, DocUse, StreamDecoder, StreamEncoder, StreamEncoderConfig, StreamStrings, + StringDictionary, +}; +use mockalloc::{record_allocs, Mockalloc}; +use serde::ser::{SerializeMap, SerializeSeq}; +use serde::{Serialize, Serializer}; + +#[global_allocator] +static ALLOCATOR: Mockalloc = Mockalloc(std::alloc::System); + +fn config(budget: usize) -> StreamEncoderConfig { + StreamEncoderConfig { + table_budget_bytes: budget, + } +} + +struct Document<'a, 'e> { + strings: &'a StreamStrings<'e>, + key: &'a str, + values: &'a [String], +} + +impl Serialize for Document<'_, '_> { + fn serialize(&self, serializer: S) -> Result { + struct Values<'a, 'e>(&'a Document<'a, 'e>); + impl Serialize for Values<'_, '_> { + fn serialize(&self, serializer: S) -> Result { + let mut seq = serializer.serialize_seq(Some(self.0.values.len()))?; + for value in self.0.values { + seq.serialize_element(&self.0.strings.string(value))?; + } + seq.end() + } + } + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry(&self.strings.string(self.key), &Values(self))?; + map.end() + } +} + +#[test] +fn external_strings_are_owned_and_keys_and_values_share_ids() { + let mut encoder = StreamEncoder::with_dictionary(config(4096), ContentDictionary::default()); + let key = String::from("shared object key"); + let values = vec![ + String::from("shared object key"), + String::from("another shared value"), + ]; + let mut first = Vec::new(); + let used = encoder + .encode_with(&mut first, |strings, out| { + ciborium::into_writer( + &Document { + strings, + key: &key, + values: &values, + }, + out, + ) + }) + .unwrap(); + assert_eq!(used, DocUse { defs: 2, refs: 1 }); + drop((key, values)); + + let mut second = Vec::new(); + let used = encoder + .encode_with(&mut second, |strings, out| { + ciborium::into_writer(&strings.string(&String::from("another shared value")), out) + }) + .unwrap(); + assert_eq!(used, DocUse { defs: 0, refs: 1 }); + assert_eq!(second, [0xc7, 1]); + let mut decoder = StreamDecoder::new(); + let first = decoder.decode(&first, true).unwrap(); + assert_eq!( + serde_json::to_value(first).unwrap(), + serde_json::json!({"shared object key": ["shared object key", "another shared value"]}) + ); + assert_eq!( + decoder + .decode(&second, true) + .unwrap() + .as_string() + .unwrap() + .as_str(), + "another shared value" + ); +} + +#[test] +fn content_budget_counts_owned_bytes_and_resets_release_them() { + let mut encoder = StreamEncoder::with_dictionary(config(256), ContentDictionary::default()); + let mut decoder = StreamDecoder::new(); + let mut resets = 0; + for index in 0..20 { + if encoder.reset_if_needed() { + decoder.reset(); + resets += 1; + } + let text = format!("{index:04}{}", "x".repeat(96)); + let mut bytes = Vec::new(); + let used = encoder + .encode_with(&mut bytes, |strings, out| { + ciborium::into_writer(&strings.string(&text), out) + }) + .unwrap(); + assert!(used.defs <= 1); + assert!(encoder.next_id() <= 1); + assert_eq!( + decoder + .decode(&bytes, true) + .unwrap() + .as_string() + .unwrap() + .as_str(), + text + ); + } + assert!(resets > 0); +} + +#[test] +fn failed_serialization_discards_bytes_and_poisons_later_references() { + let mut encoder = StreamEncoder::with_dictionary(config(4096), ContentDictionary::default()); + let mut bytes = vec![42]; + let result = encoder.encode_with(&mut bytes, |strings, out| { + ciborium::into_writer(&strings.string("a definition in a failed document"), out).unwrap(); + Err::<(), _>("serializer failed") + }); + assert_eq!(result, Err("serializer failed")); + assert_eq!(bytes, [42]); + assert!(encoder.is_poisoned()); + + let mut later = Vec::new(); + let used = encoder + .encode_with(&mut later, |strings, out| { + ciborium::into_writer(&strings.string("a definition in a failed document"), out) + }) + .unwrap(); + assert_eq!(used, DocUse::default()); + assert_eq!( + ijson::decode(&later).unwrap().as_string().unwrap().as_str(), + "a definition in a failed document" + ); +} + +#[test] +fn dictionary_policy_does_not_change_the_wire_format() { + let json = r#"{"shared object key":["shared string value","shared string value",42]}"#; + let values: Vec = (0..2) + .map(|_| serde_json::from_str::(json).unwrap()) + .collect(); + let mut address = StreamEncoder::new(config(4096)); + let mut content = StreamEncoder::with_dictionary(config(4096), ContentDictionary::default()); + let mut decoder = StreamDecoder::new(); + for value in &values { + let (mut a, mut c) = (Vec::new(), Vec::new()); + assert_eq!( + address.encode_into(&mut a, value), + content.encode_into(&mut c, value) + ); + assert_eq!(a, c); + assert_eq!(decoder.decode(&c, true).unwrap(), *value); + } +} + +#[test] +fn owned_dictionary_growth_and_repeated_resets_fit_the_allocation_budget() { + let texts: Vec<_> = (0..2000) + .map(|i| format!("{i:08}{}", "x".repeat(i % 100))) + .collect(); + for budget in [512, 4096, 65536] { + let mut resets = 0; + let allocations = record_allocs(|| { + let mut dictionary = ContentDictionary::default(); + for text in &texts { + if !dictionary.insert(&text.into(), dictionary.len() as u32, budget) { + dictionary.clear(); + resets += 1; + assert!(dictionary.insert(&text.into(), 0, budget)); + } + } + }); + assert!(resets > 0); + assert!( + allocations.peak_mem() <= budget as u64, + "budget {budget}, peak {}", + allocations.peak_mem() + ); + } +} + +#[test] +fn a_zero_budget_content_dictionary_writes_plain_values() { + let mut encoder = StreamEncoder::with_dictionary(config(0), ContentDictionary::default()); + for _ in 0..2 { + let mut bytes = Vec::new(); + let used = encoder + .encode_with(&mut bytes, |strings, out| { + ciborium::into_writer(&strings.string("already defined text"), out) + }) + .unwrap(); + assert_eq!(used, DocUse::default()); + assert_eq!( + ijson::decode(&bytes).unwrap().as_string().unwrap().as_str(), + "already defined text" + ); + } + assert_eq!(encoder.stats(), Default::default()); +} diff --git a/tests/cbor_stream.rs b/tests/cbor_stream.rs new file mode 100644 index 00000000..e336a11d --- /dev/null +++ b/tests/cbor_stream.rs @@ -0,0 +1,694 @@ +use ciborium::value::Value; +use ijson::array::ArraySliceRef; +use ijson::{ + decode, decode_with_depth_limit, encode, CborDecodeError, DocUse, IArray, IObject, IString, + IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig, +}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +// Serializes the tests: they look at owner counts and allocator behavior. +static TEST_LOCK: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + // One failing test must not make the others fail on a poisoned lock. + TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[ctor::ctor] +fn init_cache() { + ijson::init_shared_string_cache(true).unwrap(); +} + +fn config() -> StreamEncoderConfig { + StreamEncoderConfig { + table_budget_bytes: 64 << 20, + } +} + +fn json(text: &str) -> IValue { + serde_json::from_str(text).unwrap() +} + +fn cbor_bytes(value: &Value) -> Vec { + let mut bytes = Vec::new(); + ciborium::into_writer(value, &mut bytes).unwrap(); + bytes +} + +fn define(text: &str) -> Value { + Value::Tag(6, Box::new(Value::Text(text.into()))) +} + +fn reference(id: u64) -> Value { + Value::Tag(7, Box::new(Value::Integer(id.into()))) +} + +fn encode_doc(encoder: &mut StreamEncoder, value: &IValue) -> (Vec, DocUse) { + let mut out = Vec::new(); + let used = encoder.encode_into(&mut out, value); + (out, used) +} + +/// A document as the framing layer stores it: the id its first definition gets, +/// and the payload. +struct Doc { + base_id: u32, + bytes: Vec, + used: DocUse, +} + +impl Doc { + fn carries_definitions(&self) -> bool { + self.used.defs > 0 + } + + fn is_stream_doc(&self) -> bool { + self.used.defs + self.used.refs > 0 + } +} + +fn encode_stream(encoder: &mut StreamEncoder, values: &[IValue]) -> Vec { + values + .iter() + .map(|value| { + let base_id = encoder.next_id(); + let (bytes, used) = encode_doc(encoder, value); + Doc { + base_id, + bytes, + used, + } + }) + .collect() +} + +/// What a reader does with one document: stream documents must start where its +/// table ends, other documents are plain CBOR. +fn read_doc(decoder: &mut StreamDecoder, doc: &Doc) -> Result { + if doc.is_stream_doc() && doc.base_id as usize != decoder.len() { + return Err(format!( + "base id {} but the table holds {}", + doc.base_id, + decoder.len() + )); + } + decoder.decode(&doc.bytes, true).map_err(|e| e.to_string()) +} + +fn read_all(docs: &[&Doc]) -> Result<(Vec, StreamDecoder), String> { + let mut decoder = StreamDecoder::new(); + let values = docs + .iter() + .map(|doc| read_doc(&mut decoder, doc)) + .collect::, _>>()?; + Ok((values, decoder)) +} + +#[test] +fn encoded_bytes_match_the_wire_format_built_from_cbor_values() { + let _guard = serial(); + let field = "a field name over the minimum"; + let value = "a value that is long enough to share"; + let values = [ + json(&format!( + r#"{{"{field}":"{value}","list":["{value}","{field}"]}}"# + )), + json(&format!(r#"{{"{field}":["{value}","{value}"]}}"#)), + // A field used as a value, a value used as a field, and a document that shares nothing. + json(&format!( + r#"{{"plain":"{field}","{value}":null,"{field}":1}}"# + )), + json("[1, 2, 3]"), + ]; + let mut encoder = StreamEncoder::new(config()); + let docs = encode_stream(&mut encoder, &values); + + assert_eq!( + docs[0].bytes, + cbor_bytes(&Value::Map(vec![ + (define(field), define(value)), + ( + Value::Text("list".into()), + Value::Array(vec![reference(1), reference(0)]), + ), + ])) + ); + assert_eq!( + docs[1].bytes, + cbor_bytes(&Value::Map(vec![( + reference(0), + Value::Array(vec![reference(1), reference(1)]) + )])) + ); + assert_eq!(docs[3].bytes, encode(&values[3])); + let counts: Vec<_> = docs + .iter() + .map(|doc| (doc.used.defs, doc.used.refs)) + .collect(); + assert_eq!(counts, [(2, 2), (0, 3), (0, 3), (0, 0)]); + assert_eq!( + docs.iter().map(|doc| doc.base_id).collect::>(), + [0, 2, 2, 2] + ); + let stats = encoder.stats(); + assert_eq!((stats.defs, stats.refs), (2, 8)); + assert_eq!(stats.def_bytes, (field.len() + value.len()) as u64); + + let (decoded, decoder) = read_all(&docs.iter().collect::>()).unwrap(); + assert_eq!(decoded, values); + assert_eq!(decoder.len(), 2); + assert_eq!(decoder.stats().refs, stats.refs); +} + +#[test] +fn documents_without_shared_strings_are_byte_identical_to_plain_encoding() { + let _guard = serial(); + let mut typed = IObject::new(); + typed + .insert("f32", IArray::try_from(vec![1.5f32, 2.5]).unwrap()) + .unwrap(); + typed + .insert("u8", IArray::try_from(vec![1u8, 2, 3]).unwrap()) + .unwrap(); + let typed: IValue = typed.into(); + let cases = [ + IValue::NULL, + true.into(), + (-5i64).into(), + u64::MAX.into(), + 1.5f64.into(), + IString::from("").into(), + IString::from("short").into(), + json(r#"{"a":[1,2,{"b":"c"}],"d":"only one owner of this long string"}"#), + typed, + ]; + let mut encoder = StreamEncoder::new(config()); + for value in cases { + let (bytes, used) = encode_doc(&mut encoder, &value); + assert_eq!(bytes, encode(&value), "{value:?}"); + assert_eq!(used, DocUse::default()); + assert_eq!(decode(&bytes).unwrap(), value); + } + assert_eq!(encoder.next_id(), 0); +} + +#[test] +fn decoding_follows_byte_order_for_the_first_key_and_value() { + let _guard = serial(); + let (key, value) = ("the first key of the map", "the first value of the map"); + let mut decoder = StreamDecoder::new(); + let first = cbor_bytes(&Value::Map(vec![(define(key), define(value))])); + assert_eq!( + decoder.decode(&first, true).unwrap(), + json(&format!(r#"{{"{key}":"{value}"}}"#)) + ); + let both = cbor_bytes(&Value::Array(vec![ + reference(0), + reference(1), + reference(0), + ])); + assert_eq!( + decoder.decode(&both, true).unwrap(), + json(&format!(r#"["{key}","{value}","{key}"]"#)) + ); + + let mut decoder = StreamDecoder::new(); + let array = cbor_bytes(&Value::Array(vec![ + define("first element, id zero"), + define("second element, id one"), + reference(0), + Value::Map(vec![(define("a key defined late"), reference(1))]), + reference(2), + ])); + assert_eq!( + decoder.decode(&array, true).unwrap(), + json( + r#"["first element, id zero","second element, id one","first element, id zero", + {"a key defined late":"second element, id one"},"a key defined late"]"# + ) + ); + assert_eq!(decoder.len(), 3); + + // Every definition takes the next id, even when its text repeats. + let mut decoder = StreamDecoder::new(); + let twice = cbor_bytes(&Value::Array(vec![ + define("the same text twice"), + define("the same text twice"), + reference(1), + ])); + decoder.decode(&twice, true).unwrap(); + assert_eq!(decoder.len(), 2); +} + +#[test] +fn strings_kept_inline_are_never_registered() { + let _guard = serial(); + let mut encoder = StreamEncoder::new(config()); + for text in ["", "a", "short", "1234567"] { + let owner = IString::from(text); + let value: IValue = IArray::try_from(vec![ + IValue::from(owner.clone()), + IValue::from(owner.clone()), + ]) + .unwrap() + .into(); + let (bytes, used) = encode_doc(&mut encoder, &value); + assert_eq!(bytes, encode(&value), "{text:?}"); + assert_eq!(used, DocUse::default()); + } + assert_eq!(encoder.next_id(), 0); + + let owner = IString::from("12345678"); + let value: IValue = IArray::try_from(vec![IValue::from(owner.clone()), owner.clone().into()]) + .unwrap() + .into(); + let (_, used) = encode_doc(&mut encoder, &value); + assert_eq!(used, DocUse { defs: 1, refs: 1 }); + drop(owner); +} + +#[test] +fn poisoned_encoder_writes_plain_documents() { + let _guard = serial(); + let text = "a string shared by many docs"; + let docs = [ + json(&format!(r#"["{text}"]"#)), + json(&format!(r#"["{text}","{text}"]"#)), + ]; + let mut encoder = StreamEncoder::new(config()); + assert!(!encoder.is_poisoned()); + let (_, used) = encode_doc(&mut encoder, &docs[0]); + assert_eq!(used.defs, 1); + + encoder.poison(); + assert!(encoder.is_poisoned()); + for doc in &docs { + let (bytes, used) = encode_doc(&mut encoder, doc); + assert_eq!(bytes, encode(doc)); + assert_eq!(used, DocUse::default()); + } + assert_eq!(encoder.next_id(), 1); +} + +#[test] +fn table_budget_freezes_new_definitions_but_keeps_references() { + let _guard = serial(); + let strings: Vec = (0..40) + .map(|index| IString::from(format!("budget limited string {index:04}"))) + .collect(); + let value = |range: std::ops::Range| -> IValue { + IArray::try_from( + strings[range] + .iter() + .cloned() + .map(IValue::from) + .collect::>(), + ) + .unwrap() + .into() + }; + let budget = 700; + let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: budget, + ..config() + }); + let first = value(0..40); + let (bytes, used) = encode_doc(&mut encoder, &first); + let stats = encoder.stats(); + assert!(stats.defs > 0 && stats.defs < 40); + assert_eq!(used.defs, stats.defs); + let items: Vec = ciborium::from_reader(bytes.as_slice()).unwrap(); + assert_eq!( + items.iter().filter(|item| item.is_text()).count(), + 40 - stats.defs as usize + ); + let mut decoder = StreamDecoder::new(); + assert_eq!(decoder.decode(&bytes, true).unwrap(), first); + + let registered = stats.defs; + let (bytes, used) = encode_doc(&mut encoder, &first); + assert_eq!(used.defs, 0); + assert_eq!(used.refs, registered); + assert_eq!(decoder.decode(&bytes, true).unwrap(), first); + assert_eq!(encoder.stats().defs, registered); + assert!(!encoder.is_poisoned()); +} + +#[test] +fn a_plain_array_is_not_an_object_key() { + let _guard = serial(); + let text = || Value::Text("abcdefgh".into()); + let arrays = [ + Value::Array(vec![Value::Integer(6.into()), text()]), + Value::Array(vec![Value::Integer(7.into()), Value::Integer(0.into())]), + Value::Array(vec![Value::Integer(6.into())]), + Value::Array(vec![]), + ]; + for key in arrays { + let doc = Value::Map(vec![(key, Value::Integer(1.into()))]); + assert_eq!( + StreamDecoder::new().decode(&cbor_bytes(&doc), true), + Err(CborDecodeError::InvalidValue), + "{doc:?}" + ); + } +} + +#[test] +fn reference_ids_must_be_integers_of_defined_entries() { + let _guard = serial(); + let defined = define("the only defined string"); + let payloads = vec![ + Value::Integer(1.into()), + Value::Integer(u64::MAX.into()), + Value::Integer((-1).into()), + Value::Integer((-(u64::MAX as i128) - 1).try_into().unwrap()), + Value::Tag(2, Box::new(Value::Bytes(vec![1; 9]))), + Value::Tag(3, Box::new(Value::Bytes(vec![0]))), + Value::Float(0.0), + Value::Text("0".into()), + Value::Bytes(vec![0]), + Value::Bool(false), + Value::Null, + Value::Array(vec![]), + Value::Map(vec![]), + reference(0), + ]; + for payload in payloads { + let forged = Value::Tag(7, Box::new(payload)); + for doc in [ + Value::Array(vec![defined.clone(), forged.clone()]), + Value::Map(vec![(defined.clone(), forged.clone())]), + Value::Array(vec![ + defined.clone(), + Value::Map(vec![(forged.clone(), Value::Null)]), + ]), + ] { + assert_eq!( + StreamDecoder::new().decode(&cbor_bytes(&doc), true), + Err(CborDecodeError::InvalidStringReference), + "{doc:?}" + ); + } + } + assert_eq!( + StreamDecoder::new().decode(&cbor_bytes(&reference(0)), true), + Err(CborDecodeError::InvalidStringReference) + ); + assert!(StreamDecoder::new() + .decode(&[0xda, 0x00, 0x00, 0x00, 0x07, 0x00], true) + .is_err()); + + // Any CBOR integer form names an id, here a bignum zero. + let bignum_zero = Value::Tag(7, Box::new(Value::Tag(2, Box::new(Value::Bytes(vec![0]))))); + let doc = Value::Array(vec![defined.clone(), bignum_zero]); + assert_eq!( + StreamDecoder::new() + .decode(&cbor_bytes(&doc), true) + .unwrap(), + json(r#"["the only defined string","the only defined string"]"#) + ); +} + +#[test] +fn decoding_rejects_malformed_definitions() { + let _guard = serial(); + let wrong_payloads = [ + Value::Text("1234567".into()), + Value::Text(String::new()), + Value::Integer(8.into()), + Value::Bytes(b"12345678".to_vec()), + Value::Array(vec![Value::Text("12345678".into())]), + Value::Null, + define("12345678"), + ]; + for payload in wrong_payloads { + let forged = Value::Tag(6, Box::new(payload)); + for doc in [ + forged.clone(), + Value::Map(vec![(forged.clone(), Value::Null)]), + ] { + let mut decoder = StreamDecoder::new(); + assert_eq!( + decoder.decode(&cbor_bytes(&doc), true), + Err(CborDecodeError::InvalidValue), + "{doc:?}" + ); + assert_eq!(decoder.len(), 0); + } + } + // Invalid UTF-8 text never reaches the table either. + let mut decoder = StreamDecoder::new(); + let invalid = [0xC6, 0x68, 0xFF, 0xFE, 0xFD, 0xFC, 0xFB, 0xFA, 0xF9, 0xF8]; + assert!(decoder.decode(&invalid, true).is_err()); + assert_eq!(decoder.len(), 0); + + let mut decoder = StreamDecoder::new(); + assert_eq!( + decoder + .decode(&cbor_bytes(&define("12345678")), true) + .unwrap(), + IValue::from("12345678") + ); + assert_eq!(decoder.len(), 1); +} + +#[test] +fn decoding_rejects_trailing_bytes() { + let _guard = serial(); + let mut bytes = cbor_bytes(&define("a definition with trailing bytes")); + bytes.push(0xF6); + let mut decoder = StreamDecoder::new(); + assert!(decoder.decode(&bytes, true).is_err()); +} + +#[test] +fn stream_tags_need_a_stream_decoder() { + let _guard = serial(); + let define_bytes = cbor_bytes(&define("a defined stream string")); + assert_eq!(decode(&define_bytes), Err(CborDecodeError::InvalidValue)); + let bytes_payload = cbor_bytes(&Value::Tag(6, Box::new(Value::Bytes(vec![])))); + assert_eq!(decode(&bytes_payload), Err(CborDecodeError::UnknownTag(6))); + let reference_bytes = cbor_bytes(&reference(0)); + assert_eq!(decode(&reference_bytes), Err(CborDecodeError::InvalidValue)); +} + +#[test] +fn typed_arrays_decode_as_before_in_stream_mode() { + let _guard = serial(); + let mut object = IObject::new(); + object + .insert( + "a long enough shared key", + IArray::try_from(vec![1.5f32, -2.5]).unwrap(), + ) + .unwrap(); + object + .insert( + "a long enough shared key too", + IArray::try_from(vec![7u16, 8, 9]).unwrap(), + ) + .unwrap(); + let value: IValue = object.into(); + let again = value.clone(); + let mut encoder = StreamEncoder::new(config()); + let (bytes, used) = encode_doc(&mut encoder, &value); + assert_eq!(used.defs, 2); + let decoded = StreamDecoder::new().decode(&bytes, true).unwrap(); + assert_eq!(decoded, value); + let object = decoded.as_object().unwrap(); + let array = |key: &str| object.get(key).unwrap().as_array().unwrap(); + assert!(matches!( + array("a long enough shared key").as_slice(), + ArraySliceRef::F32(_) + )); + assert!(matches!( + array("a long enough shared key too").as_slice(), + ArraySliceRef::U16(_) + )); + drop(again); + + // A tag that is neither a string tag nor a typed array is refused. + let unknown = cbor_bytes(&Value::Tag(0x10001, Box::new(Value::Integer(0.into())))); + assert_eq!( + StreamDecoder::new().decode(&unknown, true), + Err(CborDecodeError::InvalidValue) + ); +} + +/// Wraps `leaf` in `depth` single element arrays. +fn nested_arrays(leaf: IValue, depth: usize) -> IValue { + (0..depth).fold(leaf, |inner, _| { + IArray::try_from(vec![inner]).unwrap().into() + }) +} + +/// Wraps `leaf` in `depth` single entry objects that all use `key`. +fn nested_objects(key: &IString, leaf: IValue, depth: usize) -> IValue { + (0..depth).fold(leaf, |inner, _| { + let mut object = IObject::new(); + object.insert(key.clone(), inner).unwrap(); + object.into() + }) +} + +#[test] +fn nesting_limit_treats_definitions_and_references_like_inline_strings() { + let _guard = serial(); + let leaf = IString::from("a string at the bottom of the document"); + let key = IString::from("a shared key at every level of the document"); + let shallow: IValue = + IArray::try_from(vec![IValue::from(leaf.clone()), IValue::from(key.clone())]) + .unwrap() + .into(); + for depth in [126, 127, 128, 300] { + for limit_depth in [true, false] { + let cases = [ + ("leaf", nested_arrays(leaf.clone().into(), depth)), + ("key", nested_objects(&key, IValue::NULL, depth)), + ]; + for (what, doc) in cases { + let expected = decode_with_depth_limit(&encode(&doc), limit_depth); + assert_eq!( + expected.is_ok(), + !limit_depth || depth < 128, + "{what} {depth}" + ); + + // The document holds the first use of its strings. + let mut encoder = StreamEncoder::new(config()); + let (bytes, used) = encode_doc(&mut encoder, &doc); + assert!(used.defs > 0, "{what} {depth}"); + let mut decoder = StreamDecoder::new(); + assert_eq!( + decoder.decode(&bytes, limit_depth), + expected, + "define {what} {depth} {limit_depth}" + ); + if expected.is_ok() { + assert_eq!(decoder.len() as u64, used.defs, "{what} {depth}"); + } + + // A shallow document defined them, so this one only references. + let mut encoder = StreamEncoder::new(config()); + let mut decoder = StreamDecoder::new(); + let (bytes, _) = encode_doc(&mut encoder, &shallow); + decoder.decode(&bytes, true).unwrap(); + let (bytes, used) = encode_doc(&mut encoder, &doc); + assert!(used.refs > 0 && used.defs == 0, "{what} {depth}"); + assert_eq!( + decoder.decode(&bytes, limit_depth), + expected, + "reference {what} {depth} {limit_depth}" + ); + } + } + } +} + +fn string_pool(rng: &mut StdRng) -> Vec { + let mut pool: Vec = ["", "a", "key", "héllo", "1234567"] + .into_iter() + .map(IString::from) + .collect(); + for index in 0..40 { + let len = rng.gen_range(8..64); + pool.push(IString::from(format!("m{index}-{}", "é".repeat(len / 2)))); + } + for index in 0..8 { + let len = rng.gen_range(64..2112); + pool.push(IString::from(format!("l{index}-{}", "x".repeat(len)))); + } + pool +} + +fn random_value(rng: &mut StdRng, pool: &[IString], depth: usize) -> IValue { + match rng.gen_range(0..if depth >= 4 { 6 } else { 9 }) { + 0 => IValue::NULL, + 1 => rng.gen::().into(), + 2 => rng.gen_range(-1000..1000i64).into(), + 3 => (rng.gen_range(0..1000) as f64 / 8.0).into(), + 4 => pool[rng.gen_range(0..pool.len())].clone().into(), + 5 => IString::from(format!( + "unique string {} {}", + rng.gen::(), + rng.gen::() + )) + .into(), + 6 => { + let items: Vec = (0..rng.gen_range(0..6)) + .map(|_| random_value(rng, pool, depth + 1)) + .collect(); + IArray::try_from(items).unwrap().into() + } + _ => { + let mut object = IObject::new(); + for _ in 0..rng.gen_range(0..6) { + let key = pool[rng.gen_range(0..pool.len())].clone(); + object + .insert(key, random_value(rng, pool, depth + 1)) + .unwrap(); + } + object.into() + } + } +} + +#[test] +fn random_documents_round_trip_in_order() { + let _guard = serial(); + let configs = [ + config(), + StreamEncoderConfig { + table_budget_bytes: 600, + ..config() + }, + StreamEncoderConfig { + table_budget_bytes: 600_000, + ..config() + }, + ]; + for seed in 0..8 { + let mut rng = StdRng::seed_from_u64(seed); + let pool = string_pool(&mut rng); + let values: Vec = (0..60).map(|_| random_value(&mut rng, &pool, 0)).collect(); + for config in configs { + let mut encoder = StreamEncoder::new(config); + let docs = encode_stream(&mut encoder, &values); + assert!(docs.iter().any(Doc::carries_definitions)); + assert!(docs.iter().any(|doc| !doc.is_stream_doc())); + + let (decoded, mut decoder) = read_all(&docs.iter().collect::>()).unwrap(); + assert_eq!(decoded, values, "seed {seed} {config:?}"); + let stats = encoder.stats(); + assert_eq!(decoder.len() as u64, stats.defs); + assert_eq!(decoder.stats().defs, stats.defs); + assert_eq!(decoder.stats().refs, stats.refs); + assert_eq!(decoder.stats().def_bytes, stats.def_bytes); + + let forged = cbor_bytes(&reference(decoder.len() as u64)); + assert_eq!( + decoder.decode(&forged, true), + Err(CborDecodeError::InvalidStringReference) + ); + } + } +} + +#[test] +fn a_fresh_encoder_and_decoder_make_a_document_stand_alone() { + let _guard = serial(); + let value = json( + r#"[{"a repeated object key": 1}, {"a repeated object key": 2}, {"a repeated object key": 3}]"#, + ); + let mut bytes = Vec::new(); + let used = StreamEncoder::new(config()).encode_into(&mut bytes, &value); + assert_eq!(used, DocUse { defs: 1, refs: 2 }); + assert!(bytes.len() < encode(&value).len()); + assert_eq!(StreamDecoder::new().decode(&bytes, true).unwrap(), value); + // A second document written by another fresh encoder restarts its ids at 0. + let mut other = Vec::new(); + StreamEncoder::new(config()).encode_into(&mut other, &value); + assert_eq!(other, bytes); +} diff --git a/tests/cbor_stream_address_reuse.rs b/tests/cbor_stream_address_reuse.rs new file mode 100644 index 00000000..60ec72a7 --- /dev/null +++ b/tests/cbor_stream_address_reuse.rs @@ -0,0 +1,273 @@ +//! What a stream encoder sees when the address of a freed string is given to +//! another string. +//! +//! Whether a system allocator hands a freed block to the next allocation is up to +//! it, so these tests install one that always does: the block freed last goes to +//! the next allocation of the same layout. Reuse then happens every time instead +//! of when the allocator feels like it. + +use ijson::{ + encode, DocUse, IArray, IString, IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig, +}; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::{Cell, UnsafeCell}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +/// How many freed blocks are kept back for reuse; the others are freed at once. +const SLOTS: usize = 64; + +/// Freed blocks, the most recently freed last. +struct Freed { + len: usize, + blocks: [Option<(usize, Layout)>; SLOTS], +} + +impl Freed { + /// Takes back the block freed last that has exactly this layout. + fn take(&mut self, layout: Layout) -> Option<*mut u8> { + let index = (0..self.len).rev().find(|&index| { + self.blocks[index].is_some_and(|(_, freed_layout)| freed_layout == layout) + })?; + let (address, _) = self.blocks[index].take()?; + self.blocks.copy_within(index + 1..self.len, index); + self.len -= 1; + self.blocks[self.len] = None; + Some(address as *mut u8) + } + + /// Keeps a freed block, unless there is no room for it. + fn keep(&mut self, address: *mut u8, layout: Layout) -> bool { + if self.len == SLOTS { + return false; + } + self.blocks[self.len] = Some((address as usize, layout)); + self.len += 1; + true + } +} + +thread_local! { + // Only the thread that asked for reuse gets it. Others, such as the test + // harness, allocate at any time and would take the block that is waiting. + static RECYCLING: Cell = const { Cell::new(false) }; +} + +fn recycling() -> bool { + RECYCLING.try_with(Cell::get).unwrap_or(false) +} + +struct Recycler { + busy: AtomicBool, + freed: UnsafeCell, +} + +// SAFETY: `freed` is only reached through `locked`, which serializes the access. +unsafe impl Sync for Recycler {} + +impl Recycler { + fn locked(&self, access: impl FnOnce(&mut Freed) -> R) -> R { + while self.busy.swap(true, Ordering::Acquire) { + std::hint::spin_loop(); + } + // SAFETY: `busy` was false and is now ours, so no other thread holds a + // reference into `freed` until the store below. + let result = access(unsafe { &mut *self.freed.get() }); + self.busy.store(false, Ordering::Release); + result + } + + fn free_kept_blocks(&self) { + self.locked(|freed| { + for block in freed.blocks[..freed.len] + .iter_mut() + .filter_map(Option::take) + { + // SAFETY: the block came from `System.alloc` with this layout and + // was kept back instead of being freed. + unsafe { System.dealloc(block.0 as *mut u8, block.1) }; + } + freed.len = 0; + }); + } +} + +// SAFETY: every block comes from `System` and goes back to it with the layout it +// was allocated with; blocks kept back are handed out only for that same layout. +unsafe impl GlobalAlloc for Recycler { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if recycling() { + if let Some(block) = self.locked(|freed| freed.take(layout)) { + return block; + } + } + System.alloc(layout) + } + + unsafe fn dealloc(&self, address: *mut u8, layout: Layout) { + if recycling() && self.locked(|freed| freed.keep(address, layout)) { + return; + } + System.dealloc(address, layout) + } +} + +#[global_allocator] +static ALLOCATOR: Recycler = Recycler { + busy: AtomicBool::new(false), + freed: UnsafeCell::new(Freed { + len: 0, + blocks: [None; SLOTS], + }), +}; + +/// Reuse of freed blocks on this thread lasts as long as this value lives. +struct Recycling; + +impl Recycling { + fn start() -> Self { + RECYCLING.with(|recycling| recycling.set(true)); + Recycling + } +} + +impl Drop for Recycling { + fn drop(&mut self) { + RECYCLING.with(|recycling| recycling.set(false)); + ALLOCATOR.free_kept_blocks(); + } +} + +// Serializes the tests: another thread's allocations could take the freed block. +static TEST_LOCK: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + // One failing test must not make the others fail on a poisoned lock. + TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[ctor::ctor] +fn init_cache() { + ijson::init_shared_string_cache(true).unwrap(); +} + +fn config() -> StreamEncoderConfig { + StreamEncoderConfig { + table_budget_bytes: 8 << 20, + } +} + +/// An array that holds a clone of each of `strings`, in order. Each clone is one +/// more owner of its string. +fn array_of(strings: &[&IString]) -> IValue { + let items: Vec = strings.iter().map(|s| IValue::from((*s).clone())).collect(); + IArray::try_from(items).unwrap().into() +} + +/// Strings of 33 to 40 bytes need blocks of one layout, so a block freed by one +/// of them is handed to any other. +const FIRST_LEN: usize = 33; + +/// An encoder that defined a string and whose string was freed afterwards, and a +/// new string that was given the freed address. +struct Reused { + encoder: StreamEncoder, + first_bytes: Vec, + /// The text that was defined. + old_text: String, + intruder: IString, + /// Whether the intruder sits where the defined string did. + at_old_address: bool, +} + +fn free_and_reuse(intruder_len: usize) -> Reused { + let mut encoder = StreamEncoder::new(config()); + let old_text = "o".repeat(FIRST_LEN); + let original = IString::from(old_text.as_str()); + let address = original.as_ptr(); + let first = array_of(&[&original, &original]); + let mut first_bytes = Vec::new(); + let used = encoder.encode_into(&mut first_bytes, &first); + assert_eq!(used, DocUse { defs: 1, refs: 1 }); + + // The document goes first and the string last, so that its block is the one + // the allocator hands out next. + drop(first); + drop(original); + let intruder = IString::from("i".repeat(intruder_len)); + Reused { + encoder, + first_bytes, + old_text, + at_old_address: intruder.as_ptr() == address, + intruder, + } +} + +#[test] +fn a_reused_address_with_another_length_poisons_the_encoder() { + let _guard = serial(); + let _recycling = Recycling::start(); + let mut reused = free_and_reuse(40); + assert!(reused.at_old_address); + + let second = array_of(&[&reused.intruder, &reused.intruder]); + let mut second_bytes = Vec::new(); + let used = reused.encoder.encode_into(&mut second_bytes, &second); + assert_eq!(used, DocUse::default()); + assert_eq!(second_bytes, encode(&second)); + assert!(reused.encoder.is_poisoned()); + let mut decoder = StreamDecoder::new(); + decoder.decode(&reused.first_bytes, true).unwrap(); + assert_eq!(decoder.decode(&second_bytes, true).unwrap(), second); +} + +#[test] +fn a_stale_hit_after_a_definition_in_the_same_document_keeps_the_definition() { + let _guard = serial(); + let _recycling = Recycling::start(); + let mut reused = free_and_reuse(40); + assert!(reused.at_old_address); + + // The new string needs two owners to be defined, and is met before the + // address that was reused. + let fresh = IString::from("a string defined before the stale hit"); + let second = array_of(&[&fresh, &reused.intruder, &fresh]); + let mut second_bytes = vec![2, 0, 0, 0, 0]; + let used = reused.encoder.encode_into(&mut second_bytes, &second); + // The definition written before the stale hit stays; everything after it is inline. + assert_eq!(used, DocUse { defs: 1, refs: 0 }); + assert_eq!(second_bytes[..5], [2, 0, 0, 0, 0]); + assert_eq!(reused.encoder.stats().defs, 2); + assert!(reused.encoder.is_poisoned()); + let mut decoder = StreamDecoder::new(); + decoder.decode(&reused.first_bytes, true).unwrap(); + assert_eq!(decoder.decode(&second_bytes[5..], true).unwrap(), second); + assert_eq!(decoder.len(), 2); +} + +// Known limit: the encoder sees a reused address only when the new string has another +// length. Callers keep every defined string alive until the stream ends, as `StreamEncoder` +// documents. +#[test] +fn a_reused_address_with_the_same_length_is_not_detected() { + let _guard = serial(); + let _recycling = Recycling::start(); + let mut reused = free_and_reuse(FIRST_LEN); + assert!(reused.at_old_address); + + let second = array_of(&[&reused.intruder, &reused.intruder]); + let mut second_bytes = Vec::new(); + let used = reused.encoder.encode_into(&mut second_bytes, &second); + assert_eq!(used, DocUse { defs: 0, refs: 2 }); + assert_eq!(second_bytes, [0x82, 0xC7, 0x00, 0xC7, 0x00]); + assert!(!reused.encoder.is_poisoned()); + + // The reader resolves the reference to the string that was freed. + let mut decoder = StreamDecoder::new(); + decoder.decode(&reused.first_bytes, true).unwrap(); + let decoded = decoder.decode(&second_bytes, true).unwrap(); + let old = IString::from(reused.old_text.as_str()); + assert_eq!(decoded, array_of(&[&old, &old])); + assert_ne!(decoded, second); +} diff --git a/tests/cbor_stream_read_only.rs b/tests/cbor_stream_read_only.rs new file mode 100644 index 00000000..6b7a440f --- /dev/null +++ b/tests/cbor_stream_read_only.rs @@ -0,0 +1,230 @@ +//! An encoder must never write to a string it encodes, so +//! that a forked child working on its parent's pages copies none of them. +//! +//! Every string here gets a mapping of its own. The child makes the first page of +//! each mapping, which holds the string's header, read-only and then encodes: a +//! write to a header ends it. Cloning a string outside the encoder updates its owner +//! count, so it serves as the control that the child does notice a write. +#![cfg(all( + any(target_os = "linux", target_os = "macos"), + target_pointer_width = "64" +))] + +use ijson::{IObject, IString, IValue, StreamEncoder, StreamEncoderConfig}; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::ffi::c_void; +use std::ptr::null_mut; + +// std links libc already; only these calls and constants are needed. +extern "C" { + fn mmap( + address: *mut c_void, + len: usize, + protection: i32, + flags: i32, + file: i32, + offset: i64, + ) -> *mut c_void; + fn munmap(address: *mut c_void, len: usize) -> i32; + fn mprotect(address: *mut c_void, len: usize, protection: i32) -> i32; + fn sysconf(name: i32) -> i64; + fn signal(signal: i32, handler: usize) -> usize; + fn fork() -> i32; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + fn _exit(code: i32) -> !; +} + +const PROT_READ: i32 = 1; +const PROT_WRITE: i32 = 2; +const MAP_PRIVATE: i32 = 2; +const MAP_FAILED: *mut c_void = !0 as *mut c_void; +const SIGSEGV: i32 = 11; +#[cfg(target_os = "linux")] +const MAP_ANONYMOUS: i32 = 0x20; +#[cfg(target_os = "linux")] +const SC_PAGESIZE: i32 = 30; +#[cfg(target_os = "linux")] +const SIGBUS: i32 = 7; +#[cfg(target_os = "macos")] +const MAP_ANONYMOUS: i32 = 0x1000; +#[cfg(target_os = "macos")] +const SC_PAGESIZE: i32 = 29; +#[cfg(target_os = "macos")] +const SIGBUS: i32 = 10; + +fn page_size() -> usize { + // SAFETY: `sysconf` only reads its integer argument. + unsafe { sysconf(SC_PAGESIZE) as usize } +} + +fn round_up_to_pages(len: usize) -> usize { + let page = page_size(); + (len + page - 1) / page * page +} + +/// Allocations from this size on get a mapping of their own, so that the page +/// that holds a string header holds nothing else. +const MAPPED_FROM: usize = 8192; + +fn gets_own_mapping(layout: Layout) -> bool { + layout.size() >= MAPPED_FROM && layout.align() <= page_size() +} + +struct Mappings; + +// SAFETY: a block is either a fresh private mapping of its rounded size, which +// `dealloc` unmaps again from the same layout, or comes from `System` and goes +// back to it. +unsafe impl GlobalAlloc for Mappings { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if !gets_own_mapping(layout) { + return System.alloc(layout); + } + let address = mmap( + null_mut(), + round_up_to_pages(layout.size()), + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0, + ); + if address == MAP_FAILED { + null_mut() + } else { + address.cast() + } + } + + unsafe fn dealloc(&self, address: *mut u8, layout: Layout) { + if gets_own_mapping(layout) { + munmap(address.cast(), round_up_to_pages(layout.size())); + } else { + System.dealloc(address, layout) + } + } +} + +#[global_allocator] +static ALLOCATOR: Mappings = Mappings; + +#[ctor::ctor] +fn init_cache() { + ijson::init_shared_string_cache(true).unwrap(); +} + +const DOCS: usize = 128; +const STRING_BYTES: usize = 20 * 1024; + +const EXIT_OK: i32 = 0; +const EXIT_WROTE_TO_A_STRING: i32 = 1; +const EXIT_PROTECT_FAILED: i32 = 2; +const EXIT_UNEXPECTED_OUTPUT: i32 = 3; + +extern "C" fn on_fault(_signal: i32) { + // SAFETY: `_exit` may be called from a signal handler and ends the process + // at once. + unsafe { _exit(EXIT_WROTE_TO_A_STRING) } +} + +/// Runs in the forked child and returns its exit code. +fn encode_in_child( + clone_control: bool, + strings: &[IString], + docs: &[IValue], + string_pages: &[usize], +) -> i32 { + // SAFETY: the handler only calls `_exit`. + unsafe { + signal(SIGSEGV, on_fault as usize); + signal(SIGBUS, on_fault as usize); + } + for &page in string_pages { + // SAFETY: `page` is the start of a mapping of this process that holds a + // string, and nothing else lives on its first page. + if unsafe { mprotect(page as *mut c_void, page_size(), PROT_READ) } != 0 { + return EXIT_PROTECT_FAILED; + } + } + if clone_control { + std::mem::forget(strings[0].clone()); + } + let mut encoder = StreamEncoder::new(StreamEncoderConfig { + table_budget_bytes: 64 << 20, + }); + let mut out = Vec::new(); + for doc in docs { + out.clear(); + encoder.encode_into(&mut out, doc); + } + let stats = encoder.stats(); + if stats.defs == DOCS as u64 && stats.refs == DOCS as u64 { + EXIT_OK + } else { + EXIT_UNEXPECTED_OUTPUT + } +} + +/// Forks, runs the child and returns its exit code, or `None` when a signal ended it. +fn run_child( + clone_control: bool, + strings: &[IString], + docs: &[IValue], + string_pages: &[usize], +) -> Option { + // SAFETY: the child encodes, then leaves through `_exit` without running the + // parent's test harness or any destructor, which would write to the strings. + let pid = unsafe { fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + let code = encode_in_child(clone_control, strings, docs, string_pages); + // SAFETY: `_exit` is always safe to call; it ends the child immediately. + unsafe { _exit(code) }; + } + let mut status = 0; + // SAFETY: `status` is a valid pointer and `pid` is a child of this process. + let waited = unsafe { waitpid(pid, &mut status, 0) }; + assert_eq!(waited, pid, "waitpid failed"); + (status & 0x7f == 0).then_some((status >> 8) & 0xff) +} + +#[test] +fn an_encoder_never_writes_to_a_string() { + // Distinct 20 KiB strings: each one has a mapping of its own. + let strings: Vec = (0..DOCS) + .map(|index| IString::from(format!("{index:08}{}", "x".repeat(STRING_BYTES - 8)))) + .collect(); + // Each string is the "a" entry of one document and the "b" entry of the + // previous one, so it is defined once and referenced once. + let docs: Vec = (0..DOCS) + .map(|index| { + let mut object = IObject::new(); + object.insert("a", strings[index].clone()).unwrap(); + object + .insert("b", strings[(index + 1) % DOCS].clone()) + .unwrap(); + IValue::from(object) + }) + .collect(); + let string_pages: Vec = strings + .iter() + .map(|string| { + let bytes = string.as_ptr() as usize; + let page = bytes & !(page_size() - 1); + // The header, 8 or 16 bytes, sits at the start of the mapping. + assert!(bytes - page <= 16, "a string does not start its mapping"); + page + }) + .collect(); + + assert_eq!( + run_child(false, &strings, &docs, &string_pages), + Some(EXIT_OK), + "the encoder wrote to a string" + ); + // The control: holding handles updates owner counts, which the child must notice. + assert_eq!( + run_child(true, &strings, &docs, &string_pages), + Some(EXIT_WROTE_TO_A_STRING), + "the child did not notice a write to a read-only string" + ); +} diff --git a/tests/cbor_stream_reset.rs b/tests/cbor_stream_reset.rs new file mode 100644 index 00000000..42c2e4a7 --- /dev/null +++ b/tests/cbor_stream_reset.rs @@ -0,0 +1,103 @@ +use ijson::{IArray, IString, IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig}; + +#[ctor::ctor] +fn init_cache() { + ijson::init_shared_string_cache(true).unwrap(); +} + +fn config(budget: usize) -> StreamEncoderConfig { + StreamEncoderConfig { + table_budget_bytes: budget, + } +} + +fn repeated(text: &IString) -> IValue { + IArray::try_from(vec![IValue::from(text.clone()), text.clone().into()]) + .unwrap() + .into() +} + +#[test] +fn reset_reuses_ids_and_reader_handles_but_preserves_statistics() { + let owners: Vec<_> = (0..100) + .map(|i| IString::from(format!("reset string {i:04}"))) + .collect(); + let mut encoder = StreamEncoder::new(config(84)); + let mut decoder = StreamDecoder::new(); + let mut resets = 0; + let mut loaded = Vec::new(); + for owner in &owners { + if encoder.reset_if_needed() { + decoder.reset(); + assert_eq!(encoder.next_id(), 0); + assert!(decoder.is_empty()); + resets += 1; + } + let value = repeated(owner); + let mut bytes = Vec::new(); + let used = encoder.encode_into(&mut bytes, &value); + loaded.push(decoder.decode(&bytes, true).unwrap()); + assert_eq!(loaded.last(), Some(&value)); + assert!(encoder.next_id() <= 3); + assert!(decoder.len() <= 3); + assert!(used.defs <= 1); + } + assert!(resets > 0); + assert!(encoder.stats().defs > 3); + assert_eq!(decoder.stats().defs, encoder.stats().defs); + assert_eq!(decoder.stats().def_bytes, encoder.stats().def_bytes); + assert_eq!(loaded, owners.iter().map(repeated).collect::>()); +} + +#[test] +fn a_full_table_requests_a_reset_only_after_a_miss_and_a_poisoned_encoder_never_does() { + let owners: Vec<_> = (0..4) + .map(|i| IString::from(format!("budget string {i:04}"))) + .collect(); + for poisoned in [false, true] { + let mut encoder = StreamEncoder::new(config(84)); + for owner in &owners[..3] { + encoder.encode_into(&mut Vec::new(), &repeated(owner)); + } + for owner in &owners[..3] { + assert!(!encoder.reset_if_needed()); + assert_eq!( + encoder.encode_into(&mut Vec::new(), &repeated(owner)).refs, + 2 + ); + } + encoder.encode_into(&mut Vec::new(), &repeated(&owners[3])); + if poisoned { + encoder.poison(); + } + assert_eq!(encoder.reset_if_needed(), !poisoned); + } +} + +#[test] +fn unusable_budget_does_not_request_resets() { + let owners: Vec<_> = (0..4).map(|i| IString::from(format!("{i:016}"))).collect(); + for budget in [0, 1, 83] { + let mut encoder = StreamEncoder::new(config(budget)); + for owner in &owners { + encoder.encode_into(&mut Vec::new(), &repeated(owner)); + assert!(!encoder.reset_if_needed()); + } + assert_eq!(encoder.stats().defs, 0); + } +} + +#[test] +fn a_failed_decoder_refuses_every_later_document_even_after_a_reset() { + let mut decoder = StreamDecoder::new(); + let define_one = [0xc6, 0x74] + .into_iter() + .chain(*b"the first definition") + .collect::>(); + decoder.decode(&define_one, true).unwrap(); + assert!(decoder.decode(&[0xc7, 9], true).is_err()); + // Later documents could resolve ids the failed one was meant to define. + assert!(decoder.decode(&[0xc7, 0], true).is_err()); + decoder.reset(); + assert!(decoder.decode(&[0xf6], true).is_err()); +} diff --git a/tests/cbor_stream_size_gates.rs b/tests/cbor_stream_size_gates.rs new file mode 100644 index 00000000..e5cfed8a --- /dev/null +++ b/tests/cbor_stream_size_gates.rs @@ -0,0 +1,181 @@ +//! Size and round-trip checks of the stream writer on small corpora. +//! +//! A total counts what a module writes per record before compression: the 1 byte marker +//! every record starts with, the 4 byte base ID of a stream record, and the CBOR payload. +//! Ratios hold at any size while the table budget does not bind, so these are small versions +//! of the corpora a live-server benchmark loads. Documents are shuffled because Redis saves in +//! hash order. Every corpus also has to decode back to the same documents, record by record, +//! as a loader would, with the default budget and with a tiny one. +use ijson::{ + DocUse, IArray, IObject, IString, IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig, +}; +use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +// Serializes the tests: they look at owner counts and hold the corpora in memory. +static TEST_LOCK: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + // One failing test must not make the others fail on a poisoned lock. + TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[ctor::ctor] +fn init_cache() { + ijson::init_shared_string_cache(true).unwrap(); +} + +// The marker byte and the base ID that a stream record starts with. +const STREAM_HEADER_LEN: usize = 5; +// What the module writes with: a 256 MiB address table budget in both the saving +// process and its forked children. +const TABLE_BUDGET: usize = 256 << 20; +// Small enough to freeze after a few dozen strings. +const TINY_BUDGET: usize = 2048; +// Ids from this value on need a four byte CBOR integer. +const FOUR_BYTE_IDS: u64 = 65_536; + +/// What a writer produced for a corpus. +struct Written { + bytes: u64, + stream_docs: u64, + defs: u64, +} + +impl Written { + fn ratio(&self, inline: u64) -> f64 { + self.bytes as f64 / inline as f64 + } +} + +fn config(table_budget_bytes: usize) -> StreamEncoderConfig { + StreamEncoderConfig { table_budget_bytes } +} + +fn inline_bytes(docs: &[IValue]) -> u64 { + docs.iter() + .map(|doc| 1 + ijson::encode(doc).len() as u64) + .sum() +} + +/// Writes the documents in order the way the module frames them, decodes every record the way a +/// loader does and checks that it equals its document. +fn write_corpus(docs: &[IValue], config: StreamEncoderConfig) -> Written { + let mut encoder = StreamEncoder::new(config); + let mut decoder = StreamDecoder::new(); + let (mut bytes, mut stream_docs) = (0, 0); + for (index, doc) in docs.iter().enumerate() { + let base_id = encoder.next_id(); + let mut record = vec![0; STREAM_HEADER_LEN]; + if encoder.encode_into(&mut record, doc) == DocUse::default() { + // Self-contained: the marker is the only framing. + bytes += (record.len() - (STREAM_HEADER_LEN - 1)) as u64; + } else { + assert_eq!(base_id as usize, decoder.len(), "document {index}"); + bytes += record.len() as u64; + stream_docs += 1; + } + let decoded = decoder + .decode(&record[STREAM_HEADER_LEN..], true) + .unwrap_or_else(|error| panic!("document {index}: {error}")); + assert!(decoded == *doc, "document {index} did not round trip"); + } + Written { + bytes, + stream_docs, + defs: encoder.stats().defs, + } +} + +/// Writes the corpus with the default and with a tiny budget. Both stay inside the worst case: +/// 1 byte per definition and 4 bytes of base ID per stream record more than the same documents +/// written inline. The inline and default totals come back for the ratios. +fn measure(docs: &[IValue]) -> (u64, Written) { + let inline = inline_bytes(docs); + let default = write_corpus(docs, config(TABLE_BUDGET)); + let tiny = write_corpus(docs, config(TINY_BUDGET)); + for written in [&default, &tiny] { + let worst = inline + 4 * written.stream_docs + written.defs; + assert!( + written.bytes <= worst, + "{} bytes, worst case {worst}", + written.bytes + ); + } + (inline, default) +} + +/// Fisher-Yates, seeded, standing in for the hash order Redis saves in. +fn scatter(docs: &mut [IValue]) { + docs.shuffle(&mut StdRng::seed_from_u64(217_194)); +} + +fn text(tag: &str, index: usize, len: usize) -> IString { + let mut text = format!("{tag}-{index:08}-"); + text.push_str(&"x".repeat(len - text.len())); + IString::from(text) +} + +fn object(entries: Vec<(&str, IValue)>) -> IValue { + let mut object = IObject::new(); + for (key, value) in entries { + object.insert(key, value).unwrap(); + } + object.into() +} + +fn array(items: impl Iterator) -> IValue { + IArray::try_from(items.map(IValue::from).collect::>()) + .unwrap() + .into() +} + +/// More distinct 1 KiB strings than a two byte id can name, 256 to a document, each document +/// stored 4 times. +fn cardinality() -> Vec { + let strings: Vec = (0..258 * 256).map(|i| text("card", i, 1024)).collect(); + let mut docs = Vec::new(); + for (group, items) in strings.chunks(256).enumerate() { + for copy in 0..4 { + docs.push(object(vec![ + ("group", (group as i64).into()), + ("copy", copy.into()), + ("items", array(items.iter().cloned())), + ])); + } + } + scatter(&mut docs); + docs +} + +/// 16 large strings, each in all 4 documents. +fn pool() -> Vec { + let strings: Vec = (0..16).map(|i| text("pool", i, 256 * 1024)).collect(); + let mut docs: Vec = (0..4) + .map(|copy| { + object(vec![ + ("copy", copy.into()), + ("pool", array(strings.iter().cloned())), + ]) + }) + .collect(); + scatter(&mut docs); + docs +} + +#[test] +fn ids_past_65535_round_trip_and_shrink_the_corpus() { + let _guard = serial(); + let (inline, written) = measure(&cardinality()); + assert!(written.defs > FOUR_BYTE_IDS, "{} entries", written.defs); + let ratio = written.ratio(inline); + assert!(ratio <= 0.30, "{ratio:.4} of inline"); +} + +#[test] +fn strings_above_64_kib_round_trip_and_shrink_a_fourfold_pool() { + let _guard = serial(); + let (inline, written) = measure(&pool()); + let ratio = written.ratio(inline); + assert!(ratio <= 0.27, "{ratio:.4} of inline"); +} diff --git a/tests/cbor_streaming_memory.rs b/tests/cbor_streaming_memory.rs new file mode 100644 index 00000000..9e846709 --- /dev/null +++ b/tests/cbor_streaming_memory.rs @@ -0,0 +1,145 @@ +use ijson::{ + decode, encode, IArray, IString, IValue, StreamDecoder, StreamEncoder, StreamEncoderConfig, +}; +use mockalloc::{record_allocs, Mockalloc}; +use std::alloc::System; + +#[global_allocator] +static ALLOCATOR: Mockalloc = Mockalloc(System); + +fn stream_config() -> StreamEncoderConfig { + StreamEncoderConfig { + table_budget_bytes: 64 << 20, + } +} + +/// An array whose elements all share one string. +fn shared_elements(count: usize) -> IValue { + let shared = IString::from("a string shared by every element of the array"); + IArray::try_from(vec![IValue::from(shared); count]) + .unwrap() + .into() +} + +#[test] +fn encoding_borrows_large_string_content() { + let value: IValue = IString::from("x".repeat(4 * 1024 * 1024)).into(); + let allocations = record_allocs(|| { + let bytes = encode(&value); + assert!(bytes.len() > 4 * 1024 * 1024); + std::hint::black_box(&bytes); + }); + assert!( + allocations.peak_mem() < 6 * 1024 * 1024, + "encoding retained an extra string copy: {} bytes", + allocations.peak_mem() + ); +} + +#[test] +fn encoding_borrows_array_elements() { + let value: IValue = IArray::try_from(vec![IValue::NULL; 262_144]) + .unwrap() + .into(); + let allocations = record_allocs(|| { + let bytes = encode(&value); + assert!(bytes.len() > 262_144); + std::hint::black_box(&bytes); + }); + assert!( + allocations.peak_mem() < 1024 * 1024, + "encoding retained an intermediate array tree: {} bytes", + allocations.peak_mem() + ); +} + +#[test] +fn decoding_builds_array_without_an_intermediate_tree() { + let value: IValue = IArray::try_from(vec![IValue::NULL; 262_144]) + .unwrap() + .into(); + let bytes = encode(&value); + let allocations = record_allocs(|| { + let decoded = decode(&bytes).unwrap(); + assert_eq!(decoded, value); + }); + assert!( + allocations.peak_mem() < 4 * 1024 * 1024, + "decoding retained an intermediate tree: {} bytes", + allocations.peak_mem() + ); +} + +#[test] +fn stream_encoding_borrows_large_string_content() { + let shared = IString::from("x".repeat(4 * 1024 * 1024)); + let value: IValue = shared.clone().into(); + let mut encoder = StreamEncoder::new(stream_config()); + let allocations = record_allocs(|| { + let mut out = Vec::new(); + assert_eq!(encoder.encode_into(&mut out, &value).defs, 1); + assert!(out.len() > 4 * 1024 * 1024); + std::hint::black_box(&out); + }); + assert!( + allocations.peak_mem() < 6 * 1024 * 1024, + "defining retained an extra string copy: {} bytes", + allocations.peak_mem() + ); + + let allocations = record_allocs(|| { + let mut out = Vec::new(); + assert_eq!(encoder.encode_into(&mut out, &value).refs, 1); + assert_eq!(out.len(), 2); + }); + assert!( + allocations.peak_mem() < 64, + "referencing allocated {} bytes", + allocations.peak_mem() + ); + drop(shared); +} + +#[test] +fn stream_encoding_of_shared_strings_needs_less_memory_than_plain_encoding() { + let value = shared_elements(262_144); + let plain = record_allocs(|| { + std::hint::black_box(encode(&value)); + }); + let mut encoder = StreamEncoder::new(stream_config()); + let stream = record_allocs(|| { + let mut out = Vec::new(); + let used = encoder.encode_into(&mut out, &value); + assert_eq!((used.defs, used.refs), (1, 262_143)); + std::hint::black_box(&out); + }); + assert!( + stream.peak_mem() < plain.peak_mem() / 8, + "stream {} bytes, plain {} bytes", + stream.peak_mem(), + plain.peak_mem() + ); +} + +#[test] +fn stream_decoding_builds_array_without_an_intermediate_tree() { + let value = shared_elements(262_144); + let mut bytes = Vec::new(); + StreamEncoder::new(stream_config()).encode_into(&mut bytes, &value); + let inline = encode(&value); + let plain = record_allocs(|| { + assert_eq!(decode(&inline).unwrap(), value); + }); + let mut decoder = StreamDecoder::new(); + let stream = record_allocs(|| { + assert_eq!(decoder.decode(&bytes, true).unwrap(), value); + }); + // The stream pays for one chunk of the ID table, 64Ki handles of 8 bytes. + let chunk = 512 * 1024; + assert!( + stream.peak_mem() <= plain.peak_mem() + chunk + 64 * 1024, + "stream {} bytes, plain {} bytes", + stream.peak_mem(), + plain.peak_mem() + ); +} diff --git a/tests/memory_share.rs b/tests/memory_share.rs index f62e8709..e2c0446f 100644 --- a/tests/memory_share.rs +++ b/tests/memory_share.rs @@ -26,6 +26,11 @@ fn proportional_memory_across_documents() { for json in ["null", "true", "[]", "{}", r#""short""#] { let value: IValue = serde_json::from_str(json).unwrap(); - assert_eq!(value.mem_allocated(), 0.0); + let expected = if cfg!(feature = "thread_safe") && value.is_string() { + 24.0 + } else { + 0.0 + }; + assert_eq!(value.mem_allocated(), expected); } } diff --git a/tests/string_cache_reinit.rs b/tests/string_cache_reinit.rs new file mode 100644 index 00000000..82e95068 --- /dev/null +++ b/tests/string_cache_reinit.rs @@ -0,0 +1,16 @@ +use ijson::IString; + +// Dropping the last handle of a string from before a reset must not evict its replacement. +#[test] +fn a_string_kept_across_a_cache_reset_is_not_the_one_interned_afterwards() { + ijson::init_shared_string_cache(true).unwrap(); + let before_reset = IString::from("string retained across cache reset"); + ijson::reinit_shared_string_cache(); + let after_reset = IString::from(before_reset.as_str()); + assert_ne!(before_reset.as_ptr(), after_reset.as_ptr()); + drop(before_reset); + assert_eq!( + IString::from(after_reset.as_str()).as_ptr(), + after_reset.as_ptr() + ); +} diff --git a/tests/string_cache_reinit_equality.rs b/tests/string_cache_reinit_equality.rs new file mode 100644 index 00000000..be9caf7c --- /dev/null +++ b/tests/string_cache_reinit_equality.rs @@ -0,0 +1,65 @@ +use std::collections::hash_map::DefaultHasher; +use std::collections::HashSet; +use std::hash::{Hash, Hasher}; + +use ijson::{IObject, IString, IValue}; + +// A string from before a cache reset and the same text interned afterwards are two allocations. +// They must still compare, order and hash as the same string. + +fn init_cache() { + // Another test in this binary may have initialized it already. + let _ = ijson::init_shared_string_cache(true); +} + +fn hash_of(value: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() +} + +#[test] +fn retained_object_keys_still_match_after_cache_reset() { + init_cache(); + let key = IString::from("retained object key"); + let mut object = IObject::new(); + object.insert(key.clone(), 42).unwrap(); + + ijson::reinit_shared_string_cache(); + + let replacement = IString::from("retained object key"); + assert_eq!(object.get(&replacement), Some(&IValue::from(42))); +} + +#[test] +fn a_string_decoded_after_cache_reset_equals_the_retained_one() { + init_cache(); + let retained = IValue::from("retained string across encode and decode"); + let bytes = ijson::encode(&retained); + + ijson::reinit_shared_string_cache(); + + let decoded = ijson::decode(&bytes).unwrap(); + assert_eq!(decoded, retained); + assert_eq!( + decoded.partial_cmp(&retained), + Some(std::cmp::Ordering::Equal) + ); + assert_eq!(hash_of(&decoded), hash_of(&retained)); +} + +#[test] +fn equal_strings_from_either_side_of_a_cache_reset_hash_and_order_the_same() { + init_cache(); + let before = IString::from("string interned on both sides of a reset"); + + ijson::reinit_shared_string_cache(); + + let after = IString::from(before.as_str()); + assert_eq!(before, after); + assert_eq!(before.cmp(&after), std::cmp::Ordering::Equal); + assert_eq!(hash_of(&before), hash_of(&after)); + + let values: HashSet = [IValue::from(before)].into_iter().collect(); + assert!(values.contains(&IValue::from(after))); +}