icu_provider_blob/
blob_data_provider.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use crate::blob_schema::BlobSchema;
6use icu_provider::buf::BufferFormat;
7use icu_provider::prelude::*;
8use icu_provider::Cart;
9use icu_provider::DynamicDryDataProvider;
10use yoke::*;
11
12/// A data provider that reads from serialized blobs of data.
13///
14/// This enables data blobs to be read from arbitrary sources at runtime, allowing code and data
15/// to be separated. Alternatively, blobs can also be statically included at compile time.
16///
17/// [`BlobDataProvider`] implements [`BufferProvider`], so it can be used in
18/// `*_with_buffer_provider` constructors across ICU4X.
19///
20/// # `Sync + Send`
21///
22/// This provider uses reference counting internally. When the `sync` Cargo feature on the [`icu_provider`]
23/// crate is enabled, it uses [`Arc`](alloc::sync::Arc) instead of [`Rc`](alloc::rc::Rc), making
24/// it `Sync + Send`.
25///
26/// # Examples
27///
28/// ## Dynamic loading
29///
30/// Load "hello world" data from a postcard blob loaded at runtime:
31///
32/// ```
33/// use icu_locale_core::locale;
34/// use icu_provider::hello_world::HelloWorldFormatter;
35/// use icu_provider_blob::BlobDataProvider;
36/// use writeable::assert_writeable_eq;
37///
38/// // Read an ICU4X data blob dynamically:
39/// let blob = std::fs::read("tests/data/v3.postcard")
40///     .expect("Reading pre-computed postcard buffer");
41///
42/// // Create a DataProvider from it:
43/// let provider = BlobDataProvider::try_new_from_blob(blob.into_boxed_slice())
44///     .expect("Deserialization should succeed");
45///
46/// // Check that it works:
47/// let formatter = HelloWorldFormatter::try_new_with_buffer_provider(
48///     &provider,
49///     locale!("la").into(),
50/// )
51/// .expect("locale exists");
52///
53/// assert_writeable_eq!(formatter.format(), "Ave, munde");
54/// ```
55///
56/// ## Static loading
57///
58/// Load "hello world" data from a postcard blob statically linked at compile time:
59///
60/// ```
61/// use icu_locale_core::locale;
62/// use icu_provider::hello_world::HelloWorldFormatter;
63/// use icu_provider_blob::BlobDataProvider;
64/// use writeable::assert_writeable_eq;
65///
66/// // Read an ICU4X data blob statically:
67/// const HELLO_WORLD_BLOB: &[u8] = include_bytes!("../tests/data/v3.postcard");
68///
69/// // Create a DataProvider from it:
70/// let provider = BlobDataProvider::try_new_from_static_blob(HELLO_WORLD_BLOB)
71///     .expect("Deserialization should succeed");
72///
73/// // Check that it works:
74/// let formatter = HelloWorldFormatter::try_new_with_buffer_provider(
75///     &provider,
76///     locale!("la").into(),
77/// )
78/// .expect("locale exists");
79///
80/// assert_writeable_eq!(formatter.format(), "Ave, munde");
81/// ```
82#[derive(Clone)]
83pub struct BlobDataProvider {
84    pub(crate) data: Yoke<BlobSchema<'static>, Option<Cart>>,
85}
86
87impl core::fmt::Debug for BlobDataProvider {
88    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
89        f.debug_struct("BlobDataProvider")
90            .field("data", &"[...]")
91            .finish()
92    }
93}
94
95impl BlobDataProvider {
96    /// Create a [`BlobDataProvider`] from a blob of ICU4X data.
97    ///
98    /// ✨ *Enabled with the `alloc` Cargo feature.*
99    #[cfg(feature = "alloc")]
100    pub fn try_new_from_blob(blob: alloc::boxed::Box<[u8]>) -> Result<Self, DataError> {
101        Ok(Self {
102            data: Cart::try_make_yoke(blob, |bytes| {
103                BlobSchema::deserialize_and_check(&mut postcard::Deserializer::from_bytes(bytes))
104            })?,
105        })
106    }
107
108    /// Create a [`BlobDataProvider`] from a static blob. This is a special case of
109    /// [`try_new_from_blob`](BlobDataProvider::try_new_from_blob) and is allocation-free.
110    pub fn try_new_from_static_blob(blob: &'static [u8]) -> Result<Self, DataError> {
111        Ok(Self {
112            data: Yoke::new_owned(BlobSchema::deserialize_and_check(
113                &mut postcard::Deserializer::from_bytes(blob),
114            )?),
115        })
116    }
117
118    #[doc(hidden)] // for testing purposes only: checks if it is using the Bigger format
119    pub fn internal_is_using_bigger_format(&self) -> bool {
120        matches!(self.data.get(), BlobSchema::V003Bigger(..))
121    }
122}
123
124impl DynamicDataProvider<BufferMarker> for BlobDataProvider {
125    fn load_data(
126        &self,
127        marker: DataMarkerInfo,
128        req: DataRequest,
129    ) -> Result<DataResponse<BufferMarker>, DataError> {
130        let payload: Yoke<(&[u8], Option<u64>), Option<Cart>> = self
131            .data
132            .try_map_project_cloned(|blob, _| blob.load(marker, req))?;
133        let mut metadata = DataResponseMetadata::default();
134        metadata.buffer_format = Some(BufferFormat::Postcard1);
135        metadata.checksum = payload.get().1;
136        Ok(DataResponse {
137            metadata,
138            payload: DataPayload::from_yoked_buffer(payload.map_project(|(bytes, _), _| bytes)),
139        })
140    }
141}
142
143impl DynamicDryDataProvider<BufferMarker> for BlobDataProvider {
144    fn dry_load_data(
145        &self,
146        marker: DataMarkerInfo,
147        req: DataRequest,
148    ) -> Result<DataResponseMetadata, DataError> {
149        self.data.get().load(marker, req)?;
150        let mut metadata = DataResponseMetadata::default();
151        metadata.buffer_format = Some(BufferFormat::Postcard1);
152        Ok(metadata)
153    }
154}
155
156/// ✨ *Enabled with the `alloc` Cargo feature.*
157#[cfg(feature = "alloc")]
158impl IterableDynamicDataProvider<BufferMarker> for BlobDataProvider {
159    fn iter_ids_for_marker(
160        &self,
161        marker: DataMarkerInfo,
162    ) -> Result<alloc::collections::BTreeSet<DataIdentifierCow<'_>>, DataError> {
163        self.data.get().iter_ids(marker)
164    }
165}
166
167#[cfg(test)]
168mod test {
169    use super::*;
170    use crate::export::*;
171    use icu_provider::export::*;
172    use icu_provider::hello_world::*;
173
174    icu_provider::data_marker!(HelloSingletonV1, HelloSingleton, is_singleton = true);
175    #[derive(Clone, Copy, yoke::Yokeable, zerofrom::ZeroFrom)]
176    pub struct HelloSingleton;
177
178    #[test]
179    fn test_empty() {
180        let mut blob: Vec<u8> = Vec::new();
181
182        {
183            let mut exporter = BlobExporter::new_with_sink(Box::new(&mut blob));
184
185            exporter
186                .flush(HelloWorldV1::INFO, Default::default())
187                .unwrap();
188
189            exporter.close().unwrap();
190        }
191
192        let provider = BlobDataProvider::try_new_from_blob(blob.into()).unwrap();
193
194        assert!(
195            matches!(
196                provider.load_data(HelloWorldV1::INFO, Default::default()),
197                Err(DataError {
198                    kind: DataErrorKind::IdentifierNotFound,
199                    ..
200                })
201            ),
202            "Empty blob test"
203        );
204    }
205
206    #[test]
207    fn test_singleton() {
208        let mut blob: Vec<u8> = Vec::new();
209
210        {
211            let mut exporter = BlobExporter::new_with_sink(Box::new(&mut blob));
212
213            exporter
214                .flush(HelloSingletonV1::INFO, Default::default())
215                .unwrap();
216
217            exporter.close().unwrap();
218        }
219
220        let provider = BlobDataProvider::try_new_from_blob(blob.into()).unwrap();
221
222        assert!(
223            matches!(
224                provider.load_data(
225                    HelloSingletonV1::INFO,
226                    DataRequest {
227                        id: DataIdentifierBorrowed::for_locale(
228                            &icu_locale_core::langid!("de").into()
229                        ),
230                        ..Default::default()
231                    }
232                ),
233                Err(DataError {
234                    kind: DataErrorKind::InvalidRequest,
235                    ..
236                })
237            ),
238            "Singleton blob test"
239        );
240
241        assert!(
242            matches!(
243                provider.load_data(HelloSingletonV1::INFO, Default::default()),
244                Err(DataError {
245                    kind: DataErrorKind::IdentifierNotFound,
246                    ..
247                })
248            ),
249            "Singleton blob test"
250        );
251    }
252}
icu_provider_blob/blob_data_provider.rs

icu_provider_blob/
blob_data_provider.rs
