icu_provider_blob/
blob_data_provider.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use crate::blob_schema::BlobSchema;
6use icu_provider::buf::BufferFormat;
7use icu_provider::prelude::*;
8use icu_provider::Cart;
9use icu_provider::DynamicDryDataProvider;
10use yoke::*;
11
12/// A data provider that reads from serialized blobs of data.
13///
14/// This enables data blobs to be read from arbitrary sources at runtime, allowing code and data
15/// to be separated. Alternatively, blobs can also be statically included at compile time.
16///
17/// [`BlobDataProvider`] implements [`BufferProvider`], so it can be used in
18/// `*_with_buffer_provider` constructors across ICU4X.
19///
20/// # `Sync + Send`
21///
22/// This provider uses reference counting internally. When the `sync` Cargo feature on the [`icu_provider`]
23/// crate is enabled, it uses [`Arc`](alloc::sync::Arc) instead of [`Rc`](alloc::rc::Rc), making
24/// it `Sync + Send`.
25///
26/// # Examples
27///
28/// ## Dynamic loading
29///
30/// Load "hello world" data from a postcard blob loaded at runtime:
31///
32/// ```
33/// use icu_locale_core::locale;
34/// use icu_provider::hello_world::HelloWorldFormatter;
35/// use icu_provider_blob::BlobDataProvider;
36/// use writeable::assert_writeable_eq;
37///
38/// // Read an ICU4X data blob dynamically:
39/// let blob = std::fs::read("tests/data/v3.postcard")
40///     .expect("Reading pre-computed postcard buffer");
41///
42/// // Create a DataProvider from it:
43/// let provider = BlobDataProvider::try_new_from_blob(blob.into_boxed_slice())
44///     .expect("Deserialization should succeed");
45///
46/// // Check that it works:
47/// let formatter = HelloWorldFormatter::try_new_with_buffer_provider(
48///     &provider,
49///     locale!("la").into(),
50/// )
51/// .expect("locale exists");
52///
53/// assert_writeable_eq!(formatter.format(), "Ave, munde");
54/// ```
55///
56/// ## Static loading
57///
58/// Load "hello world" data from a postcard blob statically linked at compile time:
59///
60/// ```
61/// use icu_locale_core::locale;
62/// use icu_provider::hello_world::HelloWorldFormatter;
63/// use icu_provider_blob::BlobDataProvider;
64/// use writeable::assert_writeable_eq;
65///
66/// // Read an ICU4X data blob statically:
67/// const HELLO_WORLD_BLOB: &[u8] = include_bytes!("../tests/data/v3.postcard");
68///
69/// // Create a DataProvider from it:
70/// let provider = BlobDataProvider::try_new_from_static_blob(HELLO_WORLD_BLOB)
71///     .expect("Deserialization should succeed");
72///
73/// // Check that it works:
74/// let formatter = HelloWorldFormatter::try_new_with_buffer_provider(
75///     &provider,
76///     locale!("la").into(),
77/// )
78/// .expect("locale exists");
79///
80/// assert_writeable_eq!(formatter.format(), "Ave, munde");
81/// ```
82#[derive(Clone)]
83pub struct BlobDataProvider {
84    pub(crate) data: Yoke<BlobSchema<'static>, Option<Cart>>,
85}
86
87impl core::fmt::Debug for BlobDataProvider {
88    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
89        f.debug_struct("BlobDataProvider")
90            .field("data", &"[...]")
91            .finish()
92    }
93}
94
95impl BlobDataProvider {
96    /// Create a [`BlobDataProvider`] from a blob of ICU4X data.
97    #[cfg(feature = "alloc")]
98    pub fn try_new_from_blob(blob: alloc::boxed::Box<[u8]>) -> Result<Self, DataError> {
99        Ok(Self {
100            data: Cart::try_make_yoke(blob, |bytes| {
101                BlobSchema::deserialize_and_check(&mut postcard::Deserializer::from_bytes(bytes))
102            })?,
103        })
104    }
105
106    /// Create a [`BlobDataProvider`] from a static blob. This is a special case of
107    /// [`try_new_from_blob`](BlobDataProvider::try_new_from_blob) and is allocation-free.
108    pub fn try_new_from_static_blob(blob: &'static [u8]) -> Result<Self, DataError> {
109        Ok(Self {
110            data: Yoke::new_owned(BlobSchema::deserialize_and_check(
111                &mut postcard::Deserializer::from_bytes(blob),
112            )?),
113        })
114    }
115
116    #[doc(hidden)] // for testing purposes only: checks if it is using the Bigger format
117    pub fn internal_is_using_bigger_format(&self) -> bool {
118        matches!(self.data.get(), BlobSchema::V003Bigger(..))
119    }
120}
121
122impl DynamicDataProvider<BufferMarker> for BlobDataProvider {
123    fn load_data(
124        &self,
125        marker: DataMarkerInfo,
126        req: DataRequest,
127    ) -> Result<DataResponse<BufferMarker>, DataError> {
128        let payload: Yoke<(&[u8], Option<u64>), Option<Cart>> = self
129            .data
130            .try_map_project_cloned(|blob, _| blob.load(marker, req))?;
131        let mut metadata = DataResponseMetadata::default();
132        metadata.buffer_format = Some(BufferFormat::Postcard1);
133        metadata.checksum = payload.get().1;
134        Ok(DataResponse {
135            metadata,
136            payload: DataPayload::from_yoked_buffer(payload.map_project(|(bytes, _), _| bytes)),
137        })
138    }
139}
140
141impl DynamicDryDataProvider<BufferMarker> for BlobDataProvider {
142    fn dry_load_data(
143        &self,
144        marker: DataMarkerInfo,
145        req: DataRequest,
146    ) -> Result<DataResponseMetadata, DataError> {
147        self.data.get().load(marker, req)?;
148        let mut metadata = DataResponseMetadata::default();
149        metadata.buffer_format = Some(BufferFormat::Postcard1);
150        Ok(metadata)
151    }
152}
153
154#[cfg(feature = "alloc")]
155impl IterableDynamicDataProvider<BufferMarker> for BlobDataProvider {
156    fn iter_ids_for_marker(
157        &self,
158        marker: DataMarkerInfo,
159    ) -> Result<alloc::collections::BTreeSet<DataIdentifierCow>, DataError> {
160        self.data.get().iter_ids(marker)
161    }
162}
163
164#[cfg(test)]
165mod test {
166    use super::*;
167    use crate::export::*;
168    use icu_provider::export::*;
169    use icu_provider::hello_world::*;
170
171    icu_provider::data_marker!(HelloSingletonV1, HelloSingleton, is_singleton = true);
172    #[derive(Clone, Copy, yoke::Yokeable, zerofrom::ZeroFrom)]
173    pub struct HelloSingleton;
174
175    #[test]
176    fn test_empty() {
177        let mut blob: Vec<u8> = Vec::new();
178
179        {
180            let mut exporter = BlobExporter::new_with_sink(Box::new(&mut blob));
181
182            exporter
183                .flush(HelloWorldV1::INFO, Default::default())
184                .unwrap();
185
186            exporter.close().unwrap();
187        }
188
189        let provider = BlobDataProvider::try_new_from_blob(blob.into()).unwrap();
190
191        assert!(
192            matches!(
193                provider.load_data(HelloWorldV1::INFO, Default::default()),
194                Err(DataError {
195                    kind: DataErrorKind::IdentifierNotFound,
196                    ..
197                })
198            ),
199            "Empty blob test"
200        );
201    }
202
203    #[test]
204    fn test_singleton() {
205        let mut blob: Vec<u8> = Vec::new();
206
207        {
208            let mut exporter = BlobExporter::new_with_sink(Box::new(&mut blob));
209
210            exporter
211                .flush(HelloSingletonV1::INFO, Default::default())
212                .unwrap();
213
214            exporter.close().unwrap();
215        }
216
217        let provider = BlobDataProvider::try_new_from_blob(blob.into()).unwrap();
218
219        assert!(
220            matches!(
221                provider.load_data(
222                    HelloSingletonV1::INFO,
223                    DataRequest {
224                        id: DataIdentifierBorrowed::for_locale(
225                            &icu_locale_core::langid!("de").into()
226                        ),
227                        ..Default::default()
228                    }
229                ),
230                Err(DataError {
231                    kind: DataErrorKind::InvalidRequest,
232                    ..
233                })
234            ),
235            "Singleton blob test"
236        );
237
238        assert!(
239            matches!(
240                provider.load_data(HelloSingletonV1::INFO, Default::default()),
241                Err(DataError {
242                    kind: DataErrorKind::IdentifierNotFound,
243                    ..
244                })
245            ),
246            "Singleton blob test"
247        );
248    }
249}
icu_provider_blob/blob_data_provider.rs

icu_provider_blob/
blob_data_provider.rs
