shuyi_assets/
pack.rs

1//! The pack (`.shuyipak`, ADR-0007 §4, ADR-0023): the optimized format a
2//! game may ship in place of its loose files. `shuyi bake` turns the
3//! sources into one file, and the game lays it in its bundle like any
4//! other layer ([`Pack::into_layer`]), under its loose files or alone.
5//!
6//! A pack holds every file of the bundle it was baked from, each at its
7//! own path:
8//!
9//! - **a baked file** holds the assets read from its source, already in
10//!   their binary form (postcard, one section per kind it was read as),
11//!   so loading it parses no text. [`Packed`] is the format that reads
12//!   one: a game registers it for every kind it bakes
13//!   ([`Bake::register`]);
14//! - **every other file** (a picture, a sound) holds its own bytes.
15//!
16//! The file is a header (magic bytes, the [`VERSION`], a checksum of the
17//! rest), an index (the stamp, the target, the fingerprint of its sources,
18//! and each file's path, kinds, offset and sizes) and each file's bytes,
19//! compressed with LZ4 when that makes them smaller.
20//!
21//! A pack stands for its sources: its layer ([`Pack::into_layer`]) gives a
22//! bundle the fingerprint of the bundle it was baked from
23//! ([`Bundle::fingerprint`](crate::bundle::Bundle::fingerprint)), so a
24//! record made on the loose files plays on the pack as made with the same
25//! content.
26//!
27//! The rules:
28//!
29//! - **Text stays the source.** A pack is derived by [`Bake::bake`], and
30//!   [`Bake::check`] proves every asset read from it equal to the one
31//!   read from its source.
32//! - **No migration.** A pack of another [`VERSION`], or baked for
33//!   another stamp (the game's word for the shapes of its own kinds), is
34//!   refused with a message to bake it again.
35//! - **Never a panic.** A pack cut short or changed anywhere is refused:
36//!   the checksum covers every byte after the header, and every size is
37//!   checked before anything is allocated for it.
38//!
39//! The reader builds without `std`: a Playdate loads only packs
40//! (ADR-0008 §5).
41use crate::prelude::*;
42
43use alloc::borrow::Cow;
44use alloc::collections::BTreeMap;
45use core::marker::PhantomData;
46
47use serde::de::DeserializeOwned;
48use serde::{Deserialize, Serialize};
49use shuyi_core::ContentHash;
50
51use crate::bundle::Layer;
52use crate::error::ContentError;
53use crate::fnv::Hasher;
54use crate::format::{AssetKind, AssetPath, DecodeCx, Format};
55
56mod bake;
57
58pub use bake::{Bake, Bakeable, Baked, BakedFile};
59
60/// A pack's file extension, without the dot.
61pub const EXTENSION: &str = "shuyipak";
62
63/// The bytes a pack starts with.
64pub const MAGIC: [u8; 8] = *b"SHUYIPAK";
65
66/// The version of the pack format this engine reads and writes: the
67/// header, the index, and the binary form of the engine's own kinds. A
68/// change to any of them makes it the next number, and a pack of another
69/// version is refused, never migrated.
70pub const VERSION: u32 = 9;
71
72/// The bytes a baked file in a pack starts with.
73pub const BAKED_MAGIC: [u8; 8] = *b"SHUYIBKD";
74
75/// The most files a pack holds.
76pub const MAX_FILES: usize = 1 << 16;
77
78/// The most bytes a pack's files add up to, unpacked: 1 GiB.
79pub const MAX_UNPACKED_BYTES: u64 = 1 << 30;
80
81/// The most bytes a baked file has: 64 MiB, as the engine's biggest
82/// sources (a PNG, a sound).
83pub const MAX_BAKED_BYTES: usize = 64 << 20;
84
85/// How many bytes LZ4 can make of one, at most: a file claiming more
86/// than this many times its stored bytes is broken.
87const MAX_RATIO: u64 = 255;
88
89/// The header: the magic bytes, the version (`u32`), the checksum of
90/// everything after the header (`u64`) and the index's length (`u32`).
91const HEADER: usize = 24;
92
93/// What a pack says of itself.
94#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
95pub struct PackInfo {
96    /// The game's word for the shapes of its own kinds, the ones a
97    /// [`VERSION`] doesn't cover (its version, say). A game opens only a
98    /// pack baked with its own stamp.
99    pub stamp: String,
100    /// The target the pack was baked for (`shuyi bake --target`), if any.
101    pub target: Option<String>,
102    /// The fingerprint of the bundle it was baked from
103    /// ([`Bundle::fingerprint`](crate::bundle::Bundle::fingerprint)),
104    /// which its layer stands for.
105    pub content: ContentHash,
106}
107
108/// How a file's bytes are kept.
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
110enum Codec {
111    /// As they are.
112    Stored,
113    /// Compressed with LZ4 (its block format).
114    Lz4,
115}
116
117/// One file in the index.
118#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
119struct Entry {
120    /// Its path in the bundle.
121    path: String,
122    /// The kinds it was baked as: none for a file kept as its own bytes.
123    kinds: Vec<String>,
124    /// How its bytes are kept.
125    codec: Codec,
126    /// Where its bytes start, after the index.
127    offset: u64,
128    /// How many bytes are kept.
129    stored: u64,
130    /// How many bytes it has, unpacked.
131    size: u64,
132}
133
134/// The index: what the pack says of itself, and its files in path order.
135#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
136struct Index {
137    info: PackInfo,
138    entries: Vec<Entry>,
139}
140
141/// What goes into a pack at one path.
142#[derive(Debug, Clone, PartialEq, Eq)]
143pub enum Content {
144    /// The file's own bytes.
145    Raw(Vec<u8>),
146    /// The assets read from the file: each kind's name and its binary form
147    /// (postcard).
148    Baked(BTreeMap<&'static str, Vec<u8>>),
149}
150
151/// Why a pack was refused.
152#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
153pub enum PackError {
154    /// It doesn't start with [`MAGIC`].
155    #[error("not a Shuyi pack: a .shuyipak file starts with \"SHUYIPAK\"")]
156    NotAPack,
157    /// It is of another [`VERSION`].
158    #[error(
159        "the pack is of pack format version {found} and this engine reads version {VERSION}: \
160         bake it again with `shuyi bake`"
161    )]
162    Version {
163        /// Its version.
164        found: u32,
165    },
166    /// It was baked for another stamp.
167    #[error(
168        "the pack was baked for \"{found}\" and this game is \"{expected}\": bake it again with \
169         `shuyi bake`"
170    )]
171    Stamp {
172        /// Its stamp.
173        found: String,
174        /// The game's.
175        expected: String,
176    },
177    /// It is shorter than its header or its index says.
178    #[error("the pack is cut short: bake it again with `shuyi bake`")]
179    Truncated,
180    /// A byte after the header isn't the one baked.
181    #[error("the pack was changed after it was baked (its checksum differs): bake it again")]
182    Checksum,
183    /// Its index or one of its files doesn't hold together.
184    #[error("the pack is broken: {0}")]
185    Broken(String),
186}
187
188/// A pack's files, read and unpacked: what a game lays in its bundle. A
189/// file the pack keeps as it is borrows the pack's bytes (`'a`); only a
190/// compressed one is unpacked into memory of its own.
191#[derive(Debug, Clone, PartialEq, Eq)]
192pub struct Pack<'a> {
193    info: PackInfo,
194    files: Vec<PackFile<'a>>,
195}
196
197/// One file of a [`Pack`].
198#[derive(Debug, Clone, PartialEq, Eq)]
199pub struct PackFile<'a> {
200    /// Its path in the bundle.
201    pub path: String,
202    /// The kinds it was baked as: none for a file kept as its own bytes.
203    pub kinds: Vec<String>,
204    /// How many bytes the pack keeps of it.
205    pub stored: usize,
206    /// Its bytes, unpacked: a baked file's ([`Packed`] reads them), or the
207    /// file's own.
208    pub bytes: Cow<'a, [u8]>,
209}
210
211/// The checksum of a pack: the word-wise FNV-1a hash (`crate::fnv`) of
212/// its bytes, then their length.
213fn checksum(bytes: &[u8]) -> u64 {
214    let mut hash = Hasher::new();
215    hash.bytes(bytes);
216    hash.word(bytes.len() as u64);
217    hash.0
218}
219
220/// The `N` bytes at `at` in `bytes`, or [`PackError::Truncated`].
221fn take<const N: usize>(bytes: &[u8], at: usize) -> Result<[u8; N], PackError> {
222    bytes
223        .get(at..at + N)
224        .and_then(|b| b.try_into().ok())
225        .ok_or(PackError::Truncated)
226}
227
228/// A broken pack, for `why`.
229fn broken(why: impl Into<String>) -> PackError {
230    PackError::Broken(why.into())
231}
232
233/// The bytes of a baked file of `sections` (kind name → binary form).
234fn baked_file(sections: &BTreeMap<&'static str, Vec<u8>>) -> Vec<u8> {
235    let list: Vec<(&str, &[u8])> = sections
236        .iter()
237        .map(|(kind, data)| (*kind, data.as_slice()))
238        .collect();
239    let mut bytes = BAKED_MAGIC.to_vec();
240    // A list of strings and byte strings always serializes.
241    bytes.extend(postcard::to_allocvec(&list).unwrap_or_default());
242    bytes
243}
244
245/// The sections of a baked file (kind name → binary form), if `bytes` is
246/// one whole.
247fn sections(bytes: &[u8]) -> Option<Vec<(&str, &[u8])>> {
248    let rest = bytes.strip_prefix(&BAKED_MAGIC)?;
249    let (list, left) = postcard::take_from_bytes::<Vec<(&str, &[u8])>>(rest).ok()?;
250    left.is_empty().then_some(list)
251}
252
253/// A pack of `files` (path → content), described by `info`. Each file's
254/// bytes are compressed when that makes them smaller.
255pub fn write(info: &PackInfo, files: &BTreeMap<String, Content>) -> Vec<u8> {
256    let mut entries = Vec::with_capacity(files.len());
257    let mut data = Vec::new();
258    for (path, content) in files {
259        let (kinds, bytes) = match content {
260            Content::Raw(bytes) => (Vec::new(), Cow::Borrowed(bytes.as_slice())),
261            Content::Baked(sections) => (
262                sections.keys().map(|&kind| kind.to_owned()).collect(),
263                Cow::Owned(baked_file(sections)),
264            ),
265        };
266        let packed = lz4_flex::block::compress(&bytes);
267        let (codec, kept) = if packed.len() < bytes.len() {
268            (Codec::Lz4, packed.as_slice())
269        } else {
270            (Codec::Stored, &*bytes)
271        };
272        entries.push(Entry {
273            path: path.clone(),
274            kinds,
275            codec,
276            offset: data.len() as u64,
277            stored: kept.len() as u64,
278            size: bytes.len() as u64,
279        });
280        data.extend_from_slice(kept);
281    }
282    let index = Index {
283        info: info.clone(),
284        entries,
285    };
286    // Strings, numbers and lists always serialize.
287    let index = postcard::to_allocvec(&index).unwrap_or_default();
288    let mut body = Vec::with_capacity(index.len() + data.len());
289    body.extend_from_slice(&index);
290    body.extend_from_slice(&data);
291    let mut out = Vec::with_capacity(HEADER + body.len());
292    out.extend_from_slice(&MAGIC);
293    out.extend_from_slice(&VERSION.to_le_bytes());
294    out.extend_from_slice(&checksum(&body).to_le_bytes());
295    // An index over 4 GiB would hold more than `MAX_FILES` files.
296    out.extend_from_slice(&u32::try_from(index.len()).unwrap_or(u32::MAX).to_le_bytes());
297    out.extend_from_slice(&body);
298    out
299}
300
301/// The body of pack `bytes`, the rest of them after its header, and its
302/// index's length, once its magic bytes and version are right.
303fn body(bytes: &[u8]) -> Result<(&[u8], usize), PackError> {
304    if !bytes.starts_with(&MAGIC) {
305        return Err(if MAGIC.starts_with(bytes) {
306            PackError::Truncated
307        } else {
308            PackError::NotAPack
309        });
310    }
311    let found = u32::from_le_bytes(take(bytes, 8)?);
312    if found != VERSION {
313        return Err(PackError::Version { found });
314    }
315    let length = u32::from_le_bytes(take(bytes, 20)?);
316    let body = bytes.get(HEADER..).ok_or(PackError::Truncated)?;
317    let length = usize::try_from(length).map_err(|_| PackError::Truncated)?;
318    if body.len() < length {
319        return Err(PackError::Truncated);
320    }
321    Ok((body, length))
322}
323
324/// How many bytes the pack at the start of `bytes` takes, by its header and
325/// its index: for a tool that looks for a pack inside a bigger file, such as
326/// a program that embeds one with `include_bytes!`. Nothing here is checked
327/// against the checksum: [`Pack::open`] the bytes it counts.
328pub fn length(bytes: &[u8]) -> Result<usize, PackError> {
329    let (body, length) = body(bytes)?;
330    let index: Index =
331        postcard::from_bytes(&body[..length]).map_err(|_| broken("its index doesn't read"))?;
332    let data = index
333        .entries
334        .last()
335        .map_or(0, |entry| entry.offset.saturating_add(entry.stored));
336    usize::try_from(data)
337        .ok()
338        .and_then(|data| (HEADER + length).checked_add(data))
339        .filter(|&total| total <= bytes.len())
340        .ok_or(PackError::Truncated)
341}
342
343/// The index of pack `bytes` and where its files' bytes start, once its
344/// header, version and checksum are right.
345fn index(bytes: &[u8]) -> Result<(Index, &[u8]), PackError> {
346    let (body, length) = body(bytes)?;
347    let sum = u64::from_le_bytes(take(bytes, 12)?);
348    if checksum(body) != sum {
349        return Err(PackError::Checksum);
350    }
351    let (index, data) = body.split_at(length);
352    let index: Index = postcard::from_bytes(index).map_err(|_| broken("its index doesn't read"))?;
353    Ok((index, data))
354}
355
356/// The bytes of `entry` in `data`, unpacked, after the files before it
357/// (`end` bytes of `data`) and `unpacked` bytes unpacked so far.
358fn unpack<'a>(
359    entry: &Entry,
360    data: &'a [u8],
361    end: u64,
362    unpacked: u64,
363) -> Result<Cow<'a, [u8]>, PackError> {
364    let path = &entry.path;
365    if entry.offset != end {
366        return Err(broken(format!(
367            "{path} starts at {} and the file before it ends at {end}",
368            entry.offset
369        )));
370    }
371    if unpacked.saturating_add(entry.size) > MAX_UNPACKED_BYTES {
372        return Err(broken(format!(
373            "its files add up to more than {MAX_UNPACKED_BYTES} bytes"
374        )));
375    }
376    let size = usize::try_from(entry.size).map_err(|_| broken(format!("{path} is too big")))?;
377    let range = usize::try_from(entry.offset)
378        .ok()
379        .zip(usize::try_from(entry.offset.saturating_add(entry.stored)).ok())
380        .and_then(|(start, end)| data.get(start..end))
381        .ok_or_else(|| broken(format!("{path} goes past the end of the pack")))?;
382    match entry.codec {
383        Codec::Stored if entry.stored == entry.size => Ok(Cow::Borrowed(range)),
384        Codec::Stored => Err(broken(format!("{path} is stored with two sizes"))),
385        Codec::Lz4 if entry.size > entry.stored.saturating_mul(MAX_RATIO) => Err(broken(format!(
386            "{path} claims {size} bytes from {} stored",
387            entry.stored
388        ))),
389        Codec::Lz4 => match lz4_flex::block::decompress(range, size) {
390            Ok(bytes) if bytes.len() == size => Ok(Cow::Owned(bytes)),
391            _ => Err(broken(format!("{path} doesn't unpack"))),
392        },
393    }
394}
395
396/// What pack `bytes` says of itself, once its header, version and
397/// checksum are right: for a tool that looks at a pack without opening
398/// it.
399pub fn info(bytes: &[u8]) -> Result<PackInfo, PackError> {
400    index(bytes).map(|(index, _)| index.info)
401}
402
403impl<'a> Pack<'a> {
404    /// The pack `bytes`, every file unpacked, if it is of this engine's
405    /// [`VERSION`], was baked with `stamp`, and holds together.
406    pub fn open(bytes: &'a [u8], stamp: &str) -> Result<Self, PackError> {
407        let (index, data) = index(bytes)?;
408        if index.info.stamp != stamp {
409            return Err(PackError::Stamp {
410                found: index.info.stamp,
411                expected: stamp.to_owned(),
412            });
413        }
414        if index.entries.len() > MAX_FILES {
415            return Err(broken(format!("it has more than {MAX_FILES} files")));
416        }
417        let mut files: Vec<PackFile<'a>> = Vec::with_capacity(index.entries.len());
418        let (mut end, mut unpacked) = (0, 0);
419        for entry in index.entries {
420            if files.last().is_some_and(|last| last.path >= entry.path) {
421                return Err(broken(format!("{} is out of path order", entry.path)));
422            }
423            let bytes = unpack(&entry, data, end, unpacked)?;
424            if !entry.kinds.is_empty() && sections(&bytes).is_none() {
425                return Err(broken(format!("{} isn't a whole baked file", entry.path)));
426            }
427            end = entry.offset.saturating_add(entry.stored);
428            unpacked = unpacked.saturating_add(entry.size);
429            files.push(PackFile {
430                path: entry.path,
431                kinds: entry.kinds,
432                stored: usize::try_from(entry.stored).unwrap_or(usize::MAX),
433                bytes,
434            });
435        }
436        if usize::try_from(end).ok() != Some(data.len()) {
437            return Err(broken("bytes follow its last file"));
438        }
439        Ok(Self {
440            info: index.info,
441            files,
442        })
443    }
444
445    /// What the pack says of itself.
446    pub fn info(&self) -> &PackInfo {
447        &self.info
448    }
449
450    /// Its files, in path order.
451    pub fn files(&self) -> &[PackFile<'a>] {
452        &self.files
453    }
454
455    /// The bytes of the file at `path`, unpacked, if the pack has it.
456    pub fn get(&self, path: &str) -> Option<&[u8]> {
457        let at = self
458            .files
459            .binary_search_by(|file| file.path.as_str().cmp(path))
460            .ok()?;
461        Some(&self.files[at].bytes)
462    }
463
464    /// The pack, every file in memory of its own: for a pack read at run
465    /// time, whose bytes the game doesn't keep.
466    pub fn into_owned(self) -> Pack<'static> {
467        let files = self
468            .files
469            .into_iter()
470            .map(|file| PackFile {
471                bytes: Cow::Owned(file.bytes.into_owned()),
472                path: file.path,
473                kinds: file.kinds,
474                stored: file.stored,
475            })
476            .collect();
477        Pack {
478            info: self.info,
479            files,
480        }
481    }
482}
483
484impl Pack<'static> {
485    /// A bundle layer of the pack's files, shown as `name`, standing for
486    /// the bundle it was baked from ([`Layer::standing_for`]). A game opens
487    /// its pack once, at start-up (most simply one it embeds with
488    /// `include_bytes!`): the files it unpacked are kept for the rest of
489    /// the run, as the files a game embeds are.
490    pub fn into_layer(self, name: &'static str) -> Layer {
491        let content = self.info.content;
492        let table: Vec<(&'static str, &'static [u8])> = self
493            .files
494            .into_iter()
495            .map(|file| {
496                let path: &'static str = Box::leak(file.path.into_boxed_str());
497                let bytes: &'static [u8] = match file.bytes {
498                    Cow::Borrowed(bytes) => bytes,
499                    Cow::Owned(bytes) => Box::leak(bytes.into_boxed_slice()),
500                };
501                (path, bytes)
502            })
503            .collect();
504        Layer::table(name, Box::leak(table.into_boxed_slice())).standing_for(content)
505    }
506}
507
508/// The format of a baked file of kind `K` in a pack: it reads the asset
509/// baked from the source, with no parsing of text. It claims no
510/// extension, and knows a baked file by its magic bytes, so the file keeps
511/// its source's path (`data/palette.ron`) and code that asks for the asset
512/// doesn't change. [`Bake::register`] registers it for every kind a game
513/// bakes.
514pub struct Packed<K>(PhantomData<fn() -> K>);
515
516impl<K> Packed<K> {
517    /// The id of every kind's baked format.
518    pub const ID: &'static str = "baked";
519
520    /// The format.
521    pub const fn new() -> Self {
522        Self(PhantomData)
523    }
524}
525
526impl<K> Default for Packed<K> {
527    fn default() -> Self {
528        Self::new()
529    }
530}
531
532impl<K> core::fmt::Debug for Packed<K> {
533    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
534        f.write_str("Packed")
535    }
536}
537
538impl<K: AssetKind + DeserializeOwned> Format for Packed<K> {
539    type Asset = K;
540
541    fn id(&self) -> &'static str {
542        Self::ID
543    }
544
545    fn extensions(&self) -> &'static [&'static str] {
546        &[]
547    }
548
549    fn sniff(&self, bytes: &[u8]) -> bool {
550        bytes.starts_with(&BAKED_MAGIC)
551    }
552
553    fn max_bytes(&self) -> usize {
554        MAX_BAKED_BYTES
555    }
556
557    fn decode(
558        &self,
559        file: &AssetPath,
560        bytes: &[u8],
561        _cx: &DecodeCx<'_>,
562    ) -> Result<K, Vec<ContentError>> {
563        let error = |message: String| vec![ContentError::new(file.shown(), message)];
564        let sections =
565            sections(bytes).ok_or_else(|| error("a baked file that is broken".to_owned()))?;
566        let Some(&(_, data)) = sections.iter().find(|(kind, _)| *kind == K::NAME) else {
567            let kinds: Vec<&str> = sections.iter().map(|&(kind, _)| kind).collect();
568            return Err(error(format!(
569                "baked as {} and not as a {}: bake the {} kind too",
570                kinds.join(", "),
571                K::NAME,
572                K::NAME
573            )));
574        };
575        match postcard::take_from_bytes::<K>(data) {
576            Ok((asset, [])) => Ok(asset),
577            _ => Err(error(format!(
578                "its baked {} doesn't read as this game's: bake it again with `shuyi bake`",
579                K::NAME
580            ))),
581        }
582    }
583}
584
585#[cfg(test)]
586mod one_bit_tests;
587#[cfg(test)]
588mod tests;
shuyi_assets/pack.rs

shuyi_assets/
pack.rs
