Store kind "strings": string packs install like themes and plugins (dropped .strings.json, links, the signed Store), validated as plain data on this side (envelope, plain text, size caps, the app's own site), kept in app data strings/ with their order and which are on (at most 8); licence and credits are optional fields of every catalog item; five shell-only commands (strings_list, strings_read, strings_set, strings_remove, strings_apply); the shell's delivery (packs + the inventory slice) goes into site tabs' session storage by revision, as the first document-start layer of a site tab and never in safe mode

This commit is contained in:
CRTkafa 2026-10-01 03:56:54 +07:00
parent fd6ae707ca
commit c3bfc7a6f8
10 changed files with 593 additions and 13 deletions

View file

@ -0,0 +1,417 @@
//! String packs (waybetter/docs/STRING-PACKS.md) on disk: data-only packs that replace the wrapped
//! site's own UI strings. The shell checks a pack against the site's inventory (coverage, unknown
//! keys); this side only accepts well-formed data and keeps it: `strings/<id>.json`, and
//! `strings/state.json` (which packs are on, in which order). A pack carries text, never code.
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::path::{Path, PathBuf};
pub const MAX_STRINGS_BYTES: usize = 2 * 1024 * 1024;
pub const MAX_PACK_STRINGS: usize = 40_000;
pub const MAX_PACK_VALUE_CHARS: usize = 2000;
/// At most this many packs are installed (and at most 8 are on at once).
pub const MAX_PACKS: usize = 64;
pub const MAX_ENABLED: usize = 8;
const STATE_FILE: &str = "state.json";
const DELIVERY_FILE: &str = "delivery.json";
/// A delivery (packs + the inventory slice their keys need) the shell computed; handed to site tabs.
pub const MAX_DELIVERY_BYTES: usize = 8 * 1024 * 1024;
/// What the install screen and the settings list show about a pack.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StringsInfo {
pub site: String,
pub catalog: String,
pub language: String,
pub name: String,
pub authors: Vec<String>,
pub version: String,
pub description: String,
/// Number of strings in the pack.
pub strings: usize,
}
fn plain(value: &str, max: usize) -> bool {
value.chars().count() <= max && !value.chars().any(|ch| ch.is_control() || ch == '<' || ch == '>' || ('\u{202a}'..='\u{202e}').contains(&ch) || ('\u{2066}'..='\u{2069}').contains(&ch))
}
fn ident(value: &str) -> bool {
!value.is_empty() && value.len() <= 32 && value.bytes().next().is_some_and(|b| b.is_ascii_lowercase()) && value.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
}
/// A BCP 47-shaped language tag (`eo`, `pt-BR`, `en-x-pirate`).
pub fn valid_language(tag: &str) -> bool {
let mut parts = tag.split('-');
let first = parts.next().unwrap_or_default();
(2..=8).contains(&first.len()) && first.bytes().all(|b| b.is_ascii_alphabetic()) && parts.all(|part| (1..=8).contains(&part.len()) && part.bytes().all(|b| b.is_ascii_alphanumeric()))
}
/// A string pack file: at most [`MAX_STRINGS_BYTES`], UTF-8 JSON with the pack envelope, key ids
/// mapped to plain text (no markup, no control or bidi-override characters). Whether the ids and
/// parameters fit the site is the shell's check against the inventory.
pub fn validate_strings(bytes: &[u8]) -> Result<StringsInfo, String> {
if bytes.is_empty() {
return Err("VALIDATION: the string pack is empty".into());
}
if bytes.len() > MAX_STRINGS_BYTES {
return Err("VALIDATION: string packs are limited to 2 MiB".into());
}
let text = std::str::from_utf8(bytes).map_err(|_| "VALIDATION: a string pack must be UTF-8 JSON")?;
let value: serde_json::Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|_| "VALIDATION: this file is not valid JSON")?;
let object = value.as_object().ok_or("VALIDATION: a string pack is a JSON object")?;
let text_field = |key: &str| object.get(key).and_then(serde_json::Value::as_str);
if object.get("format").and_then(serde_json::Value::as_u64) != Some(1) {
return Err("VALIDATION: a string pack needs \"format\": 1".into());
}
let site = text_field("site").filter(|v| ident(v)).ok_or("VALIDATION: invalid \"site\"")?;
let catalog = text_field("catalog").filter(|v| ident(v)).ok_or("VALIDATION: invalid \"catalog\"")?;
let language = text_field("language").filter(|v| valid_language(v)).ok_or("VALIDATION: invalid \"language\"")?;
if let Some(base) = object.get("base") {
if !base.as_str().is_some_and(valid_language) {
return Err("VALIDATION: invalid \"base\"".into());
}
}
let name = text_field("name").filter(|v| !v.trim().is_empty() && plain(v, 80)).ok_or("VALIDATION: a string pack needs a plain-text \"name\"")?;
let authors: Vec<String> = match object.get("authors").and_then(serde_json::Value::as_array) {
Some(list) if list.len() <= 16 => list
.iter()
.map(|author| author.as_str().filter(|a| !a.trim().is_empty() && plain(a, 80)).map(str::to_owned))
.collect::<Option<Vec<_>>>()
.ok_or("VALIDATION: invalid \"authors\"")?,
_ => return Err("VALIDATION: \"authors\" must be a list of at most 16 names".into()),
};
let version = match object.get("version") {
None => String::new(),
Some(v) => v.as_str().filter(|v| plain(v, 32)).ok_or("VALIDATION: invalid \"version\"")?.to_owned(),
};
let description = match object.get("description") {
None => String::new(),
Some(v) => v.as_str().filter(|v| plain(v, 400)).ok_or("VALIDATION: invalid \"description\"")?.to_owned(),
};
let strings = object.get("strings").and_then(serde_json::Value::as_object).ok_or("VALIDATION: \"strings\" must be an object of key id -> text")?;
if strings.len() > MAX_PACK_STRINGS {
return Err("VALIDATION: too many strings".into());
}
for (id, value) in strings {
let ok_id = !id.is_empty() && id.len() <= 64 && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-' || b == b'.' || b == b'$');
let ok_value = value.as_str().is_some_and(|v| !v.is_empty() && plain(v, MAX_PACK_VALUE_CHARS));
if !ok_id || !ok_value {
return Err(format!("VALIDATION: string {} is not plain text", id.chars().take(64).collect::<String>()));
}
}
const KNOWN: &[&str] = &["$schema", "format", "site", "catalog", "language", "base", "name", "authors", "version", "description", "strings"];
if let Some(key) = object.keys().find(|key| !KNOWN.contains(&key.as_str())) {
return Err(format!("VALIDATION: unknown field \"{}\"", key.chars().take(40).collect::<String>()));
}
Ok(StringsInfo {
site: site.to_owned(),
catalog: catalog.to_owned(),
language: language.to_owned(),
name: name.to_owned(),
authors,
version,
description,
strings: strings.len(),
})
}
/// The id a pack is installed under: the store item's id, else derived from its language and name.
pub fn pack_id(info: &StringsInfo, store_id: Option<&str>) -> String {
if let Some(id) = store_id {
return id.to_owned();
}
let mut id = String::new();
for ch in format!("{}-{}", info.language, info.name).chars() {
let ch = ch.to_ascii_lowercase();
if ch.is_ascii_alphanumeric() {
id.push(ch);
} else if !id.ends_with('-') {
id.push('-');
}
}
let id: String = id.trim_matches('-').chars().take(40).collect();
let id = id.trim_end_matches('-').to_owned();
if id.is_empty() { "strings".to_owned() } else { id }
}
fn valid_pack_id(id: &str) -> bool {
!id.is_empty() && id.len() <= 40 && id.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
}
/// Which packs are on, in application order (a later one wins per key).
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StringsState {
pub order: Vec<String>,
pub enabled: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct InstalledStrings {
pub id: String,
#[serde(flatten)]
pub info: StringsInfo,
pub sha256: String,
pub enabled: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StringsList {
/// In application order.
pub packs: Vec<InstalledStrings>,
}
/// The installed packs of one app: `<data>/strings/`.
pub struct StringsStore {
dir: PathBuf,
}
impl StringsStore {
pub fn new(data_dir: &Path) -> Self {
Self { dir: data_dir.join("strings") }
}
fn file(&self, id: &str) -> PathBuf {
self.dir.join(format!("{id}.json"))
}
pub fn state(&self) -> StringsState {
std::fs::read_to_string(self.dir.join(STATE_FILE)).ok().and_then(|text| serde_json::from_str(&text).ok()).unwrap_or_default()
}
fn save_state(&self, state: &StringsState) -> Result<(), String> {
std::fs::create_dir_all(&self.dir).map_err(|e| format!("STRINGS: {e}"))?;
let text = serde_json::to_string_pretty(state).map_err(|e| e.to_string())?;
write_atomic(&self.dir.join(STATE_FILE), text.as_bytes())
}
/// The installed pack's bytes, if any.
pub fn read(&self, id: &str) -> Option<Vec<u8>> {
if !valid_pack_id(id) {
return None;
}
std::fs::read(self.file(id)).ok()
}
/// Writes a pack (validated again) under `id`; a new pack goes last in the order, on when `enable`.
pub fn install(&self, id: &str, bytes: &[u8], enable: bool) -> Result<StringsInfo, String> {
if !valid_pack_id(id) {
return Err("VALIDATION: invalid string pack id".into());
}
let info = validate_strings(bytes)?;
let mut state = self.state();
let known: HashSet<&String> = state.order.iter().collect();
if !known.contains(&id.to_owned()) && state.order.len() >= MAX_PACKS {
return Err("INSTALL: too many string packs installed".into());
}
std::fs::create_dir_all(&self.dir).map_err(|e| format!("STRINGS: {e}"))?;
write_atomic(&self.file(id), bytes)?;
if !state.order.iter().any(|item| item == id) {
state.order.push(id.to_owned());
}
if enable && !state.enabled.iter().any(|item| item == id) && state.enabled.len() < MAX_ENABLED {
state.enabled.push(id.to_owned());
}
self.save_state(&state)?;
Ok(info)
}
pub fn remove(&self, id: &str) -> Result<(), String> {
if !valid_pack_id(id) {
return Err("VALIDATION: invalid string pack id".into());
}
let _ = std::fs::remove_file(self.file(id));
let mut state = self.state();
state.order.retain(|item| item != id);
state.enabled.retain(|item| item != id);
self.save_state(&state)
}
/// Sets the order and which packs are on. Unknown ids are dropped; installed packs missing
/// from `order` keep their place at the end.
pub fn set(&self, order: &[String], enabled: &[String]) -> Result<StringsList, String> {
let installed = self.installed_ids();
let mut next = StringsState::default();
for id in order.iter().chain(installed.iter()) {
if installed.contains(id) && !next.order.contains(id) {
next.order.push(id.clone());
}
}
for id in &next.order {
if enabled.contains(id) && next.enabled.len() < MAX_ENABLED {
next.enabled.push(id.clone());
}
}
self.save_state(&next)?;
Ok(self.list())
}
fn installed_ids(&self) -> Vec<String> {
let mut ids: Vec<String> = std::fs::read_dir(&self.dir)
.map(|entries| {
entries
.filter_map(Result::ok)
.filter_map(|entry| entry.file_name().to_str().and_then(|name| name.strip_suffix(".json")).map(str::to_owned))
.filter(|id| valid_pack_id(id) && id != "state" && id != "delivery")
.collect()
})
.unwrap_or_default();
ids.sort();
ids
}
/// Installed packs in application order (packs the state does not name yet come last).
pub fn list(&self) -> StringsList {
let state = self.state();
let installed = self.installed_ids();
let mut order: Vec<String> = state.order.iter().filter(|id| installed.contains(id)).cloned().collect();
for id in installed {
if !order.contains(&id) {
order.push(id);
}
}
let packs = order
.into_iter()
.filter_map(|id| {
let bytes = self.read(&id)?;
let info = validate_strings(&bytes).ok()?;
Some(InstalledStrings { enabled: state.enabled.contains(&id), sha256: super::catalog::sha256_hex(&bytes), id, info })
})
.collect();
StringsList { packs }
}
/// The enabled packs' JSON texts, in application order.
pub fn enabled_texts(&self) -> Vec<String> {
self.list().packs.into_iter().filter(|pack| pack.enabled).filter_map(|pack| self.read(&pack.id)).filter_map(|bytes| String::from_utf8(bytes).ok()).collect()
}
/// Keeps the delivery the shell computed (JSON: `{ inventory, packs }`; `None`: no pack on)
/// under a new revision, and returns the stored text (`{ rev, inventory?, packs? }`). A site
/// tab keeps the newest revision it was given, so a tab created before a change never goes
/// back to the older delivery it was created with.
pub fn set_delivery(&self, delivery: Option<&str>, rev: u64) -> Result<String, String> {
let mut stored = serde_json::Map::new();
if let Some(text) = delivery {
if text.len() > MAX_DELIVERY_BYTES {
return Err("VALIDATION: the delivery is too large".into());
}
let value: serde_json::Value = serde_json::from_str(text).map_err(|_| "VALIDATION: the delivery is not JSON")?;
let inventory = value.get("inventory").filter(|v| v.is_object()).cloned();
let packs = value.get("packs").filter(|v| v.as_array().is_some_and(|packs| !packs.is_empty() && packs.len() <= MAX_ENABLED)).cloned();
let (Some(inventory), Some(packs)) = (inventory, packs) else {
return Err("VALIDATION: a delivery is { inventory, packs } with 1 to 8 packs".into());
};
stored.insert("inventory".into(), inventory);
stored.insert("packs".into(), packs);
}
stored.insert("rev".into(), rev.into());
let text = serde_json::Value::Object(stored).to_string();
std::fs::create_dir_all(&self.dir).map_err(|e| format!("STRINGS: {e}"))?;
write_atomic(&self.dir.join(DELIVERY_FILE), text.as_bytes())?;
Ok(text)
}
/// The stored delivery (`{ rev, inventory?, packs? }`), if the shell ever set one.
pub fn delivery(&self) -> Option<String> {
std::fs::read_to_string(self.dir.join(DELIVERY_FILE)).ok()
}
}
/// The session-storage key a page runtime reads its string packs from at document start.
pub const DELIVERY_KEY: &str = "wayBetter.stringPacks";
/// A script for a site tab: on one of `hosts` (https) only, puts `stored` (a stored delivery with
/// its `rev`) in the tab's session storage unless the tab already holds that revision or a newer
/// one; with `reload`, reloads the page after a change so the runtime applies it from the start.
/// Data only: the delivery is a JSON string literal, never code.
pub fn delivery_script(hosts: &[&str], stored: &str, reload: bool) -> String {
let hosts = serde_json::to_string(hosts).unwrap_or_else(|_| "[]".into());
let key = serde_json::to_string(DELIVERY_KEY).unwrap_or_default();
let rev = serde_json::from_str::<serde_json::Value>(stored).ok().and_then(|v| v.get("rev").and_then(serde_json::Value::as_u64)).unwrap_or(0);
let text = serde_json::to_string(stored).unwrap_or_else(|_| "\"{}\"".into());
format!(
"(() => {{ try {{ if (location.protocol !== 'https:' || !{hosts}.includes(location.host)) return; \
let held = 0; try {{ held = Number(JSON.parse(sessionStorage.getItem({key}) || '{{}}').rev) || 0; }} catch (e) {{}} \
if (held >= {rev}) return; sessionStorage.setItem({key}, {text});{} }} catch (e) {{}} }})();",
if reload { " location.reload();" } else { "" }
)
}
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
let tmp = path.with_extension("tmp");
std::fs::write(&tmp, bytes).map_err(|e| format!("STRINGS: {e}"))?;
std::fs::rename(&tmp, path).map_err(|e| format!("STRINGS: {e}"))
}
#[cfg(test)]
mod tests {
use super::*;
fn pack(strings: &str) -> Vec<u8> {
format!(r#"{{"format":1,"site":"x","catalog":"web","language":"eo","name":"Esperanto","authors":["A"],"version":"1.0.0","strings":{strings}}}"#).into_bytes()
}
#[test]
fn packs_are_plain_data_with_the_envelope() {
let info = validate_strings(&pack(r#"{"a0000001":"Hejmo","b0000002":"{count, plural, one {# afiŝo} other {# afiŝoj}}"}"#)).unwrap();
assert_eq!((info.language.as_str(), info.strings, info.name.as_str()), ("eo", 2, "Esperanto"));
for bad in [
pack(r#"{"a":"<b>x</b>"}"#),
pack(r#"{"a":"x > y"}"#),
pack(&format!(r#"{{"a":"{}x"}}"#, char::from_u32(0x202e).unwrap())),
pack(r#"{"a":""}"#),
pack(r#"{"a":1}"#),
pack(r#"{"a b":"x"}"#),
pack(r#"[]"#),
br#"{"format":2,"site":"x","catalog":"web","language":"eo","name":"E","authors":[],"strings":{}}"#.to_vec(),
br#"{"format":1,"site":"X","catalog":"web","language":"eo","name":"E","authors":[],"strings":{}}"#.to_vec(),
br#"{"format":1,"site":"x","catalog":"web","language":"not a tag","name":"E","authors":[],"strings":{}}"#.to_vec(),
br#"{"format":1,"site":"x","catalog":"web","language":"eo","name":"E","authors":[],"strings":{},"code":"x"}"#.to_vec(),
b"not json".to_vec(),
Vec::new(),
] {
assert!(validate_strings(&bad).is_err(), "{}", String::from_utf8_lossy(&bad));
}
assert!(valid_language("en-x-pirate") && valid_language("pt-BR") && !valid_language("e") && !valid_language("en_US"));
}
#[test]
fn install_order_enable_remove_and_delivery() {
let dir = std::env::temp_dir().join(format!("wb-strings-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let store = StringsStore::new(&dir);
let info = validate_strings(&pack(r#"{"a":"x"}"#)).unwrap();
assert_eq!(pack_id(&info, None), "eo-esperanto");
assert_eq!(pack_id(&info, Some("esperanto")), "esperanto");
store.install("eo-esperanto", &pack(r#"{"a":"x"}"#), true).unwrap();
store.install("second", &pack(r#"{"a":"y"}"#), false).unwrap();
let list = store.list();
assert_eq!(list.packs.iter().map(|p| (p.id.as_str(), p.enabled)).collect::<Vec<_>>(), [("eo-esperanto", true), ("second", false)]);
let list = store.set(&["second".into(), "nope".into()], &["second".into(), "eo-esperanto".into()]).unwrap();
assert_eq!(list.packs.iter().map(|p| (p.id.as_str(), p.enabled)).collect::<Vec<_>>(), [("second", true), ("eo-esperanto", true)]);
assert_eq!(store.enabled_texts().len(), 2);
assert!(store.install("../evil", &pack("{}"), true).is_err());
assert!(store.install("bad", b"{}", true).is_err());
let stored = store.set_delivery(Some(r#"{"inventory":{"site":"x"},"packs":[{"a":1}],"extra":"dropped"}"#), 7).unwrap();
assert_eq!(stored, r#"{"inventory":{"site":"x"},"packs":[{"a":1}],"rev":7}"#);
assert_eq!(store.delivery().as_deref(), Some(stored.as_str()));
assert!(store.set_delivery(Some("[]"), 8).is_err());
assert!(store.set_delivery(Some(r#"{"inventory":{},"packs":[]}"#), 8).is_err(), "no packs is None, not an empty list");
assert_eq!(store.list().packs.len(), 2, "the delivery file is not a pack");
assert_eq!(store.set_delivery(None, 9).unwrap(), r#"{"rev":9}"#);
store.remove("second").unwrap();
assert_eq!(store.list().packs.len(), 1);
assert_eq!(store.state().order, ["eo-esperanto"]);
let script = delivery_script(&["x.com"], r#"{"rev":12,"packs":["</script>"]}"#, true);
assert!(script.contains(r#"!["x.com"].includes(location.host)"#), "{script}");
assert!(script.contains("if (held >= 12) return;") && script.contains("sessionStorage.setItem(\"wayBetter.stringPacks\", \"{\\\"rev\\\":12"));
assert!(script.contains("location.reload();"));
assert!(!delivery_script(&["x.com"], r#"{"rev":1}"#, false).contains("reload"));
let _ = std::fs::remove_dir_all(&dir);
}
}