Store: two catalog files from one source, so apps from before a kind keep a working Store
An app that does not know a catalog item's kind refuses the whole signed file, so the icons and builtin kinds can never go into catalog.json. `waybetter store build` now writes catalog.json (format 2: theme, plugin, strings only; what every app reads today) and catalog.v3.json (format 3: every kind) from the same items; `sign` and `verify` handle the pair; the rules refuse a format 2 file that lists a newer kind (CLI, JSON Schema, host). The host reads catalog.v3.json first and falls back to catalog.json only when the newer file is not there (a newer file that does not verify is an error). An item of a kind a version does not know is skipped instead of refusing the file, so the next new kind costs current apps nothing.
This commit is contained in:
parent
a8e5e20ef4
commit
8588c5e63c
6 changed files with 165 additions and 35 deletions
|
|
@ -17,8 +17,15 @@ pub const DEFAULT_HOSTS: &[AllowedHost] = &[
|
||||||
AllowedHost { host: "cdn1.crt.fyi", raw_files_only: false },
|
AllowedHost { host: "cdn1.crt.fyi", raw_files_only: false },
|
||||||
AllowedHost { host: "git.crt.fyi", raw_files_only: true },
|
AllowedHost { host: "git.crt.fyi", raw_files_only: true },
|
||||||
];
|
];
|
||||||
/// The catalog format this code reads. Format 2: every item names the apps it is for (`apps`).
|
/// The catalog format every app reads. Format 2: every item names the apps it is for (`apps`), and only
|
||||||
|
/// the kinds every app knows (theme, plugin, strings): an app from before a kind existed refuses a
|
||||||
|
/// whole catalog that lists it, so `catalog.json` never does.
|
||||||
pub const CATALOG_FORMAT: u32 = 2;
|
pub const CATALOG_FORMAT: u32 = 2;
|
||||||
|
/// Format 3 is the same file with every kind (icons, builtin, ...), as `catalog.v3.json`. Apps that
|
||||||
|
/// know it read it first and fall back to `catalog.json` when it is not published.
|
||||||
|
pub const CATALOG_FORMAT_NEXT: u32 = 3;
|
||||||
|
pub const CATALOG_FILE: &str = "catalog.json";
|
||||||
|
pub const CATALOG_FILE_NEXT: &str = "catalog.v3.json";
|
||||||
pub const MAX_THEME_BYTES: usize = 2 * 1024 * 1024;
|
pub const MAX_THEME_BYTES: usize = 2 * 1024 * 1024;
|
||||||
/// An icon pack file (the runtime's `ICON_LIMITS.packBytes`).
|
/// An icon pack file (the runtime's `ICON_LIMITS.packBytes`).
|
||||||
pub const MAX_ICONS_BYTES: usize = 120_000;
|
pub const MAX_ICONS_BYTES: usize = 120_000;
|
||||||
|
|
@ -82,6 +89,11 @@ impl Kind {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Kinds every app knows, the only ones a format 2 catalog may list.
|
||||||
|
pub fn is_legacy(self) -> bool {
|
||||||
|
matches!(self, Kind::Theme | Kind::Plugin | Kind::Strings)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn as_str(self) -> &'static str {
|
pub fn as_str(self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Kind::Theme => "theme",
|
Kind::Theme => "theme",
|
||||||
|
|
@ -380,6 +392,16 @@ pub struct CatalogItem {
|
||||||
pub credits: Option<String>,
|
pub credits: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The catalog as signed, before its items are looked at one by one.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||||
|
struct RawCatalog {
|
||||||
|
format: u32,
|
||||||
|
name: String,
|
||||||
|
generated_at: String,
|
||||||
|
items: Vec<serde_json::Value>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||||
pub struct Catalog {
|
pub struct Catalog {
|
||||||
|
|
@ -539,13 +561,27 @@ pub fn verify_catalog(config: &StoreConfig, body: &[u8], signature: &str) -> Res
|
||||||
if !verified {
|
if !verified {
|
||||||
return Err("STORE: the catalog signature does not verify".into());
|
return Err("STORE: the catalog signature does not verify".into());
|
||||||
}
|
}
|
||||||
let catalog: Catalog = serde_json::from_slice(body).map_err(|error| format!("STORE: catalog schema: {error}"))?;
|
let raw: RawCatalog = serde_json::from_slice(body).map_err(|error| format!("STORE: catalog schema: {error}"))?;
|
||||||
if catalog.format != CATALOG_FORMAT {
|
if raw.format != CATALOG_FORMAT && raw.format != CATALOG_FORMAT_NEXT {
|
||||||
return Err("STORE: unsupported catalog format (update the app)".into());
|
return Err("STORE: unsupported catalog format (update the app)".into());
|
||||||
}
|
}
|
||||||
if catalog.items.len() > MAX_ITEMS {
|
if raw.items.len() > MAX_ITEMS {
|
||||||
return Err("STORE: too many items".into());
|
return Err("STORE: too many items".into());
|
||||||
}
|
}
|
||||||
|
// An item of a kind this version does not know is skipped, not a reason to refuse the whole
|
||||||
|
// signed file (a later catalog may list kinds this version has no use for). Everything else
|
||||||
|
// about an item is held to the rules below, as ever.
|
||||||
|
let mut items = Vec::with_capacity(raw.items.len());
|
||||||
|
for value in raw.items {
|
||||||
|
if value.get("kind").and_then(serde_json::Value::as_str).is_some_and(|kind| Kind::parse(kind).is_err()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
items.push(serde_json::from_value::<CatalogItem>(value).map_err(|error| format!("STORE: catalog schema: {error}"))?);
|
||||||
|
}
|
||||||
|
if raw.format == CATALOG_FORMAT && items.iter().any(|item| !item.kind.is_legacy()) {
|
||||||
|
return Err("STORE: a format 2 catalog lists only theme, plugin and strings items".into());
|
||||||
|
}
|
||||||
|
let catalog = Catalog { format: raw.format, name: raw.name, generated_at: raw.generated_at, items };
|
||||||
let mut ids = HashSet::new();
|
let mut ids = HashSet::new();
|
||||||
let mut files = HashSet::new();
|
let mut files = HashSet::new();
|
||||||
for item in &catalog.items {
|
for item in &catalog.items {
|
||||||
|
|
@ -745,7 +781,11 @@ pub(crate) mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn catalog_json(items: serde_json::Value) -> Vec<u8> {
|
pub(crate) fn catalog_json(items: serde_json::Value) -> Vec<u8> {
|
||||||
serde_json::to_vec(&serde_json::json!({ "format": CATALOG_FORMAT, "name": "Test", "generatedAt": "2026-09-28T00:00:00Z", "items": items })).unwrap()
|
catalog_json_format(CATALOG_FORMAT, items)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn catalog_json_format(format: u32, items: serde_json::Value) -> Vec<u8> {
|
||||||
|
serde_json::to_vec(&serde_json::json!({ "format": format, "name": "Test", "generatedAt": "2026-09-28T00:00:00Z", "items": items })).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn plugin_item(body: &[u8]) -> serde_json::Value {
|
pub(crate) fn plugin_item(body: &[u8]) -> serde_json::Value {
|
||||||
|
|
@ -908,8 +948,14 @@ pub(crate) mod tests {
|
||||||
fn icon_packs_and_built_in_items_are_catalog_kinds() {
|
fn icon_packs_and_built_in_items_are_catalog_kinds() {
|
||||||
let (key, public) = test_key(7);
|
let (key, public) = test_key(7);
|
||||||
let c = config("x", &[public]);
|
let c = config("x", &[public]);
|
||||||
let body = catalog_json(serde_json::json!([icons_item(ICONS), builtin_item()]));
|
let body = catalog_json_format(CATALOG_FORMAT_NEXT, serde_json::json!([icons_item(ICONS), builtin_item()]));
|
||||||
let catalog = verify_catalog(c, &body, &sign(&key, &body)).unwrap();
|
let catalog = verify_catalog(c, &body, &sign(&key, &body)).unwrap();
|
||||||
|
// The newer kinds are a format 3 catalog only: a format 2 file that lists them is refused, so
|
||||||
|
// the file older apps read can never carry a kind they would refuse.
|
||||||
|
let old = catalog_json(serde_json::json!([icons_item(ICONS)]));
|
||||||
|
assert!(verify_catalog(c, &old, &sign(&key, &old)).unwrap_err().contains("format 2"));
|
||||||
|
let both = catalog_json_format(CATALOG_FORMAT_NEXT, serde_json::json!([plugin_item(PLUGIN)]));
|
||||||
|
assert_eq!(verify_catalog(c, &both, &sign(&key, &both)).unwrap().format, CATALOG_FORMAT_NEXT);
|
||||||
assert_eq!(catalog.items.iter().map(|item| item.kind).collect::<Vec<_>>(), [Kind::Icons, Kind::Builtin]);
|
assert_eq!(catalog.items.iter().map(|item| item.kind).collect::<Vec<_>>(), [Kind::Icons, Kind::Builtin]);
|
||||||
verify_item(&catalog.items[0], ICONS).unwrap();
|
verify_item(&catalog.items[0], ICONS).unwrap();
|
||||||
assert!(verify_item(&catalog.items[0], b"{}").is_err());
|
assert!(verify_item(&catalog.items[0], b"{}").is_err());
|
||||||
|
|
@ -950,14 +996,23 @@ pub(crate) mod tests {
|
||||||
big["size"] = serde_json::json!(MAX_ICONS_BYTES + 1);
|
big["size"] = serde_json::json!(MAX_ICONS_BYTES + 1);
|
||||||
cases.push(("icon pack over the cap", big));
|
cases.push(("icon pack over the cap", big));
|
||||||
for (what, item) in cases {
|
for (what, item) in cases {
|
||||||
let body = catalog_json(serde_json::json!([item]));
|
let body = catalog_json_format(CATALOG_FORMAT_NEXT, serde_json::json!([item]));
|
||||||
assert!(verify_catalog(c, &body, &sign(&key, &body)).is_err(), "accepted bad {what}");
|
assert!(verify_catalog(c, &body, &sign(&key, &body)).is_err(), "accepted bad {what}");
|
||||||
}
|
}
|
||||||
// Another app's built-in items are not this app's.
|
// Another app's built-in items are not this app's.
|
||||||
let mut other = builtin_item();
|
let mut other = builtin_item();
|
||||||
other["apps"] = serde_json::json!(["kick"]);
|
other["apps"] = serde_json::json!(["kick"]);
|
||||||
let body = catalog_json(serde_json::json!([other]));
|
let body = catalog_json_format(CATALOG_FORMAT_NEXT, serde_json::json!([other]));
|
||||||
assert!(verify_catalog(c, &body, &sign(&key, &body)).unwrap().items.is_empty());
|
assert!(verify_catalog(c, &body, &sign(&key, &body)).unwrap().items.is_empty());
|
||||||
|
// A kind this version has never heard of is skipped (a later catalog may list it); a malformed
|
||||||
|
// item of a known kind, or an item without a kind, still refuses the file.
|
||||||
|
let future = serde_json::json!({ "apps": ["x"], "kind": "widget", "id": "w", "name": "W", "version": "1", "extra": [1, 2] });
|
||||||
|
let body = catalog_json_format(CATALOG_FORMAT_NEXT, serde_json::json!([future.clone(), plugin_item(PLUGIN)]));
|
||||||
|
assert_eq!(verify_catalog(c, &body, &sign(&key, &body)).unwrap().items.iter().map(|item| item.id.as_str()).collect::<Vec<_>>(), ["tool"]);
|
||||||
|
let mut kindless = future;
|
||||||
|
kindless.as_object_mut().unwrap().remove("kind");
|
||||||
|
let body = catalog_json_format(CATALOG_FORMAT_NEXT, serde_json::json!([kindless]));
|
||||||
|
assert!(verify_catalog(c, &body, &sign(&key, &body)).is_err());
|
||||||
// Links and the shell's kind names: icons yes, built-in never.
|
// Links and the shell's kind names: icons yes, built-in never.
|
||||||
assert_eq!(parse_install_link(c, "waybetter://install?kind=icons&url=https://waybetter.fyi/store/icons/line-2.icons.json").unwrap().kind, Kind::Icons);
|
assert_eq!(parse_install_link(c, "waybetter://install?kind=icons&url=https://waybetter.fyi/store/icons/line-2.icons.json").unwrap().kind, Kind::Icons);
|
||||||
assert!(parse_install_link(c, "waybetter://install?kind=builtin&url=https://waybetter.fyi/a.json").unwrap_err().contains("built-in"));
|
assert!(parse_install_link(c, "waybetter://install?kind=builtin&url=https://waybetter.fyi/a.json").unwrap_err().contains("built-in"));
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
//! Store downloads (`net` feature): allow-listed https only, no redirects, no cookies, size caps.
|
//! Store downloads (`net` feature): allow-listed https only, no redirects, no cookies, size caps.
|
||||||
|
|
||||||
use super::catalog::{self as dist, StoreConfig, MAX_CATALOG_BYTES, MAX_SIGNATURE_BYTES};
|
use super::catalog::{self as dist, StoreConfig, CATALOG_FILE, CATALOG_FILE_NEXT, MAX_CATALOG_BYTES, MAX_SIGNATURE_BYTES};
|
||||||
use super::install::{InstallPreview, Store, StoreItemRef, StoreView};
|
use super::install::{InstallPreview, Store, StoreItemRef, StoreView};
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
@ -48,6 +48,13 @@ pub async fn fetch(config: &StoreConfig, url: &str, max: usize) -> Result<Vec<u8
|
||||||
Ok(bytes)
|
Ok(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A catalog file and its detached signature.
|
||||||
|
async fn fetch_pair(config: &StoreConfig, url: &str) -> Result<(Vec<u8>, String), String> {
|
||||||
|
let body = fetch(config, url, MAX_CATALOG_BYTES).await?;
|
||||||
|
let signature = fetch(config, &format!("{url}.sig"), MAX_SIGNATURE_BYTES).await?;
|
||||||
|
Ok((body, String::from_utf8(signature).map_err(|_| "STORE: invalid signature")?))
|
||||||
|
}
|
||||||
|
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Downloads what an install link points at and stages it for the confirmation screen.
|
/// Downloads what an install link points at and stages it for the confirmation screen.
|
||||||
pub async fn stage_link(&self, kind: &str, url: &str) -> Result<InstallPreview, String> {
|
pub async fn stage_link(&self, kind: &str, url: &str) -> Result<InstallPreview, String> {
|
||||||
|
|
@ -79,12 +86,16 @@ impl Store {
|
||||||
self.view(error)
|
self.view(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The newer catalog first (`catalog.v3.json`: every kind), then `catalog.json` (what every app
|
||||||
|
/// knows) when the newer one is not there. A newer catalog that is there but does not verify is an
|
||||||
|
/// error, never a reason to fall back.
|
||||||
async fn fetch_catalog(&self) -> Result<(), String> {
|
async fn fetch_catalog(&self) -> Result<(), String> {
|
||||||
let base = self.config().store_base;
|
let base = self.config().store_base;
|
||||||
let body = fetch(self.config(), &format!("{base}catalog.json"), MAX_CATALOG_BYTES).await?;
|
let pair = match fetch_pair(self.config(), &format!("{base}{CATALOG_FILE_NEXT}")).await {
|
||||||
let signature = fetch(self.config(), &format!("{base}catalog.json.sig"), MAX_SIGNATURE_BYTES).await?;
|
Ok(pair) => pair,
|
||||||
let signature = String::from_utf8(signature).map_err(|_| "STORE: invalid signature")?;
|
Err(_) => fetch_pair(self.config(), &format!("{base}{CATALOG_FILE}")).await?,
|
||||||
self.accept_catalog(&body, &signature)
|
};
|
||||||
|
self.accept_catalog(&pair.0, &pair.1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ JSON Schema: `waybetter/store-catalog.schema.json` (npm package). The rules apps
|
||||||
|
|
||||||
**Built-in items.** `builtin` lists something the app already has, such as one of its area plugins, so the Store can show it with the app's own switch (`StoreView`'s `areas` prop: whether the app has that id, whether it is on, turning it on or off). It has no `url`, `sha256`, `size`, `file`, `permissions`, `colors` or `language`; `Store::stage_*` refuses it, `verify_item` has nothing to verify, and an install link cannot name it. It is data for display only. An app that does not know the id just shows "Built in".
|
**Built-in items.** `builtin` lists something the app already has, such as one of its area plugins, so the Store can show it with the app's own switch (`StoreView`'s `areas` prop: whether the app has that id, whether it is on, turning it on or off). It has no `url`, `sha256`, `size`, `file`, `permissions`, `colors` or `language`; `Store::stage_*` refuses it, `verify_item` has nothing to verify, and an install link cannot name it. It is data for display only. An app that does not know the id just shows "Built in".
|
||||||
|
|
||||||
**Old apps.** An app from before a kind existed refuses the whole signed catalog that lists it (unknown kind: "catalog schema", the Store shows as unreachable). Publish a catalog with a new kind only once the apps that read it ship the kind.
|
**Two files, one source.** An app from before a kind existed refuses the whole signed catalog that lists it (the Store shows as unreachable), so a new kind never goes into the file those apps read. `waybetter store build` writes both from the same items: `catalog.json` (format 2, only `theme`, `plugin` and `strings`; unchanged for old apps) and `catalog.v3.json` (format 3, every kind), each with its own `.sig`; `sign` and `verify` handle the pair. The rules refuse a format 2 file that lists a newer kind. `Store::fetch_catalog` reads `catalog.v3.json` first and falls back to `catalog.json` only when the newer file is not there (a newer file that is there and does not verify is an error, not a fallback). An item of a kind a version does not know is skipped rather than refusing the whole file, so the next new kind costs current apps nothing; every other rule is unchanged.
|
||||||
|
|
||||||
Also refused: unknown fields anywhere, more than 500 items, a catalog over 1 MiB, a signature file
|
Also refused: unknown fields anywhere, more than 500 items, a catalog over 1 MiB, a signature file
|
||||||
over 512 bytes. A malformed item for another app refuses the whole catalog (one file, one set of
|
over 512 bytes. A malformed item for another app refuses the whole catalog (one file, one set of
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,19 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node
|
||||||
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
||||||
import { pathToFileURL } from 'node:url';
|
import { pathToFileURL } from 'node:url';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Two catalog files, one source. `catalog.json` is format 2 and lists only the kinds every app
|
||||||
|
* since the first release knows (theme, plugin, strings): an app from before a kind existed refuses
|
||||||
|
* a whole catalog that lists it. `catalog.v3.json` is format 3, lists everything (icons, builtin, ...)
|
||||||
|
* and is read first by apps that know it; they fall back to `catalog.json` when it is not published.
|
||||||
|
* `waybetter store build` writes both from the same items; `sign` and `verify` handle both.
|
||||||
|
*/
|
||||||
export const CATALOG_FORMAT = 2;
|
export const CATALOG_FORMAT = 2;
|
||||||
|
export const CATALOG_FORMAT_NEXT = 3;
|
||||||
|
export const CATALOG_FILE = 'catalog.json';
|
||||||
|
export const CATALOG_FILE_NEXT = 'catalog.v3.json';
|
||||||
|
/** The kinds format 2 may list; a newer kind is format 3 only. */
|
||||||
|
export const LEGACY_KINDS = ['theme', 'plugin', 'strings'];
|
||||||
export const DEFAULT_BASE = 'https://waybetter.fyi/store/';
|
export const DEFAULT_BASE = 'https://waybetter.fyi/store/';
|
||||||
/** The framework's download hosts (waybetter-host `DEFAULT_HOSTS`). */
|
/** The framework's download hosts (waybetter-host `DEFAULT_HOSTS`). */
|
||||||
export const DEFAULT_HOSTS = [
|
export const DEFAULT_HOSTS = [
|
||||||
|
|
@ -129,7 +141,7 @@ export function catalogProblems(catalog, { app, hosts = DEFAULT_HOSTS } = {}) {
|
||||||
const out = [];
|
const out = [];
|
||||||
if (!catalog || typeof catalog !== 'object') return ['catalog: not an object'];
|
if (!catalog || typeof catalog !== 'object') return ['catalog: not an object'];
|
||||||
for (const key of Object.keys(catalog)) if (!['format', 'name', 'generatedAt', 'items'].includes(key)) out.push(`catalog: unknown field ${key}`);
|
for (const key of Object.keys(catalog)) if (!['format', 'name', 'generatedAt', 'items'].includes(key)) out.push(`catalog: unknown field ${key}`);
|
||||||
if (catalog.format !== CATALOG_FORMAT) out.push(`catalog: format must be ${CATALOG_FORMAT}`);
|
if (catalog.format !== CATALOG_FORMAT && catalog.format !== CATALOG_FORMAT_NEXT) out.push(`catalog: format must be ${CATALOG_FORMAT} or ${CATALOG_FORMAT_NEXT}`);
|
||||||
if (typeof catalog.name !== 'string' || typeof catalog.generatedAt !== 'string') out.push('catalog: name and generatedAt are strings');
|
if (typeof catalog.name !== 'string' || typeof catalog.generatedAt !== 'string') out.push('catalog: name and generatedAt are strings');
|
||||||
if (!Array.isArray(catalog.items)) return [...out, 'catalog: items must be a list'];
|
if (!Array.isArray(catalog.items)) return [...out, 'catalog: items must be a list'];
|
||||||
if (catalog.items.length > MAX_ITEMS) out.push(`catalog: more than ${MAX_ITEMS} items`);
|
if (catalog.items.length > MAX_ITEMS) out.push(`catalog: more than ${MAX_ITEMS} items`);
|
||||||
|
|
@ -138,6 +150,7 @@ export function catalogProblems(catalog, { app, hosts = DEFAULT_HOSTS } = {}) {
|
||||||
for (const item of catalog.items) {
|
for (const item of catalog.items) {
|
||||||
const where = `item ${item?.kind}:${item?.id}`;
|
const where = `item ${item?.kind}:${item?.id}`;
|
||||||
for (const problem of itemProblems(item, app && item?.apps?.includes(app) ? hosts : undefined)) out.push(`${where}: invalid ${problem}`);
|
for (const problem of itemProblems(item, app && item?.apps?.includes(app) ? hosts : undefined)) out.push(`${where}: invalid ${problem}`);
|
||||||
|
if (catalog.format === CATALOG_FORMAT && !LEGACY_KINDS.includes(item?.kind)) out.push(`${where}: a format ${CATALOG_FORMAT} catalog lists only ${LEGACY_KINDS.join(', ')} (older apps refuse the whole file otherwise); this kind belongs in ${CATALOG_FILE_NEXT}`);
|
||||||
for (const a of Array.isArray(item?.apps) ? item.apps : []) {
|
for (const a of Array.isArray(item?.apps) ? item.apps : []) {
|
||||||
const id = `${a}/${item.kind}/${item.id}`;
|
const id = `${a}/${item.kind}/${item.id}`;
|
||||||
if (ids.has(id)) out.push(`${where}: duplicate for app ${a}`);
|
if (ids.has(id)) out.push(`${where}: duplicate for app ${a}`);
|
||||||
|
|
@ -157,7 +170,8 @@ export function catalogProblems(catalog, { app, hosts = DEFAULT_HOSTS } = {}) {
|
||||||
/**
|
/**
|
||||||
* Builds `config.app`'s part of the shared catalog in `out`: writes the files the config returns,
|
* Builds `config.app`'s part of the shared catalog in `out`: writes the files the config returns,
|
||||||
* replaces the app's items (other apps' items stay as they are), removes files only the app's old
|
* replaces the app's items (other apps' items stay as they are), removes files only the app's old
|
||||||
* items used, validates the whole catalog and writes `catalog.json` (sign it afterwards).
|
* items used, validates the whole catalog and writes `catalog.v3.json` (every item) and `catalog.json`
|
||||||
|
* (the kinds older apps know), sign both afterwards.
|
||||||
*
|
*
|
||||||
* config: { app, name?, base?, out?, hosts?, items(context) → [{ kind, id, name, author, version,
|
* config: { app, name?, base?, out?, hosts?, items(context) → [{ kind, id, name, author, version,
|
||||||
* description, category, tags, file (plugins), permissions (plugins), colors (themes), language
|
* description, category, tags, file (plugins), permissions (plugins), colors (themes), language
|
||||||
|
|
@ -173,11 +187,14 @@ export async function buildStore(config, options = {}) {
|
||||||
if (!/^https:\/\/[^/]+\/(.*\/)?$/.test(base)) throw new Error('base must be an https URL ending in /');
|
if (!/^https:\/\/[^/]+\/(.*\/)?$/.test(base)) throw new Error('base must be an https URL ending in /');
|
||||||
const out = resolve(options.out ?? config.out ?? '.');
|
const out = resolve(options.out ?? config.out ?? '.');
|
||||||
const hosts = config.hosts ?? DEFAULT_HOSTS;
|
const hosts = config.hosts ?? DEFAULT_HOSTS;
|
||||||
const catalogPath = join(out, 'catalog.json');
|
const catalogPath = join(out, CATALOG_FILE);
|
||||||
const previous = existsSync(catalogPath) ? JSON.parse(readFileSync(catalogPath, 'utf8')) : null;
|
const nextPath = join(out, CATALOG_FILE_NEXT);
|
||||||
|
// The full list (format 3) is what to start from: it has every app's items of every kind.
|
||||||
|
const source = existsSync(nextPath) ? nextPath : catalogPath;
|
||||||
|
const previous = existsSync(source) ? JSON.parse(readFileSync(source, 'utf8')) : null;
|
||||||
const kept = [];
|
const kept = [];
|
||||||
const oldFiles = new Set();
|
const oldFiles = new Set();
|
||||||
for (const item of previous?.format === CATALOG_FORMAT ? previous.items : []) {
|
for (const item of previous?.format === CATALOG_FORMAT || previous?.format === CATALOG_FORMAT_NEXT ? previous.items : []) {
|
||||||
if (!item.apps?.includes(app)) { kept.push(item); continue; }
|
if (!item.apps?.includes(app)) { kept.push(item); continue; }
|
||||||
for (const url of [item.url, item.preview]) if (url?.startsWith(base)) oldFiles.add(url.slice(base.length));
|
for (const url of [item.url, item.preview]) if (url?.startsWith(base)) oldFiles.add(url.slice(base.length));
|
||||||
const others = item.apps.filter(a => a !== app);
|
const others = item.apps.filter(a => a !== app);
|
||||||
|
|
@ -216,23 +233,27 @@ export async function buildStore(config, options = {}) {
|
||||||
items.push(item);
|
items.push(item);
|
||||||
}
|
}
|
||||||
const catalog = {
|
const catalog = {
|
||||||
format: CATALOG_FORMAT,
|
format: CATALOG_FORMAT_NEXT,
|
||||||
name: config.name ?? previous?.name ?? 'wayBetter Store',
|
name: config.name ?? previous?.name ?? 'wayBetter Store',
|
||||||
generatedAt: new Date().toISOString().replace(/\.\d+Z$/, 'Z'),
|
generatedAt: new Date().toISOString().replace(/\.\d+Z$/, 'Z'),
|
||||||
items: [...kept, ...items],
|
items: [...kept, ...items],
|
||||||
};
|
};
|
||||||
// Nothing is written unless the whole catalog is valid.
|
// Nothing is written unless the whole catalog is valid.
|
||||||
const problems = catalogProblems(catalog, { app, hosts });
|
// The same items twice: everything for apps that know the newer kinds, and what every app knows.
|
||||||
|
const legacy = { ...catalog, format: CATALOG_FORMAT, items: catalog.items.filter(item => LEGACY_KINDS.includes(item.kind)) };
|
||||||
|
const problems = [...catalogProblems(catalog, { app, hosts }), ...catalogProblems(legacy, { app, hosts })];
|
||||||
if (problems.length) throw new Error(`the catalog would be invalid:\n ${problems.join('\n ')}`);
|
if (problems.length) throw new Error(`the catalog would be invalid:\n ${problems.join('\n ')}`);
|
||||||
const json = `${JSON.stringify(catalog, null, 2)}\n`;
|
const json = `${JSON.stringify(catalog, null, 2)}\n`;
|
||||||
|
const legacyJson = `${JSON.stringify(legacy, null, 2)}\n`;
|
||||||
if (Buffer.byteLength(json) > MAX_CATALOG_BYTES) throw new Error('the catalog is larger than apps accept (1 MiB)');
|
if (Buffer.byteLength(json) > MAX_CATALOG_BYTES) throw new Error('the catalog is larger than apps accept (1 MiB)');
|
||||||
for (const { target, bytes } of files.values()) { mkdirSync(dirname(target), { recursive: true }); writeFileSync(target, bytes); }
|
for (const { target, bytes } of files.values()) { mkdirSync(dirname(target), { recursive: true }); writeFileSync(target, bytes); }
|
||||||
// Files this app's previous build wrote and no item uses any more (never another app's files).
|
// Files this app's previous build wrote and no item uses any more (never another app's files).
|
||||||
const stillUsed = new Set(kept.flatMap(item => [item.url, item.preview]).filter(url => url?.startsWith(base)).map(url => url.slice(base.length)));
|
const stillUsed = new Set(kept.flatMap(item => [item.url, item.preview]).filter(url => url?.startsWith(base)).map(url => url.slice(base.length)));
|
||||||
for (const path of oldFiles) if (!files.has(path) && !stillUsed.has(path)) rmSync(inside(path), { force: true });
|
for (const path of oldFiles) if (!files.has(path) && !stillUsed.has(path)) rmSync(inside(path), { force: true });
|
||||||
mkdirSync(out, { recursive: true });
|
mkdirSync(out, { recursive: true });
|
||||||
writeFileSync(catalogPath, json);
|
writeFileSync(nextPath, json);
|
||||||
return { catalog, path: catalogPath, items };
|
writeFileSync(catalogPath, legacyJson);
|
||||||
|
return { catalog, legacy, path: catalogPath, nextPath, items };
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- Keys and signatures ----------------------------------------------------------------------
|
// ---- Keys and signatures ----------------------------------------------------------------------
|
||||||
|
|
@ -303,6 +324,12 @@ function parse(argv) {
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** `catalog.json` and, next to it, `catalog.v3.json` when it exists (one source, two files); any other file alone. */
|
||||||
|
export function catalogFiles(file) {
|
||||||
|
const next = join(dirname(file), CATALOG_FILE_NEXT);
|
||||||
|
return file.endsWith(CATALOG_FILE) && !file.endsWith(CATALOG_FILE_NEXT) && existsSync(next) ? [file, next] : [file];
|
||||||
|
}
|
||||||
|
|
||||||
/** Runs `waybetter store …`; returns the lines to print. Throws on errors. */
|
/** Runs `waybetter store …`; returns the lines to print. Throws on errors. */
|
||||||
export async function storeCommand(argv) {
|
export async function storeCommand(argv) {
|
||||||
const [command, ...rest] = argv;
|
const [command, ...rest] = argv;
|
||||||
|
|
@ -314,7 +341,7 @@ export async function storeCommand(argv) {
|
||||||
const config = (await import(pathToFileURL(resolve(options.config)).href)).default;
|
const config = (await import(pathToFileURL(resolve(options.config)).href)).default;
|
||||||
const { path, items } = await buildStore(config, { out: options.out, base: options.base });
|
const { path, items } = await buildStore(config, { out: options.out, base: options.base });
|
||||||
const count = kind => items.filter(item => item.kind === kind).length;
|
const count = kind => items.filter(item => item.kind === kind).length;
|
||||||
return [`store: ${config.app}: ${count('plugin')} plugins, ${count('theme')} themes → ${path}`, 'Now sign it: waybetter store sign <catalog.json> --key <private key>'];
|
return [`store: ${config.app}: ${count('plugin')} plugins, ${count('theme')} themes, ${count('icons')} icon packs, ${count('builtin')} built-in → ${path} and ${CATALOG_FILE_NEXT}`, 'Now sign both: waybetter store sign <catalog.json> --key <private key>'];
|
||||||
}
|
}
|
||||||
if (command === 'sign') {
|
if (command === 'sign') {
|
||||||
if (!key) throw new Error('sign needs --key (or WAYBETTER_STORE_KEY)');
|
if (!key) throw new Error('sign needs --key (or WAYBETTER_STORE_KEY)');
|
||||||
|
|
@ -324,18 +351,26 @@ export async function storeCommand(argv) {
|
||||||
return [`${created ? 'Created the store key' : 'Store key already exists'}; public key ${publicKey} → ${options.pub[0]}`];
|
return [`${created ? 'Created the store key' : 'Store key already exists'}; public key ${publicKey} → ${options.pub[0]}`];
|
||||||
}
|
}
|
||||||
if (!existsSync(key)) throw new Error(`no store key at ${key} (create it once with --init)`);
|
if (!existsSync(key)) throw new Error(`no store key at ${key} (create it once with --init)`);
|
||||||
const publicKey = signFile(catalog, key);
|
const lines = [];
|
||||||
const keys = options.pub.length ? options.pub.map(file => readFileSync(file, 'utf8').trim()) : [publicKey];
|
for (const file of catalogFiles(catalog)) {
|
||||||
if (!verifyFile(catalog, keys)) throw new Error(`signature of ${catalog} does NOT verify with ${options.pub.join(', ') || 'the key'}`);
|
const publicKey = signFile(file, key);
|
||||||
return [`Signed and verified ${catalog}.sig`];
|
const keys = options.pub.length ? options.pub.map(pub => readFileSync(pub, 'utf8').trim()) : [publicKey];
|
||||||
|
if (!verifyFile(file, keys)) throw new Error(`signature of ${file} does NOT verify with ${options.pub.join(', ') || 'the key'}`);
|
||||||
|
lines.push(`Signed and verified ${file}.sig`);
|
||||||
|
}
|
||||||
|
return lines;
|
||||||
}
|
}
|
||||||
if (command === 'verify') {
|
if (command === 'verify') {
|
||||||
if (!options.pub.length) throw new Error('verify needs --pub');
|
if (!options.pub.length) throw new Error('verify needs --pub');
|
||||||
const keys = options.pub.map(file => readFileSync(file, 'utf8').trim());
|
const keys = options.pub.map(file => readFileSync(file, 'utf8').trim());
|
||||||
if (!verifyFile(catalog, keys)) throw new Error(`signature of ${catalog} does NOT verify with ${options.pub.join(', ')}`);
|
const lines = [];
|
||||||
const problems = catalogProblems(JSON.parse(readFileSync(catalog, 'utf8')), { app: options.app });
|
for (const file of catalogFiles(catalog)) {
|
||||||
if (problems.length) throw new Error(`${catalog} is signed but invalid:\n ${problems.join('\n ')}`);
|
if (!existsSync(`${file}.sig`) || !verifyFile(file, keys)) throw new Error(`signature of ${file} does NOT verify with ${options.pub.join(', ')}`);
|
||||||
return [`Verified ${catalog}.sig`];
|
const problems = catalogProblems(JSON.parse(readFileSync(file, 'utf8')), { app: options.app });
|
||||||
|
if (problems.length) throw new Error(`${file} is signed but invalid:\n ${problems.join('\n ')}`);
|
||||||
|
lines.push(`Verified ${file}.sig`);
|
||||||
|
}
|
||||||
|
return lines;
|
||||||
}
|
}
|
||||||
throw new Error(USAGE);
|
throw new Error(USAGE);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,14 @@
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "https://unpkg.com/waybetter/store-catalog.schema.json",
|
"$id": "https://unpkg.com/waybetter/store-catalog.schema.json",
|
||||||
"title": "wayBetter Store catalog",
|
"title": "wayBetter Store catalog",
|
||||||
"description": "The shared, signed catalog of the wayBetter Store (format 2). Every wayBetter app reads the same file (catalog.json, signed by catalog.json.sig) and keeps only the items whose `apps` list its id. Apps verify the Ed25519 signature over the exact bytes first, then these rules (waybetter-host store::catalog), then each download's sha256 and size. Build it with `waybetter store build`, sign it with `waybetter store sign`.",
|
"description": "The shared, signed catalog of the wayBetter Store, in two files built from one source: catalog.json (format 2: only theme, plugin and strings items, which every app knows) and catalog.v3.json (format 3: every kind, read first by apps that know the newer kinds; they fall back to catalog.json). Each is signed by its own .sig. Every wayBetter app reads the same files and keeps only the items whose `apps` list its id. Apps verify the Ed25519 signature over the exact bytes first, then these rules (waybetter-host store::catalog), then each download's sha256 and size. Build it with `waybetter store build`, sign it with `waybetter store sign`.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": ["format", "name", "generatedAt", "items"],
|
"required": ["format", "name", "generatedAt", "items"],
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
|
"if": { "properties": { "format": { "const": 2 } } },
|
||||||
|
"then": { "properties": { "items": { "items": { "properties": { "kind": { "enum": ["theme", "plugin", "strings"] } } } } } },
|
||||||
"properties": {
|
"properties": {
|
||||||
"format": { "const": 2 },
|
"format": { "enum": [2, 3], "description": "2: only theme, plugin and strings items (an app from before a kind existed refuses a whole catalog that lists it). 3: every kind." },
|
||||||
"name": { "type": "string" },
|
"name": { "type": "string" },
|
||||||
"generatedAt": { "type": "string", "description": "When the catalog was built (RFC 3339, UTC)." },
|
"generatedAt": { "type": "string", "description": "When the catalog was built (RFC 3339, UTC)." },
|
||||||
"items": { "type": "array", "maxItems": 500, "items": { "$ref": "#/$defs/item" } }
|
"items": { "type": "array", "maxItems": 500, "items": { "$ref": "#/$defs/item" } }
|
||||||
|
|
|
||||||
|
|
@ -112,7 +112,20 @@ test('icon packs are catalog items with a file; built-in items have none; the sc
|
||||||
const builtin = { kind: 'builtin', id: 'compose', name: 'Compose+', version: '1.0.0', category: 'writing', description: 'Writing tools.', preview: { path: 'previews/builtin-compose.svg', bytes: '<svg xmlns="http://www.w3.org/2000/svg"/>' } };
|
const builtin = { kind: 'builtin', id: 'compose', name: 'Compose+', version: '1.0.0', category: 'writing', description: 'Writing tools.', preview: { path: 'previews/builtin-compose.svg', bytes: '<svg xmlns="http://www.w3.org/2000/svg"/>' } };
|
||||||
const icons = { kind: 'icons', id: 'line-2', name: 'Line 2', version: '1.0.0', category: 'icons', path: 'icons/line-2.icons.json', bytes: ICONS, license: 'CC0-1.0' };
|
const icons = { kind: 'icons', id: 'line-2', name: 'Line 2', version: '1.0.0', category: 'icons', path: 'icons/line-2.icons.json', bytes: ICONS, license: 'CC0-1.0' };
|
||||||
await buildStore(config('x', [icons, builtin]), { out });
|
await buildStore(config('x', [icons, builtin]), { out });
|
||||||
const catalog = JSON.parse(readFileSync(join(out, 'catalog.json'), 'utf8'));
|
const catalog = JSON.parse(readFileSync(join(out, 'catalog.v3.json'), 'utf8'));
|
||||||
|
assert.equal(catalog.format, 3);
|
||||||
|
// Older apps refuse a whole catalog that lists a kind they do not know: catalog.json (format 2) has only the old kinds.
|
||||||
|
const legacy = JSON.parse(readFileSync(join(out, 'catalog.json'), 'utf8'));
|
||||||
|
assert.equal(legacy.format, 2);
|
||||||
|
assert.deepEqual(legacy.items.map(item => item.kind).filter(kind => kind === 'icons' || kind === 'builtin'), []);
|
||||||
|
assert.deepEqual(legacy.items.map(item => `${item.kind}:${item.id}`), catalog.items.filter(item => item.kind !== 'icons' && item.kind !== 'builtin').map(item => `${item.kind}:${item.id}`), 'the same items, minus the newer kinds');
|
||||||
|
assert.deepEqual(validate(SCHEMA, legacy), []);
|
||||||
|
assert.deepEqual(catalogProblems(legacy, { app: 'x' }), []);
|
||||||
|
assert.notDeepEqual(catalogProblems({ ...legacy, items: catalog.items }, { app: 'x' }), [], 'a format 2 catalog may not list icons or builtin');
|
||||||
|
assert.notDeepEqual(validate(SCHEMA, { ...legacy, items: catalog.items }), []);
|
||||||
|
// A later build starts from the full list, so the newer kinds survive another build.
|
||||||
|
await buildStore(config('x', [icons, builtin]), { out });
|
||||||
|
assert.equal(JSON.parse(readFileSync(join(out, 'catalog.v3.json'), 'utf8')).items.filter(item => item.kind === 'icons').length, 1);
|
||||||
const pack = catalog.items.find(item => item.kind === 'icons');
|
const pack = catalog.items.find(item => item.kind === 'icons');
|
||||||
assert.deepEqual([pack.url, pack.sha256, pack.size, pack.license], ['https://waybetter.fyi/store/icons/line-2.icons.json', sha256(Buffer.from(ICONS)), Buffer.byteLength(ICONS), 'CC0-1.0']);
|
assert.deepEqual([pack.url, pack.sha256, pack.size, pack.license], ['https://waybetter.fyi/store/icons/line-2.icons.json', sha256(Buffer.from(ICONS)), Buffer.byteLength(ICONS), 'CC0-1.0']);
|
||||||
const area = catalog.items.find(item => item.kind === 'builtin');
|
const area = catalog.items.find(item => item.kind === 'builtin');
|
||||||
|
|
@ -144,3 +157,17 @@ test('icon packs are catalog items with a file; built-in items have none; the sc
|
||||||
// Building refuses an icon pack with no file as well.
|
// Building refuses an icon pack with no file as well.
|
||||||
await assert.rejects(buildStore(config('x', [{ kind: 'icons', id: 'bare', name: 'Bare', version: '1' }]), { out }));
|
await assert.rejects(buildStore(config('x', [{ kind: 'icons', id: 'bare', name: 'Bare', version: '1' }]), { out }));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('sign and verify handle both catalog files; a verify fails when either signature is wrong', async () => {
|
||||||
|
const out = mkdtempSync(join(tmpdir(), 'wb-store-'));
|
||||||
|
await buildStore(config('x', [{ kind: 'builtin', id: 'compose', name: 'Compose+', version: '1.0.0', category: 'writing' }]), { out });
|
||||||
|
const key = join(out, 'keys', 'store.key');
|
||||||
|
const pub = join(out, 'store.pub');
|
||||||
|
run('store', 'sign', '--init', '--key', key, '--pub', pub);
|
||||||
|
const signed = run('store', 'sign', join(out, 'catalog.json'), '--key', key, '--pub', pub);
|
||||||
|
assert.match(signed, /catalog.json.sig/);
|
||||||
|
assert.match(signed, /catalog.v3.json.sig/);
|
||||||
|
assert.equal(run('store', 'verify', join(out, 'catalog.json'), '--pub', pub, '--app', 'x').trim().split(/[\r\n]+/).length, 2);
|
||||||
|
writeFileSync(join(out, 'catalog.v3.json'), `${readFileSync(join(out, 'catalog.v3.json'), 'utf8')} `);
|
||||||
|
assert.match(fails('store', 'verify', join(out, 'catalog.json'), '--pub', pub), /catalog.v3.json does NOT verify/);
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue