String packs: a page gets only the slice of the inventory its packs' keys need (sliceInventory, isPageInventory); a slice does not call other keys new

This commit is contained in:
CRTkafa 2026-10-01 03:30:21 +07:00
parent e09dc55e25
commit 57524ae2ff
3 changed files with 70 additions and 8 deletions

View file

@ -83,7 +83,10 @@ no text of the site. `group` names the part of the catalog a key arrives with (d
`reworded` (the hash changed), `reshaped` (kind or parameters changed) and `vanished`. A key is `reworded` (the hash changed), `reshaped` (kind or parameters changed) and `vanished`. A key is
called vanished only when its group's base catalog was actually read. called vanished only when its group's base catalog was actually read.
`packInventory(inventory)` is the form a page runtime ships: ids and shapes, no hashes. `packInventory(inventory)` is the compact form: ids and shapes, no hashes. A page does not need
the whole inventory: `sliceInventory(inventory, ids)` is the part that the packs' keys need, and it
travels to the page together with the packs (`isPageInventory` checks its shape on arrival). The
full inventory stays where packs are installed and checked.
## The engine ## The engine

View file

@ -116,19 +116,56 @@ export type InventoryLookup = (id: string) => EntryShape | undefined;
* the size. `text` and `closed` are space-separated ids; a message is `name name*` (`*` = plural, * the size. `text` and `closed` are space-separated ids; a message is `name name*` (`*` = plural,
* `?` = select). * `?` = select).
*/ */
export type PageInventory = { site: string; catalog: string; text: string; message: Record<string, string>; closed: string }; export type PageInventory = {
site: string;
catalog: string;
text: string;
message: Record<string, string>;
closed: string;
/** A slice: only the keys some pack names. A key missing from a slice is not a new key of the site. */
partial?: boolean;
};
export function packInventory(inventory: StringInventory): PageInventory { function compact(site: string, catalog: string, shapes: Iterable<[string, EntryShape]>): PageInventory {
const text: string[] = []; const text: string[] = [];
const closed: string[] = []; const closed: string[] = [];
const message: Record<string, string> = {}; const message: Record<string, string> = {};
for (const [id, entry] of Object.entries(inventory.entries)) { for (const [id, entry] of shapes) {
if (entry.kind === 'text' && !entry.tags?.length) text.push(id); if (entry.kind === 'text' && !entry.tags?.length) text.push(id);
else if (entry.kind === 'message' && !entry.tags?.length) { else if (entry.kind === 'message' && !entry.tags?.length) {
message[id] = (entry.params ?? []).map(name => `${name}${entry.plural?.includes(name) ? '*' : ''}${entry.select?.includes(name) ? '?' : ''}`).join(' '); message[id] = (entry.params ?? []).map(name => `${name}${entry.plural?.includes(name) ? '*' : ''}${entry.select?.includes(name) ? '?' : ''}`).join(' ');
} else closed.push(id); } else closed.push(id);
} }
return { site: inventory.site, catalog: inventory.catalog, text: text.join(' '), message, closed: closed.join(' ') }; return { site, catalog, text: text.join(' '), message, closed: closed.join(' ') };
}
export function packInventory(inventory: StringInventory): PageInventory {
return compact(inventory.site, inventory.catalog, Object.entries(inventory.entries));
}
/**
* The part of an inventory that the given keys need: what travels to a page together with the
* packs it should apply (a page never needs the rest). Keys the inventory does not have are left
* out, so they stay unknown there too.
*/
export function sliceInventory(inventory: StringInventory | PageInventory, ids: Iterable<string>): PageInventory {
const lookup = inventoryLookup(inventory);
const shapes: [string, EntryShape][] = [];
for (const id of new Set(ids)) {
const shape = lookup(id);
if (shape) shapes.push([id, shape]);
}
return { ...compact(inventory.site, inventory.catalog, shapes), partial: true };
}
/** Whether untrusted data has the shape of a page inventory (ids and parameter names only). */
export function isPageInventory(input: unknown): input is PageInventory {
if (!isRecord(input)) return false;
const ids = (value: unknown) => typeof value === 'string' && value.length <= 2_000_000 && /^[\w$. -]*$/.test(value);
return typeof input.site === 'string' && IDENT.test(input.site) && typeof input.catalog === 'string' && IDENT.test(input.catalog)
&& ids(input.text) && ids(input.closed) && isRecord(input.message)
&& Object.entries(input.message).every(([id, names]) => /^[\w$.-]+$/.test(id) && typeof names === 'string' && /^[\w$*? ]*$/.test(names))
&& (input.partial === undefined || typeof input.partial === 'boolean');
} }
const TEXT_SHAPE: EntryShape = Object.freeze({ kind: 'text' }); const TEXT_SHAPE: EntryShape = Object.freeze({ kind: 'text' });
@ -409,6 +446,8 @@ const MAX_LISTED = 200;
/** `packs` in application order: a later pack wins over an earlier one for the same key. */ /** `packs` in application order: a later pack wins over an earlier one for the same key. */
export function createStringEngine(inventory: StringInventory | PageInventory | InventoryLookup, packs: readonly CompiledPack[]): StringEngine { export function createStringEngine(inventory: StringInventory | PageInventory | InventoryLookup, packs: readonly CompiledPack[]): StringEngine {
const lookup = typeof inventory === 'function' ? inventory : inventoryLookup(inventory); const lookup = typeof inventory === 'function' ? inventory : inventoryLookup(inventory);
// A slice only knows the packs' keys: what it lacks is not a new key of the site.
const whole = typeof inventory === 'function' || !('partial' in inventory && inventory.partial);
const originals = new Map<string, unknown>(); const originals = new Map<string, unknown>();
const applied = new Map<string, unknown>(); const applied = new Map<string, unknown>();
const produced = new WeakMap<object, unknown>(); const produced = new WeakMap<object, unknown>();
@ -432,7 +471,7 @@ export function createStringEngine(inventory: StringInventory | PageInventory |
seen.add(id); seen.add(id);
const found = winner(id); const found = winner(id);
if (!found) { if (!found) {
if (unlisted.size < MAX_LISTED && !lookup(id)) unlisted.add(id); if (whole && unlisted.size < MAX_LISTED && !lookup(id)) unlisted.add(id);
return original; return original;
} }
const shape = lookup(id); const shape = lookup(id);

View file

@ -1,8 +1,8 @@
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import { test } from 'node:test'; import { test } from 'node:test';
import { import {
applyPacks, checkPackShape, compilePack, coverage, createStringEngine, driftReport, hashText, inventoryLookup, packInventory, applyPacks, checkPackShape, compilePack, coverage, createStringEngine, driftReport, hashText, inventoryLookup, isPageInventory, packInventory,
renderMessage, restoreCatalog, validateString, type InventoryEntry, type StringInventory, type StringPack, renderMessage, restoreCatalog, sliceInventory, validateString, type InventoryEntry, type StringInventory, type StringPack,
} from '../src/stringPacks.ts'; } from '../src/stringPacks.ts';
import { parseMessage } from '../src/i18n.ts'; import { parseMessage } from '../src/i18n.ts';
import { installStringPacks, mapChunkRequires, watchChunkArray, type CatalogHookAdapter, type CatalogSink } from '../src/stringPackHook.ts'; import { installStringPacks, mapChunkRequires, watchChunkArray, type CatalogHookAdapter, type CatalogSink } from '../src/stringPackHook.ts';
@ -226,6 +226,26 @@ test('page inventory: the compact form answers like the full one', () => {
assert.deepEqual(a.report.rejected.map(item => item.id), b.report.rejected.map(item => item.id)); assert.deepEqual(a.report.rejected.map(item => item.id), b.report.rejected.map(item => item.id));
}); });
test('inventory slice: only what the packs name travels to the page, and decides the same', () => {
const strings = { a0000001: 'Hoom', b0000002: '{count, plural, one {#} other {#}}', c0000001: 'x', zzzzzzzz: 'y' };
const slice = sliceInventory(INVENTORY, Object.keys(strings));
assert.deepEqual(slice, { site: 'demo', catalog: 'web', text: 'a0000001', message: { b0000002: 'count* name' }, closed: 'c0000001', partial: true });
assert.deepEqual(sliceInventory(packInventory(INVENTORY), Object.keys(strings)), slice, 'a slice of the compact form is the same');
const full = compilePack(pack(strings), INVENTORY);
const sliced = compilePack(pack(strings), slice);
assert.deepEqual([...sliced.values.keys()], [...full.values.keys()]);
assert.deepEqual(sliced.report, full.report, 'unknown and refused keys are the same');
// A key outside the slice is the site's own value, and is not reported as a new key of the site.
const engine = createStringEngine(slice, [sliced]);
assert.equal(engine.resolve('a0000002', 'Following'), 'Following');
assert.equal(engine.resolve('a0000001', 'Home'), 'Hoom');
assert.deepEqual(engine.stats().unlisted, []);
// Untrusted data: the shape check.
assert.ok(isPageInventory(slice));
assert.ok(isPageInventory(JSON.parse(JSON.stringify(packInventory(INVENTORY)))));
for (const bad of [null, [], {}, { ...slice, text: 5 }, { ...slice, text: 'a <b>' }, { ...slice, message: { a: 'x<y' } }, { ...slice, site: 'Demo Site' }, { ...slice, partial: 'yes' }]) assert.equal(isPageInventory(bad), false);
});
test('coverage: per group, with what cannot be translated yet counted apart', () => { test('coverage: per group, with what cannot be translated yet counted apart', () => {
const compiled = compilePack(pack({ a0000001: 'Hoom', b0000001: 'Ahoy {name}', e0000001: 'Grin' }), INVENTORY); const compiled = compilePack(pack({ a0000001: 'Hoom', b0000001: 'Ahoy {name}', e0000001: 'Grin' }), INVENTORY);
assert.deepEqual(coverage(INVENTORY, compiled), { assert.deepEqual(coverage(INVENTORY, compiled), {