String packs: a page gets only the slice of the inventory its packs' keys need (sliceInventory, isPageInventory); a slice does not call other keys new
This commit is contained in:
parent
e09dc55e25
commit
57524ae2ff
3 changed files with 70 additions and 8 deletions
|
|
@ -83,7 +83,10 @@ no text of the site. `group` names the part of the catalog a key arrives with (d
|
||||||
`reworded` (the hash changed), `reshaped` (kind or parameters changed) and `vanished`. A key is
|
`reworded` (the hash changed), `reshaped` (kind or parameters changed) and `vanished`. A key is
|
||||||
called vanished only when its group's base catalog was actually read.
|
called vanished only when its group's base catalog was actually read.
|
||||||
|
|
||||||
`packInventory(inventory)` is the form a page runtime ships: ids and shapes, no hashes.
|
`packInventory(inventory)` is the compact form: ids and shapes, no hashes. A page does not need
|
||||||
|
the whole inventory: `sliceInventory(inventory, ids)` is the part that the packs' keys need, and it
|
||||||
|
travels to the page together with the packs (`isPageInventory` checks its shape on arrival). The
|
||||||
|
full inventory stays where packs are installed and checked.
|
||||||
|
|
||||||
## The engine
|
## The engine
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -116,19 +116,56 @@ export type InventoryLookup = (id: string) => EntryShape | undefined;
|
||||||
* the size. `text` and `closed` are space-separated ids; a message is `name name*` (`*` = plural,
|
* the size. `text` and `closed` are space-separated ids; a message is `name name*` (`*` = plural,
|
||||||
* `?` = select).
|
* `?` = select).
|
||||||
*/
|
*/
|
||||||
export type PageInventory = { site: string; catalog: string; text: string; message: Record<string, string>; closed: string };
|
export type PageInventory = {
|
||||||
|
site: string;
|
||||||
|
catalog: string;
|
||||||
|
text: string;
|
||||||
|
message: Record<string, string>;
|
||||||
|
closed: string;
|
||||||
|
/** A slice: only the keys some pack names. A key missing from a slice is not a new key of the site. */
|
||||||
|
partial?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export function packInventory(inventory: StringInventory): PageInventory {
|
function compact(site: string, catalog: string, shapes: Iterable<[string, EntryShape]>): PageInventory {
|
||||||
const text: string[] = [];
|
const text: string[] = [];
|
||||||
const closed: string[] = [];
|
const closed: string[] = [];
|
||||||
const message: Record<string, string> = {};
|
const message: Record<string, string> = {};
|
||||||
for (const [id, entry] of Object.entries(inventory.entries)) {
|
for (const [id, entry] of shapes) {
|
||||||
if (entry.kind === 'text' && !entry.tags?.length) text.push(id);
|
if (entry.kind === 'text' && !entry.tags?.length) text.push(id);
|
||||||
else if (entry.kind === 'message' && !entry.tags?.length) {
|
else if (entry.kind === 'message' && !entry.tags?.length) {
|
||||||
message[id] = (entry.params ?? []).map(name => `${name}${entry.plural?.includes(name) ? '*' : ''}${entry.select?.includes(name) ? '?' : ''}`).join(' ');
|
message[id] = (entry.params ?? []).map(name => `${name}${entry.plural?.includes(name) ? '*' : ''}${entry.select?.includes(name) ? '?' : ''}`).join(' ');
|
||||||
} else closed.push(id);
|
} else closed.push(id);
|
||||||
}
|
}
|
||||||
return { site: inventory.site, catalog: inventory.catalog, text: text.join(' '), message, closed: closed.join(' ') };
|
return { site, catalog, text: text.join(' '), message, closed: closed.join(' ') };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function packInventory(inventory: StringInventory): PageInventory {
|
||||||
|
return compact(inventory.site, inventory.catalog, Object.entries(inventory.entries));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The part of an inventory that the given keys need: what travels to a page together with the
|
||||||
|
* packs it should apply (a page never needs the rest). Keys the inventory does not have are left
|
||||||
|
* out, so they stay unknown there too.
|
||||||
|
*/
|
||||||
|
export function sliceInventory(inventory: StringInventory | PageInventory, ids: Iterable<string>): PageInventory {
|
||||||
|
const lookup = inventoryLookup(inventory);
|
||||||
|
const shapes: [string, EntryShape][] = [];
|
||||||
|
for (const id of new Set(ids)) {
|
||||||
|
const shape = lookup(id);
|
||||||
|
if (shape) shapes.push([id, shape]);
|
||||||
|
}
|
||||||
|
return { ...compact(inventory.site, inventory.catalog, shapes), partial: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether untrusted data has the shape of a page inventory (ids and parameter names only). */
|
||||||
|
export function isPageInventory(input: unknown): input is PageInventory {
|
||||||
|
if (!isRecord(input)) return false;
|
||||||
|
const ids = (value: unknown) => typeof value === 'string' && value.length <= 2_000_000 && /^[\w$. -]*$/.test(value);
|
||||||
|
return typeof input.site === 'string' && IDENT.test(input.site) && typeof input.catalog === 'string' && IDENT.test(input.catalog)
|
||||||
|
&& ids(input.text) && ids(input.closed) && isRecord(input.message)
|
||||||
|
&& Object.entries(input.message).every(([id, names]) => /^[\w$.-]+$/.test(id) && typeof names === 'string' && /^[\w$*? ]*$/.test(names))
|
||||||
|
&& (input.partial === undefined || typeof input.partial === 'boolean');
|
||||||
}
|
}
|
||||||
|
|
||||||
const TEXT_SHAPE: EntryShape = Object.freeze({ kind: 'text' });
|
const TEXT_SHAPE: EntryShape = Object.freeze({ kind: 'text' });
|
||||||
|
|
@ -409,6 +446,8 @@ const MAX_LISTED = 200;
|
||||||
/** `packs` in application order: a later pack wins over an earlier one for the same key. */
|
/** `packs` in application order: a later pack wins over an earlier one for the same key. */
|
||||||
export function createStringEngine(inventory: StringInventory | PageInventory | InventoryLookup, packs: readonly CompiledPack[]): StringEngine {
|
export function createStringEngine(inventory: StringInventory | PageInventory | InventoryLookup, packs: readonly CompiledPack[]): StringEngine {
|
||||||
const lookup = typeof inventory === 'function' ? inventory : inventoryLookup(inventory);
|
const lookup = typeof inventory === 'function' ? inventory : inventoryLookup(inventory);
|
||||||
|
// A slice only knows the packs' keys: what it lacks is not a new key of the site.
|
||||||
|
const whole = typeof inventory === 'function' || !('partial' in inventory && inventory.partial);
|
||||||
const originals = new Map<string, unknown>();
|
const originals = new Map<string, unknown>();
|
||||||
const applied = new Map<string, unknown>();
|
const applied = new Map<string, unknown>();
|
||||||
const produced = new WeakMap<object, unknown>();
|
const produced = new WeakMap<object, unknown>();
|
||||||
|
|
@ -432,7 +471,7 @@ export function createStringEngine(inventory: StringInventory | PageInventory |
|
||||||
seen.add(id);
|
seen.add(id);
|
||||||
const found = winner(id);
|
const found = winner(id);
|
||||||
if (!found) {
|
if (!found) {
|
||||||
if (unlisted.size < MAX_LISTED && !lookup(id)) unlisted.add(id);
|
if (whole && unlisted.size < MAX_LISTED && !lookup(id)) unlisted.add(id);
|
||||||
return original;
|
return original;
|
||||||
}
|
}
|
||||||
const shape = lookup(id);
|
const shape = lookup(id);
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { test } from 'node:test';
|
import { test } from 'node:test';
|
||||||
import {
|
import {
|
||||||
applyPacks, checkPackShape, compilePack, coverage, createStringEngine, driftReport, hashText, inventoryLookup, packInventory,
|
applyPacks, checkPackShape, compilePack, coverage, createStringEngine, driftReport, hashText, inventoryLookup, isPageInventory, packInventory,
|
||||||
renderMessage, restoreCatalog, validateString, type InventoryEntry, type StringInventory, type StringPack,
|
renderMessage, restoreCatalog, sliceInventory, validateString, type InventoryEntry, type StringInventory, type StringPack,
|
||||||
} from '../src/stringPacks.ts';
|
} from '../src/stringPacks.ts';
|
||||||
import { parseMessage } from '../src/i18n.ts';
|
import { parseMessage } from '../src/i18n.ts';
|
||||||
import { installStringPacks, mapChunkRequires, watchChunkArray, type CatalogHookAdapter, type CatalogSink } from '../src/stringPackHook.ts';
|
import { installStringPacks, mapChunkRequires, watchChunkArray, type CatalogHookAdapter, type CatalogSink } from '../src/stringPackHook.ts';
|
||||||
|
|
@ -226,6 +226,26 @@ test('page inventory: the compact form answers like the full one', () => {
|
||||||
assert.deepEqual(a.report.rejected.map(item => item.id), b.report.rejected.map(item => item.id));
|
assert.deepEqual(a.report.rejected.map(item => item.id), b.report.rejected.map(item => item.id));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('inventory slice: only what the packs name travels to the page, and decides the same', () => {
|
||||||
|
const strings = { a0000001: 'Hoom', b0000002: '{count, plural, one {#} other {#}}', c0000001: 'x', zzzzzzzz: 'y' };
|
||||||
|
const slice = sliceInventory(INVENTORY, Object.keys(strings));
|
||||||
|
assert.deepEqual(slice, { site: 'demo', catalog: 'web', text: 'a0000001', message: { b0000002: 'count* name' }, closed: 'c0000001', partial: true });
|
||||||
|
assert.deepEqual(sliceInventory(packInventory(INVENTORY), Object.keys(strings)), slice, 'a slice of the compact form is the same');
|
||||||
|
const full = compilePack(pack(strings), INVENTORY);
|
||||||
|
const sliced = compilePack(pack(strings), slice);
|
||||||
|
assert.deepEqual([...sliced.values.keys()], [...full.values.keys()]);
|
||||||
|
assert.deepEqual(sliced.report, full.report, 'unknown and refused keys are the same');
|
||||||
|
// A key outside the slice is the site's own value, and is not reported as a new key of the site.
|
||||||
|
const engine = createStringEngine(slice, [sliced]);
|
||||||
|
assert.equal(engine.resolve('a0000002', 'Following'), 'Following');
|
||||||
|
assert.equal(engine.resolve('a0000001', 'Home'), 'Hoom');
|
||||||
|
assert.deepEqual(engine.stats().unlisted, []);
|
||||||
|
// Untrusted data: the shape check.
|
||||||
|
assert.ok(isPageInventory(slice));
|
||||||
|
assert.ok(isPageInventory(JSON.parse(JSON.stringify(packInventory(INVENTORY)))));
|
||||||
|
for (const bad of [null, [], {}, { ...slice, text: 5 }, { ...slice, text: 'a <b>' }, { ...slice, message: { a: 'x<y' } }, { ...slice, site: 'Demo Site' }, { ...slice, partial: 'yes' }]) assert.equal(isPageInventory(bad), false);
|
||||||
|
});
|
||||||
|
|
||||||
test('coverage: per group, with what cannot be translated yet counted apart', () => {
|
test('coverage: per group, with what cannot be translated yet counted apart', () => {
|
||||||
const compiled = compilePack(pack({ a0000001: 'Hoom', b0000001: 'Ahoy {name}', e0000001: 'Grin' }), INVENTORY);
|
const compiled = compilePack(pack({ a0000001: 'Hoom', b0000001: 'Ahoy {name}', e0000001: 'Grin' }), INVENTORY);
|
||||||
assert.deepEqual(coverage(INVENTORY, compiled), {
|
assert.deepEqual(coverage(INVENTORY, compiled), {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue