wayBetter for X
This commit is contained in:
commit
2f1e35ba20
720 changed files with 147465 additions and 0 deletions
259
apps/web/i18n-html.ts
Normal file
259
apps/web/i18n-html.ts
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
// The site's pages are written in English, in the HTML. An element whose text is translated carries
|
||||
// `data-i18n="<key>"`; a translated attribute carries `data-i18n-<attribute>="<key>"` (aria-label,
|
||||
// alt, placeholder, title). From those marks this file
|
||||
// - extracts the English source messages (scripts/i18n-site.mjs writes them to locales/en.json),
|
||||
// - renders a page in another language (store-pages.ts, at build time), and
|
||||
// - removes the marks from what is published.
|
||||
//
|
||||
// A message is plain text in the framework's catalog format (ICU-lite, never HTML; docs/TRANSLATING.md).
|
||||
// Inline markup becomes placeholders named after the tag: `{a1}…{a1End}` wraps the words of the first
|
||||
// link, `{em1}…{em1End}` the first emphasis; an element that is not translated (code, a key, a number,
|
||||
// anything with translate="no") is one placeholder, `{code1}`. A translation may move them, and the
|
||||
// catalog check refuses one that drops or invents a placeholder.
|
||||
//
|
||||
// Pure string functions, no Node APIs: the build, the extract script and the tests share them.
|
||||
|
||||
export type HtmlNode =
|
||||
| { kind: 'text'; start: number; end: number }
|
||||
| { kind: 'comment'; start: number; end: number }
|
||||
| { kind: 'el'; tag: string; attrs: string; start: number; openEnd: number; closeStart: number; end: number; children: HtmlNode[] };
|
||||
type El = Extract<HtmlNode, { kind: 'el' }>;
|
||||
|
||||
const VOID = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'source', 'track', 'wbr']);
|
||||
const RAW = new Set(['script', 'style']);
|
||||
const TOKEN = /<!--[\s\S]*?-->|<!doctype[^>]*>|<\/([a-zA-Z][\w-]*)\s*>|<([a-zA-Z][\w-]*)((?:"[^"]*"|'[^']*'|[^'">])*)>/gi;
|
||||
|
||||
/** A light tree of `html` with source offsets. Enough for the site's own well-formed pages. */
|
||||
export function parseHtml(html: string): HtmlNode[] {
|
||||
const root: { children: HtmlNode[] } = { children: [] };
|
||||
const stack: { tag?: string; children: HtmlNode[] }[] = [root];
|
||||
const lower = html.toLowerCase();
|
||||
const re = new RegExp(TOKEN.source, 'gi');
|
||||
let last = 0;
|
||||
const text = (to: number) => { if (to > last) stack.at(-1)!.children.push({ kind: 'text', start: last, end: to }); };
|
||||
for (let match = re.exec(html); match; match = re.exec(html)) {
|
||||
text(match.index);
|
||||
last = re.lastIndex;
|
||||
if (match[0].startsWith('<!')) {
|
||||
stack.at(-1)!.children.push({ kind: 'comment', start: match.index, end: re.lastIndex });
|
||||
continue;
|
||||
}
|
||||
if (match[1]) {
|
||||
const tag = match[1].toLowerCase();
|
||||
let at = stack.length - 1;
|
||||
while (at > 0 && stack[at]!.tag !== tag) at -= 1;
|
||||
if (at === 0) continue;
|
||||
while (stack.length - 1 > at) { const open = stack.pop() as El; open.closeStart = open.end = match.index; }
|
||||
const el = stack.pop() as El;
|
||||
el.closeStart = match.index;
|
||||
el.end = re.lastIndex;
|
||||
continue;
|
||||
}
|
||||
const tag = match[2]!.toLowerCase();
|
||||
const el: El = { kind: 'el', tag, attrs: match[3] ?? '', start: match.index, openEnd: re.lastIndex, closeStart: re.lastIndex, end: re.lastIndex, children: [] };
|
||||
stack.at(-1)!.children.push(el);
|
||||
if (VOID.has(tag) || /\/\s*$/.test(el.attrs)) continue;
|
||||
if (RAW.has(tag)) {
|
||||
const close = lower.indexOf(`</${tag}`, re.lastIndex);
|
||||
const end = close < 0 ? html.length : html.indexOf('>', close) + 1;
|
||||
el.closeStart = close < 0 ? html.length : close;
|
||||
el.end = end;
|
||||
re.lastIndex = last = end;
|
||||
continue;
|
||||
}
|
||||
stack.push(el);
|
||||
}
|
||||
text(html.length);
|
||||
while (stack.length > 1) { const open = stack.pop() as El; open.closeStart = open.end = html.length; }
|
||||
return root.children;
|
||||
}
|
||||
|
||||
export function attrOf(el: El, name: string): string | null {
|
||||
const match = new RegExp(`(?:^|\\s)${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`).exec(el.attrs);
|
||||
return match ? match[1] ?? match[2] ?? '' : null;
|
||||
}
|
||||
|
||||
const ENTITIES: Record<string, string> = {
|
||||
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: '\u00a0', middot: '·', rarr: '→', larr: '←', rsquo: '’', lsquo: '‘', ldquo: '“', rdquo: '”',
|
||||
hellip: '…', mdash: '—', ndash: '–', empty: '∅', times: '×', copy: '©',
|
||||
};
|
||||
export function decodeEntities(text: string): string {
|
||||
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name: string) => {
|
||||
if (name[0] === '#') return String.fromCodePoint(name[1]!.toLowerCase() === 'x' ? parseInt(name.slice(2), 16) : Number(name.slice(1)));
|
||||
const value = ENTITIES[name];
|
||||
if (value === undefined) throw new Error(`i18n: unknown entity ${whole} (add it to ENTITIES in i18n-html.ts)`);
|
||||
return value;
|
||||
});
|
||||
}
|
||||
const escapeText = (text: string) => text.replace(/[&<>\u00a0]/g, ch => ({ '&': '&', '<': '<', '>': '>', '\u00a0': ' ' })[ch]!);
|
||||
const escapeAttr = (text: string) => text.replace(/[&<>"\u00a0]/g, ch => ({ '&': '&', '<': '<', '>': '>', '"': '"', '\u00a0': ' ' })[ch]!);
|
||||
|
||||
/**
|
||||
* Values the build fills in, as message placeholders: `<!-- store:count:theme -->` in a translated
|
||||
* sentence is `{themes}`, `%FORGE_HOST%` is `{host}`. Numbers may be pluralized by a translation.
|
||||
*/
|
||||
export const VALUE_TOKENS: Record<string, string> = {
|
||||
'<!-- store:count:theme -->': 'themes',
|
||||
'<!-- store:count:plugin -->': 'plugins',
|
||||
'<!-- store:total:all -->': 'items',
|
||||
'<!-- store:generated -->': 'date',
|
||||
'<!-- store:apps -->': 'apps',
|
||||
'<!-- store:since -->': 'version',
|
||||
'%FORGE_HOST%': 'host',
|
||||
};
|
||||
const TOKEN_SPLIT = /(%[A-Z_]+%)/;
|
||||
|
||||
const hasLetters = (text: string) => /\p{L}/u.test(text);
|
||||
const textOf = (html: string, nodes: HtmlNode[]): string => nodes.map(node => (node.kind === 'text' ? html.slice(node.start, node.end) : node.kind === 'el' ? textOf(html, node.children) : '')).join('');
|
||||
const OPAQUE_TAGS = new Set(['code', 'kbd', 'svg', 'img', 'br', 'video', 'pre', 'input', 'picture']);
|
||||
/** An element a translation carries along whole: code, keys, icons, numbers, and anything marked translate="no". */
|
||||
export function isOpaque(html: string, el: El): boolean {
|
||||
return OPAQUE_TAGS.has(el.tag) || attrOf(el, 'translate') === 'no' || !hasLetters(decodeEntities(textOf(html, el.children)));
|
||||
}
|
||||
/** Decoration at either end of a translated element (an icon, a dot, a caret, a symbol): it stays where it is, outside the message. */
|
||||
function isDecoration(html: string, el: El): boolean {
|
||||
return el.tag === 'svg' || el.tag === 'img' || attrOf(el, 'aria-hidden') === 'true' || !/[\p{L}\p{N}]/u.test(decodeEntities(textOf(html, el.children)));
|
||||
}
|
||||
const icuEscape = (text: string) => text.replace(/'(?=[{}#'])/g, "''").replace(/[{}]/g, ch => `'${ch}'`);
|
||||
|
||||
export type Message = {
|
||||
key: string;
|
||||
/** The English source, in catalog syntax. */
|
||||
source: string;
|
||||
/** Placeholder → the HTML it stands for (an opening tag, a closing tag, or a whole element). */
|
||||
slots: Record<string, string>;
|
||||
/** Placeholders the build gives a value (VALUE_TOKENS). */
|
||||
values: string[];
|
||||
/** Where the rendered message goes: offsets in the page, or the attribute it is the value of. */
|
||||
start: number;
|
||||
end: number;
|
||||
attr?: string;
|
||||
/** For translators: what each placeholder is. */
|
||||
notes: string[];
|
||||
};
|
||||
|
||||
const collapse = (text: string) => text.replace(/[ \t\r\n]+/g, ' ');
|
||||
|
||||
function valueText(text: string, values: Set<string>): string {
|
||||
return text.split(TOKEN_SPLIT).map(part => {
|
||||
if (!TOKEN_SPLIT.test(part)) return icuEscape(part);
|
||||
const name = VALUE_TOKENS[part];
|
||||
if (!name) throw new Error(`i18n: ${part} inside a translated text has no placeholder name (VALUE_TOKENS in i18n-html.ts)`);
|
||||
values.add(name);
|
||||
return `{${name}}`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function contentMessage(html: string, el: El, key: string): Message | null {
|
||||
const kids = el.children;
|
||||
const skip = (node: HtmlNode) => (node.kind === 'text' ? !html.slice(node.start, node.end).trim() : node.kind === 'el' ? isDecoration(html, node) : !VALUE_TOKENS[html.slice(node.start, node.end)]);
|
||||
let first = 0;
|
||||
let last = kids.length - 1;
|
||||
while (first <= last && skip(kids[first]!)) first += 1;
|
||||
while (last >= first && skip(kids[last]!)) last -= 1;
|
||||
if (first > last) return null;
|
||||
const slots: Record<string, string> = {};
|
||||
const values = new Set<string>();
|
||||
const notes: string[] = [];
|
||||
const counters = new Map<string, number>();
|
||||
const name = (tag: string) => { const n = (counters.get(tag) ?? 0) + 1; counters.set(tag, n); return `${tag.replace(/[^a-z]/g, '')}${n}`; };
|
||||
const serialize = (nodes: HtmlNode[]): string => nodes.map(node => {
|
||||
const raw = html.slice(node.start, node.end);
|
||||
if (node.kind === 'text') return valueText(decodeEntities(raw), values);
|
||||
if (node.kind === 'comment') {
|
||||
const value = VALUE_TOKENS[raw];
|
||||
if (value) { values.add(value); return `{${value}}`; }
|
||||
if (/^<!-- (store|repo|seo|lang):/.test(raw)) throw new Error(`i18n: ${key} contains ${raw}, which has no placeholder name (VALUE_TOKENS in i18n-html.ts)`);
|
||||
return '';
|
||||
}
|
||||
if (attrOf(node, 'data-i18n') !== null) throw new Error(`i18n: ${key} contains another translated element (${attrOf(node, 'data-i18n')})`);
|
||||
const slot = name(node.tag);
|
||||
if (isOpaque(html, node)) {
|
||||
slots[slot] = raw;
|
||||
const shown = collapse(decodeEntities(textOf(html, node.children))).trim();
|
||||
notes.push(`{${slot}} is ${shown ? `“${shown}”` : node.tag === 'br' ? 'a line break' : 'an icon'}${shown ? ', not translated' : ''}`);
|
||||
return `{${slot}}`;
|
||||
}
|
||||
slots[slot] = html.slice(node.start, node.openEnd);
|
||||
slots[`${slot}End`] = html.slice(node.closeStart, node.end);
|
||||
notes.push(`{${slot}}…{${slot}End} wraps ${({ a: 'the words of a link', em: 'the emphasized words', b: 'the bold words', strong: 'the bold words', small: 'the small print' } as Record<string, string>)[node.tag] ?? 'styled words'}`);
|
||||
return `{${slot}}${serialize(node.children)}{${slot}End}`;
|
||||
}).join('');
|
||||
const source = collapse(serialize(kids.slice(first, last + 1))).trim();
|
||||
if (!source) return null;
|
||||
return { key, source, slots, values: [...values], start: kids[first]!.start, end: kids[last]!.end, notes };
|
||||
}
|
||||
|
||||
const ATTR_MARK = /(?:^|\s)data-i18n-([a-z-]+)\s*=\s*"([^"]*)"/g;
|
||||
|
||||
/** Every marked message of a page, in page order. */
|
||||
export function pageMessages(html: string): Message[] {
|
||||
const out: Message[] = [];
|
||||
const walk = (nodes: HtmlNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (node.kind !== 'el') continue;
|
||||
for (const mark of node.attrs.matchAll(ATTR_MARK)) {
|
||||
const attr = mark[1]!;
|
||||
const match = new RegExp(`(?:^|\\s)${attr}\\s*=\\s*"([^"]*)"`).exec(node.attrs);
|
||||
if (!match) throw new Error(`i18n: ${mark[2]} marks the attribute ${attr}, which the element does not have`);
|
||||
const values = new Set<string>();
|
||||
const valueStart = node.start + 1 + node.tag.length + match.index + match[0].length - match[1]!.length - 1;
|
||||
out.push({ key: mark[2]!, source: collapse(valueText(decodeEntities(match[1]!), values)).trim(), slots: {}, values: [...values], start: valueStart, end: valueStart + match[1]!.length, attr, notes: [] });
|
||||
}
|
||||
const key = attrOf(node, 'data-i18n');
|
||||
if (key !== null) {
|
||||
const message = contentMessage(html, node, key);
|
||||
if (!message) throw new Error(`i18n: ${key} marks an element with nothing to translate`);
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
if (!RAW.has(node.tag)) walk(node.children);
|
||||
}
|
||||
};
|
||||
walk(parseHtml(html));
|
||||
return out;
|
||||
}
|
||||
|
||||
const OPEN = String.fromCharCode(0xe000);
|
||||
const CLOSE = String.fromCharCode(0xe001);
|
||||
|
||||
/**
|
||||
* The page in another language: every marked message replaced by `translate(key, params)`.
|
||||
* Markup placeholders are filled with the page's own tags, so a translation can only move them.
|
||||
* `values` are what the build knows (counts, the catalog date); a value it does not give stays in
|
||||
* the page as its token, for the step that fills it.
|
||||
*/
|
||||
export function localizeHtml(html: string, translate: (key: string, params: Record<string, unknown>) => string, values: Record<string, string | number> = {}): string {
|
||||
const messages = pageMessages(html).sort((a, b) => b.start - a.start);
|
||||
const tokenOf = Object.fromEntries(Object.entries(VALUE_TOKENS).map(([token, name]) => [name, token]));
|
||||
let out = html;
|
||||
for (const message of messages) {
|
||||
const params: Record<string, unknown> = {};
|
||||
const fills: Record<string, string> = {};
|
||||
for (const [slot, fill] of Object.entries(message.slots)) { params[slot] = `${OPEN}${slot}${CLOSE}`; fills[slot] = fill; }
|
||||
for (const value of message.values) {
|
||||
if (value in values) params[value] = values[value];
|
||||
else { params[value] = `${OPEN}${value}${CLOSE}`; fills[value] = tokenOf[value]!; }
|
||||
}
|
||||
const text = translate(message.key, params);
|
||||
const escaped = (message.attr ? escapeAttr(text) : escapeText(text)).replace(new RegExp(`${OPEN}(\\w+)${CLOSE}`, 'g'), (_, slot: string) => fills[slot] ?? '');
|
||||
const original = out.slice(message.start, message.end);
|
||||
const lead = !message.attr && /^\s/.test(original) ? ' ' : '';
|
||||
const tail = !message.attr && /\s$/.test(original) ? ' ' : '';
|
||||
out = out.slice(0, message.start) + lead + escaped + tail + out.slice(message.end);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The published page carries no translation marks. */
|
||||
export const stripMarks = (html: string): string => html.replace(/\sdata-i18n(?:-[a-z-]+)?="[^"]*"/g, '');
|
||||
|
||||
/**
|
||||
* A message with markup, for the HTML the build writes itself (Store cards, item pages): `slots`
|
||||
* fill the markup placeholders (`{ a1: '<a href="…">', a1End: '</a>' }`), every other value is text.
|
||||
*/
|
||||
export function messageHtml(translate: (key: string, params: Record<string, unknown>) => string, key: string, values: Record<string, unknown> = {}, slots: Record<string, string> = {}): string {
|
||||
const params: Record<string, unknown> = { ...values };
|
||||
for (const slot of Object.keys(slots)) params[slot] = `${OPEN}${slot}${CLOSE}`;
|
||||
return escapeText(translate(key, params)).replace(new RegExp(`${OPEN}(\\w+)${CLOSE}`, 'g'), (_, slot: string) => slots[slot] ?? '');
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue