waybetter-for-x/apps/desktop/scripts/strings-inventory.mjs

// Builds X's string inventory for string packs (waybetter/docs/STRING-PACKS.md) from X's own
// language chunks, without keeping any of X's text: key ids, kinds, parameter names and a hash of
// the English value.
//
//   node scripts/strings-inventory.mjs --main <file|url> [--emoji <file|url>] [--build <label>] [--check]
//   node scripts/strings-inventory.mjs --cdp <port>        prints the chunk URLs of the running build
//
// `--main` is the ENGLISH chunk of the signed-in web client (`…/client-web/i18n/en.<hash>.js`),
// `--emoji` its emoji names (`…/i18n/emoji-en.<hash>.js`). Both are static files on X's CDN; their
// names come from X's own webpack runtime, which `--cdp` asks in a background tab of its own
// (release app started with WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS=--remote-debugging-port=<port>;
// it only reads, and closes its tab). Without `--emoji` the emoji group of the existing inventory
// is kept as it is.
//
// Writes strings/x.web.inventory.json (the inventory) and strings/x.web.page.json (its compact form:
// ids and shapes, what checking a pack needs), and prints the drift against the inventory that was there: new, reworded,
// reshaped and vanished keys. `--check` writes nothing and exits 1 on any drift.
//
// The catalog this reads (the webpack client X serves to signed-in accounts): one chunk per
// language with one module that does
//   var o = require(<registry>)._register("en", {…});   o("<id>", <value>), o("<id>", <value>), …
// where a value is a string, a `function (e) { return "…" + e.name + … }`, a compiled formatter, or
// a React component; sentences with element slots are getters of a component class in the same
// chunk.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { dirname, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { driftReport, hashText, packInventory } from '../../../waybetter/packages/runtime/src/stringPacks.ts';

const root = join(dirname(fileURLToPath(import.meta.url)), '..');
export const INVENTORY_FILE = join(root, 'strings/x.web.inventory.json');
export const PAGE_FILE = join(root, 'strings/x.web.page.json');
export const SITE = 'x';
export const CATALOG = 'web';

const walk = (node, visit) => {
  if (!node || typeof node.type !== 'string') return;
  if (visit(node) === false) return;
  for (const key of Object.keys(node)) {
    const value = node[key];
    if (Array.isArray(value)) for (const child of value) walk(child, visit);
    else if (value && typeof value.type === 'string') walk(value, visit);
  }
};

class NotText extends Error {}

/**
 * A message function's return expression as canonical text: literals as they are, `e.name` as
 * `{name}`, a plural helper call as `{name, plural, a | b}`. Throws `NotText` for anything else
 * (an element, a formatter call): that value is not a plain message.
 */
function canonical(node, param, helpers, found) {
  if (node.type === 'Literal' && (typeof node.value === 'string' || typeof node.value === 'number')) return String(node.value);
  if (node.type === 'TemplateLiteral' && node.expressions.length === 0) return node.quasis.map(part => part.value.cooked).join('');
  if (node.type === 'BinaryExpression' && node.operator === '+') return canonical(node.left, param, helpers, found) + canonical(node.right, param, helpers, found);
  if (node.type === 'MemberExpression' && !node.computed && node.object.type === 'Identifier' && node.object.name === param && node.property.type === 'Identifier') {
    if (!found.params.includes(node.property.name)) found.params.push(node.property.name);
    return `{${node.property.name}}`;
  }
  if (node.type === 'CallExpression' && node.callee.type === 'Identifier' && helpers.has(node.callee.name) && node.arguments.length >= 2) {
    const [first, ...forms] = node.arguments;
    if (first.type === 'MemberExpression' && !first.computed && first.object.type === 'Identifier' && first.object.name === param && first.property.type === 'Identifier') {
      const name = first.property.name;
      if (!found.params.includes(name)) found.params.push(name);
      if (!found.plural.includes(name)) found.plural.push(name);
      return `{${name}, plural, ${forms.map(form => canonical(form, param, helpers, found)).join(' | ')}}`;
    }
  }
  throw new NotText(node.type);
}

/** One registered value as an inventory entry. */
function entryOf(value, helpers, group) {
  const extra = group === 'main' ? {} : { group };
  if (value.type === 'Literal' && typeof value.value === 'string') return { kind: 'text', hash: hashText(value.value), ...extra };
  if (value.type === 'TemplateLiteral' && value.expressions.length === 0) return { kind: 'text', hash: hashText(value.quasis.map(part => part.value.cooked).join('')), ...extra };
  if ((value.type === 'FunctionExpression' || value.type === 'ArrowFunctionExpression') && value.params.length <= 1 && (!value.params[0] || value.params[0].type === 'Identifier')) {
    const body = value.body.type === 'BlockStatement'
      ? (value.body.body.length === 1 && value.body.body[0].type === 'ReturnStatement' ? value.body.body[0].argument : null)
      : value.body;
    if (body) {
      const found = { params: [], plural: [] };
      try {
        const text = canonical(body, value.params[0]?.name ?? '', helpers, found);
        return { kind: 'message', params: found.params, ...(found.plural.length ? { plural: found.plural } : {}), hash: hashText(text), ...extra };
      } catch (error) {
        if (!(error instanceof NotText)) throw error;
      }
    }
    return { kind: 'rich', hash: hashText(`rich:${value.end - value.start}`), ...extra };
  }
  // `f.a123…`: a formatter compiled from number/date/currency patterns.
  if (value.type === 'MemberExpression') return { kind: 'format', hash: hashText('format'), ...extra };
  return { kind: 'rich', hash: hashText(`rich:${value.type}`), ...extra };
}

/**
 * Reads one language chunk. Returns `{ language, entries, counts }`; `group` is the inventory
 * group of its registry entries (`main`, `emoji`); sentences with element slots go to `rich`.
 */
export function extractCatalog(source, { group = 'main' } = {}) {
  const acorn = createRequire(import.meta.url)('acorn');
  const ast = acorn.parse(source, { ecmaVersion: 'latest' });
  // The module that registers: `var o = <require>(<id>)._register("<language>", …)`.
  let factory = null;
  let setter = null;
  let language = null;
  walk(ast, node => {
    if (factory) return false;
    if (node.type !== 'FunctionExpression' && node.type !== 'ArrowFunctionExpression' && node.type !== 'FunctionDeclaration') return undefined;
    if (node.body.type !== 'BlockStatement') return undefined;
    for (const statement of node.body.body) {
      if (statement.type !== 'VariableDeclaration') continue;
      for (const declaration of statement.declarations) {
        const init = declaration.init;
        if (init?.type === 'CallExpression' && init.callee.type === 'MemberExpression' && init.callee.property.type === 'Identifier'
          && init.callee.property.name === '_register' && init.arguments[0]?.type === 'Literal' && declaration.id.type === 'Identifier') {
          factory = node;
          setter = declaration.id.name;
          language = init.arguments[0].value;
        }
      }
    }
    return factory ? false : undefined;
  });
  if (!factory) throw new Error('no `_register("<language>")` call found: not a language chunk of this client');

  const statements = factory.body.body;
  // Top-level helper functions of the module (the plural helper is one of them).
  const helpers = new Set(statements.filter(statement => statement.type === 'FunctionDeclaration').map(statement => statement.id.name));
  const entries = {};
  const counts = { text: 0, message: 0, rich: 0, format: 0 };
  const add = (id, entry) => {
    if (Object.prototype.hasOwnProperty.call(entries, id)) throw new Error(`key ${id} is registered twice`);
    entries[id] = entry;
    counts[entry.kind] += 1;
  };
  for (const statement of statements) {
    if (statement.type === 'ExpressionStatement') {
      const calls = statement.expression.type === 'SequenceExpression' ? statement.expression.expressions : [statement.expression];
      for (const call of calls) {
        if (call.type !== 'CallExpression' || call.callee.type !== 'Identifier' || call.callee.name !== setter || call.arguments.length !== 2) continue;
        const [key, value] = call.arguments;
        if (key.type !== 'Literal' || typeof key.value !== 'string') continue;
        add(key.value, entryOf(value, helpers, group));
      }
    } else if (statement.type === 'ClassDeclaration') {
      // Sentences with element slots: `get <id>() { return ["piece", , "piece"] }`.
      for (const member of statement.body.body) {
        if (member.kind !== 'get' || member.key.type !== 'Identifier' || !/^[a-z][0-9a-f]{7}$/.test(member.key.name)) continue;
        const returned = member.value.body.body[0]?.argument;
        const pieces = returned?.type === 'ArrayExpression'
          ? returned.elements.map((element, index) => (element?.type === 'Literal' ? String(element.value) : `{${index}}`)).join('')
          : `rich:${member.end - member.start}`;
        // Piece n is followed by the caller's element n: one element position per piece (the one after the
        // last piece is not always used, so a pack may leave the last slot out).
        const slots = returned?.type === 'ArrayExpression' ? { slots: returned.elements.length } : {};
        add(member.key.name, { kind: 'rich', ...slots, hash: hashText(pieces), group: 'rich' });
      }
    }
  }
  return { language, entries, counts };
}

const sorted = entries => Object.fromEntries(Object.keys(entries).sort().map(id => [id, entries[id]]));

/** The inventory of the given chunks; `previous` supplies the groups that were not read now. */
export function buildInventory({ main, emoji, build, previous }) {
  const base = extractCatalog(main, { group: 'main' });
  if (base.language !== 'en') throw new Error(`--main is the "${base.language}" chunk; the inventory is built from English ("en")`);
  const entries = { ...base.entries };
  const loaded = ['main', 'rich'];
  const shared = [];
  if (emoji) {
    const names = extractCatalog(emoji, { group: 'emoji' });
    for (const [id, entry] of Object.entries(names.entries)) {
      // The same text has the same id: a few emoji names are main keys too and stay in `main`.
      if (!Object.prototype.hasOwnProperty.call(entries, id)) entries[id] = entry;
      else if (entries[id].kind !== entry.kind) throw new Error(`key ${id} is ${entries[id].kind} in the main chunk and ${entry.kind} in the emoji chunk`);
      else shared.push(id);
    }
    loaded.push('emoji');
  } else if (previous) {
    for (const [id, entry] of Object.entries(previous.entries)) if (entry.group === 'emoji' && !Object.prototype.hasOwnProperty.call(entries, id)) entries[id] = entry;
  }
  const inventory = { format: 1, site: SITE, catalog: CATALOG, base: 'en', build, entries: sorted(entries) };
  const observed = Object.fromEntries(Object.entries(inventory.entries).filter(([, entry]) => loaded.includes(entry.group ?? 'main')));
  const drift = previous ? driftReport(previous, observed, loaded) : null;
  return { inventory, drift, loaded, shared };
}

/** One entry per line: a readable diff when X ships a new build. */
export function renderInventory(inventory) {
  const { entries, ...head } = inventory;
  const lines = Object.entries(entries).map(([id, entry]) => `    ${JSON.stringify(id)}: ${JSON.stringify(entry)}`);
  return `${JSON.stringify(head, null, 2).replace(/\n}$/, '')},\n  "entries": {\n${lines.join(',\n')}\n  }\n}\n`;
}

async function readSource(where) {
  if (/^https:\/\//.test(where)) {
    const response = await fetch(where);
    if (!response.ok) throw new Error(`${where}: HTTP ${response.status}`);
    return response.text();
  }
  return readFileSync(where, 'utf8');
}

/** Asks X's webpack runtime in the running app for the English chunk names (read only, own tab). */
async function chunkUrls(port) {
  // One tab of its own through the HTTP endpoint and one websocket to that tab only.
  const endpoint = `http://127.0.0.1:${port}`;
  const own = await (await fetch(`${endpoint}/json/new?about:blank`, { method: 'PUT' })).json();
  const ws = new WebSocket(own.webSocketDebuggerUrl);
  await new Promise((resolve, reject) => { ws.onopen = resolve; ws.onerror = reject; });
  let id = 0;
  const pending = new Map();
  ws.onmessage = event => { const message = JSON.parse(event.data); pending.get(message.id)?.(message); pending.delete(message.id); };
  const send = (method, params = {}) => new Promise((resolve, reject) => {
    const n = ++id;
    const timer = setTimeout(() => { pending.delete(n); reject(new Error(`${method}: no answer in 60 s`)); }, 60000);
    pending.set(n, message => { clearTimeout(timer); if (message.error) reject(new Error(`${method}: ${message.error.message}`)); else resolve(message.result); });
    ws.send(JSON.stringify({ id: n, method, params }));
  });
  try {
    await send('Page.enable');
    await send('Page.navigate', { url: 'https://x.com/home' });
    await new Promise(resolve => setTimeout(resolve, 9000));
    const expression = `(async () => {
      const array = window.webpackChunk_twitter_responsive_web;
      if (!array) return { error: 'not the webpack client (signed out, or X moved this account to another build)' };
      let require; array.push([[Symbol('wb-strings')], {}, r => { require = r; }]);
      if (!require) return { error: 'no webpack require' };
      const scripts = [...document.scripts].map(script => script.src);
      const main = scripts.find(src => /\\/main\\.[0-9a-f]+\\.js$/.test(src));
      const text = await (await fetch(main)).text();
      const chunk = /"\\.\\/en":\\[(\\d+),\\[(?:\\d+,)*(\\d+)\\]\\]/.exec(text);
      if (!chunk) return { error: 'the language map was not found in main' };
      return { build: main.split('/').pop(), main: require.p + require.u(chunk[2]) };
    })()`;
    const result = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true, timeout: 45000 });
    const value = result.result?.value ?? { error: result.exceptionDetails?.text ?? 'evaluation failed' };
    if (value.error) throw new Error(value.error);
    // The emoji chunk id is named inside the English chunk: `a.e(<chunk>).then(…)` in its `get emoji()`.
    const english = await (await fetch(value.main)).text();
    const emoji = /get emoji\(\)\{return \w+\.e\((\d+)\)/.exec(english);
    if (emoji) {
      const name = await send('Runtime.evaluate', { expression: `(() => { let r; window.webpackChunk_twitter_responsive_web.push([[Symbol('wb-strings')], {}, x => { r = x; }]); return r.p + r.u(${emoji[1]}); })()`, returnByValue: true, timeout: 45000 });
      value.emoji = name.result?.value;
    }
    return value;
  } finally {
    ws.close();
    await fetch(`${endpoint}/json/close/${own.id}`).catch(() => {});
  }
}

async function run(args) {
  const value = flag => { const at = args.indexOf(flag); return at >= 0 ? args[at + 1] : undefined; };
  if (value('--cdp')) {
    const urls = await chunkUrls(value('--cdp'));
    console.log(`build  ${urls.build}\n--main ${urls.main}${urls.emoji ? `\n--emoji ${urls.emoji}` : ''}`);
    return 0;
  }
  const mainAt = value('--main');
  if (!mainAt) {
    console.log('Usage: node scripts/strings-inventory.mjs --main <file|url> [--emoji <file|url>] [--build <label>] [--check]\n       node scripts/strings-inventory.mjs --cdp <port>');
    return 1;
  }
  const check = args.includes('--check');
  const previous = existsSync(INVENTORY_FILE) ? JSON.parse(readFileSync(INVENTORY_FILE, 'utf8')) : null;
  const build = value('--build') ?? (/([0-9a-f]{8,})a?\.js$/.exec(mainAt)?.[1] ?? 'unknown');
  const { inventory, drift, loaded, shared } = buildInventory({ main: await readSource(mainAt), emoji: value('--emoji') ? await readSource(value('--emoji')) : null, build, previous });
  const kinds = {};
  for (const entry of Object.values(inventory.entries)) { const key = `${entry.group ?? 'main'}/${entry.kind}`; kinds[key] = (kinds[key] ?? 0) + 1; }
  console.log(`${Object.keys(inventory.entries).length} keys (build ${build}; read: ${loaded.join(', ')}): ${Object.entries(kinds).map(([key, count]) => `${key} ${count}`).join(', ')}`);
  if (shared.length) console.log(`${shared.length} emoji keys are main keys too (same id, kept in main): ${shared.slice(0, 8).join(' ')}`);
  let drifted = 0;
  if (drift) {
    drifted = drift.added.length + drift.reworded.length + drift.reshaped.length + drift.vanished.length;
    console.log(`drift against build ${previous.build ?? 'unknown'}: ${drift.added.length} new, ${drift.reworded.length} reworded, ${drift.reshaped.length} reshaped, ${drift.vanished.length} vanished`);
    for (const [name, ids] of Object.entries(drift)) if (ids.length) console.log(`  ${name}: ${ids.slice(0, 12).join(' ')}${ids.length > 12 ? ` … ${ids.length - 12} more` : ''}`);
  } else {
    console.log('no inventory yet: nothing to compare with');
  }
  if (check) return drifted ? 1 : 0;
  for (const file of [INVENTORY_FILE, PAGE_FILE]) mkdirSync(dirname(file), { recursive: true });
  writeFileSync(INVENTORY_FILE, renderInventory(inventory));
  writeFileSync(PAGE_FILE, `${JSON.stringify(packInventory(inventory))}\n`);
  console.log(`wrote ${INVENTORY_FILE.slice(root.length + 1).replaceAll('\\', '/')} and ${PAGE_FILE.slice(root.length + 1).replaceAll('\\', '/')}`);
  return 0;
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  try {
    process.exitCode = await run(process.argv.slice(2));
  } catch (error) {
    console.error(`strings-inventory: ${error instanceof Error ? error.message : error}`);
    process.exitCode = 1;
  }
}