From ee721335746172e29ab94791773fd44203fad130 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Mon, 12 Oct 2026 18:41:34 +0400 Subject: [PATCH] feat(i18n): find UI text that is not marked for translation npm run i18n now also lists, and fails on, markup text and title / aria-label / placeholder without data-i18n*, and string literals that reach the page (textContent, innerHTML, setAttribute, toasts, label:/title: properties and similar) without t(), tn() or msg(). Opt-outs say why where the text lives: translate="no" and data-i18n-runtime in markup, // i18n-ignore in code. src/tools/ is skipped: its text is for AI clients. 534 places are reported today; npm test does not run this part yet. Co-Authored-By: Claude Opus 5.5 --- docs/I18N.md | 33 +++-- scripts/i18n.mjs | 293 ++++++++++++++++++++++++++++++++++++-- src/i18n/catalog.test.mjs | 51 +++++++ 3 files changed, 358 insertions(+), 19 deletions(-) diff --git a/docs/I18N.md b/docs/I18N.md index 21b2697b0..d59ef8bcf 100644 --- a/docs/I18N.md +++ b/docs/I18N.md @@ -49,7 +49,7 @@ label.textContent = t(FEED_STATE_LABELS[state]); ## After you add or change text ```sh -npm run i18n # shows what each language now lacks or no longer uses +npm run i18n # what each language lacks or no longer uses, and unmarked UI text npm run i18n -- --missing ru # the missing ru entries as JSON npm run i18n -- --prune # drop entries whose English no longer exists ``` @@ -58,12 +58,29 @@ Changing the English changes the key, so the old translation goes unused and the To see a language, open the app with `?lang=ru`, or use the language switch in the Visual Presets dock. +## Text the check reports as unmarked + +`npm run i18n` also lists UI text that is not marked at all, and fails while there is any: + +- **Markup:** element text without `data-i18n`, and `title`, `aria-label` or `placeholder` without the matching `data-i18n-*`. +- **Code:** string literals that reach the page without `t()`, `tn()` or `msg()`: assigned to `textContent`, `innerText`, `innerHTML`, `title`, `placeholder`; passed to `setAttribute('title' | 'aria-label' | 'aria-valuetext' | 'placeholder' | 'alt', …)`, `createTextNode`, `insertAdjacentHTML`, `showToast`, `toast`, `setHint`, `announce`; or kept in `label:`, `title:`, `tooltip:`, `placeholder:`, `hint:`, `ariaLabel:`, `caption:`, `subtitle:`, `kicker:`, `emptyText:` properties. + +Fix it by marking the text. When the text must stay as it is, say so where it lives: + +| Case | How | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| Codes, units, product names in markup (`0.9 dB`, `ADS-B · 1090`, `OpenSky`) | `translate="no"` on the element; it covers everything inside, as in HTML | +| Markup whose text code sets through `t()` (a status that starts as `OFF`) | `data-i18n-runtime` on the element | +| A literal in code that is not UI copy (a product name, sample data) | `// i18n-ignore: ` on the line or the line above | + +Icons (`aria-hidden="true"`, Material Symbols), comments, CSS values and lowercase identifiers are skipped automatically, and so is `src/tools/`, whose text is written for AI clients rather than the page. + ## Where things live -| File | What it holds | -| ------------------------------ | ----------------------------------------------------------------------------- | -| `src/i18n/index.js` | `t`, `tn`, `msg`, the Intl formatters, markup translation, switching language | -| `src/i18n/locale.js` | shipped languages, their names, choosing the language at boot | -| `src/i18n/locales/.json` | one catalog per language, keyed by English text | -| `scripts/i18n.mjs` | `npm run i18n`: extracts messages and checks the catalogs | -| `src/i18n/catalog.test.mjs` | runs that check under `npm test` | +| File | What it holds | +| ------------------------------ | ------------------------------------------------------------------------------ | +| `src/i18n/index.js` | `t`, `tn`, `msg`, the Intl formatters, markup translation, switching language | +| `src/i18n/locale.js` | shipped languages, their names, choosing the language at boot | +| `src/i18n/locales/.json` | one catalog per language, keyed by English text | +| `scripts/i18n.mjs` | `npm run i18n`: extracts messages, checks the catalogs, finds unmarked UI text | +| `src/i18n/catalog.test.mjs` | runs that check under `npm test` | diff --git a/scripts/i18n.mjs b/scripts/i18n.mjs index b1c20bfef..773419c56 100644 --- a/scripts/i18n.mjs +++ b/scripts/i18n.mjs @@ -1,11 +1,12 @@ // Keeps the locale catalogs in step with the code. // -// npm run i18n check every catalog (also run by npm test) +// npm run i18n check every catalog and find unmarked UI text // npm run i18n -- --prune also drop entries the code no longer uses // npm run i18n -- --missing ru print what ru lacks, as JSON to translate // // Messages are the English literals passed to t(), tn() and msg() in src/, // plus the markup opted in with data-i18n* in src/ui/templates and index.html. +// It also lists UI text that is not marked at all (see findUnmarked). import { readFileSync, writeFileSync } from 'node:fs'; import { execFileSync } from 'node:child_process'; import path from 'node:path'; @@ -93,16 +94,7 @@ export function extractMessages() { messages.set(text, { plural: Boolean(plural), one: plural?.one, where }); }; - const sources = trackedFiles( - 'src/*.js', - 'src/*.mjs', - 'src/**/*.js', - 'src/**/*.mjs', - ).filter( - (file) => - !file.startsWith('src/i18n/') && - !/\.test\.mjs$|\/testSupport\//.test(file), - ); + const sources = sourceFiles(); for (const file of sources) { const source = readFileSync(path.join(ROOT, file), 'utf8'); for (const match of source.matchAll(CALL)) { @@ -140,6 +132,276 @@ export function extractMessages() { return messages; } +const hasWords = (text) => /[A-Za-z]{2,}/.test(text); +const lineAt = (source, index) => source.slice(0, index).split('\n').length; + +const VOID_TAGS = new Set( + 'area base br col embed hr img input link meta source track wbr'.split(' '), +); +const SKIP_TAGS = new Set(['script', 'style', 'svg', 'code', 'pre', 'title']); + +/** + * User-visible markup text without data-i18n*. Skipped, with everything + * inside: translate="no" (text that stays as is, as in HTML), + * data-i18n-runtime (code sets this text through t()), and icons + * (aria-hidden, Material Symbols). + */ +export function unmarkedMarkup(file, html) { + const found = []; + const source = html.replace(/|]*>/g, (c) => + c.replace(/[^\n]/g, ' '), + ); + const TOKEN = + /<(\/?)([a-zA-Z][\w-]*)((?:"[^"]*"|'[^']*'|[^>"'])*?)(\/?)>|([^<]+)/g; + const stack = []; + const skipping = () => stack.at(-1)?.skip === true; + for (const match of source.matchAll(TOKEN)) { + const [, closing, rawTag, attrs = '', selfClosing, text] = match; + if (text !== undefined) { + const top = stack.at(-1); + const value = normalize(decodeHtml(text)); + if (!skipping() && hasWords(value) && !top?.marked) + found.push({ + where: `${file}:${lineAt(source, match.index)}`, + text: value, + }); + continue; + } + const tag = rawTag.toLowerCase(); + if (closing) { + const at = stack.map((entry) => entry.tag).lastIndexOf(tag); + if (at >= 0) stack.length = at; + continue; + } + const skip = + skipping() || + SKIP_TAGS.has(tag) || + /\stranslate="no"/.test(attrs) || + /\sdata-i18n-runtime\b/.test(attrs) || + /\saria-hidden="true"/.test(attrs) || + /class="[^"]*material-symbols/.test(attrs); + if (!skip) { + for (const [, name, value] of attrs.matchAll( + /\s(title|aria-label|placeholder)="([^"]*)"/g, + )) { + if ( + hasWords(value) && + !new RegExp(String.raw`\sdata-i18n-${name}\b`).test(attrs) + ) + found.push({ + where: `${file}:${lineAt(source, match.index)}`, + text: `${name}="${decodeHtml(value)}"`, + }); + } + } + if (!VOID_TAGS.has(tag) && !selfClosing) + stack.push({ tag, skip, marked: /\sdata-i18n(\s|=|$)/.test(attrs) }); + } + return found; +} + +// Where a string becomes UI text. Each pattern ends where the value starts. +const JS_SINKS = [ + /\.(?:textContent|innerText|innerHTML|outerHTML|title|placeholder|ariaLabel)\s*=(?!=)\s*/g, + /\bsetAttribute\(\s*['"](?:title|aria-label|aria-valuetext|placeholder|alt)['"]\s*,\s*/g, + /\b(?:createTextNode|insertAdjacentText)\(\s*(?:['"][a-z]+['"]\s*,\s*)?/g, + /\binsertAdjacentHTML\(\s*['"][a-z]+['"]\s*,\s*/g, + /\b(?:showToast|toast|setHint|announce)\(\s*/g, + /(?:^|[{,\s])(?:label|title|tooltip|placeholder|hint|ariaLabel|caption|subtitle|kicker|emptyText)\s*:\s*/gm, +]; +const MARKERS = /\b(?:t|tn|msg)\(/y; + +/** The expression starting at `start`, up to its end at bracket depth 0. */ +function expressionAt(source, start) { + let depth = 0; + let i = start; + for (; i < source.length; i++) { + const c = source[i]; + if (c === "'" || c === '"' || c === '`') { + i = skipString(source, i); + continue; + } + if ('([{'.includes(c)) depth++; + else if (')]}'.includes(c)) { + if (depth === 0) break; + depth--; + } else if (depth === 0 && (c === ';' || c === ',')) break; + else if (depth === 0 && c === '\n') { + const next = source.slice(i + 1).match(/^\s*(\S\S?)/)?.[1] ?? ''; + const before = source.slice(start, i).trimEnd().at(-1) ?? ''; + if (!/^(\?|:|\+|\.|&&|\|\|)/.test(next) && !/[?:+=&|(,]/.test(before)) + break; + } + } + return source.slice(start, i); +} + +function skipString(source, i) { + const quote = source[i]; + for (let j = i + 1; j < source.length; j++) { + if (source[j] === '\\') j++; + else if (quote === '`' && source[j] === '$' && source[j + 1] === '{') { + let depth = 1; + for (j += 2; j < source.length && depth; j++) { + if (source[j] === '{') depth++; + else if (source[j] === '}') depth--; + else if ('\'"`'.includes(source[j])) j = skipString(source, j); + } + j--; + } else if (source[j] === quote) return j; + else if (quote !== '`' && source[j] === '\n') return j; + } + return source.length; +} + +/** String literals that sit in value position: not inside any call or index, + * not compared, and not already passed to t/tn/msg. */ +function valueLiterals(expression) { + const literals = []; + let callDepth = 0; + const stack = []; + for (let i = 0; i < expression.length; i++) { + const c = expression[i]; + if (c === '(' || c === '[') { + const isCall = c === '[' || /[\w$\]\)]\s*$/.test(expression.slice(0, i)); + stack.push(isCall); + if (isCall) callDepth++; + continue; + } + if (c === ')' || c === ']') { + if (stack.pop()) callDepth--; + continue; + } + MARKERS.lastIndex = i; + if (MARKERS.test(expression)) { + stack.push(true); + callDepth++; + i = MARKERS.lastIndex - 1; + continue; + } + if (c !== "'" && c !== '"' && c !== '`') continue; + const end = skipString(expression, i); + const literal = expression.slice(i, end + 1); + const before = expression.slice(0, i).trimEnd(); + const after = expression.slice(end + 1).trimStart(); + const compared = + /(===|!==|==|!=|\bcase|\bin)$/.test(before) || + /^(===|!==|==|!=|in\b)/.test(after); + if (callDepth === 0 && !compared) literals.push(literal); + i = end; + } + return literals; +} + +/** The text a literal shows: static template parts, HTML tags removed. */ +function visibleText(literal, html) { + let body = literal.slice(1, -1).replace(/\$\{(?:[^{}]|\{[^{}]*\})*\}/g, ' '); + if (html) { + const attrs = [ + ...body.matchAll(/\s(?:title|aria-label|placeholder|alt)="([^"]*)"/g), + ].map((m) => m[1]); + body = [body.replace(/<[^>]*>/g, ' '), ...attrs].join(' '); + } + const text = normalize(body); + if (!hasWords(text)) return null; + if (/[{};]/.test(text) && /:\s*[^\s]/.test(text)) return null; // CSS + if (/^(?:rgba?|hsla?)\(|\b\d+px\b/.test(text)) return null; // colours, fonts + if (/^[a-z0-9_.:/-]+$/.test(text) || /^https?:/.test(text)) return null; + return text; +} + +/** Blank out comments, keeping strings and line numbers intact. */ +function withoutComments(source) { + let out = ''; + for (let i = 0; i < source.length; i++) { + const c = source[i]; + if (c === "'" || c === '"' || c === '`') { + const end = skipString(source, i); + out += source.slice(i, end + 1); + i = end; + } else if (c === '/' && source[i + 1] === '/') { + const end = source.indexOf('\n', i); + const stop = end < 0 ? source.length : end; + out += ' '.repeat(stop - i); + i = stop - 1; + } else if (c === '/' && source[i + 1] === '*') { + const end = source.indexOf('*/', i + 2); + const stop = end < 0 ? source.length : end + 2; + out += source.slice(i, stop).replace(/[^\n]/g, ' '); + i = stop - 1; + } else out += c; + } + return out; +} + +export function unmarkedScript(file, original) { + const found = []; + const lines = original.split('\n'); + const source = withoutComments(original); + const ignored = (line) => + /i18n-ignore/.test(lines[line - 1] ?? '') || + /i18n-ignore/.test(lines[line - 2] ?? ''); + for (const sink of JS_SINKS) { + sink.lastIndex = 0; + for (const match of source.matchAll(sink)) { + const start = match.index + match[0].length; + const line = lineAt(source, start); + if (ignored(line)) continue; + const html = /HTML/.test(match[0]); + for (const literal of valueLiterals(expressionAt(source, start))) { + const text = visibleText(literal, html); + if (text) found.push({ where: `${file}:${line}`, text }); + } + } + } + return found; +} + +/** + * UI text that is not marked for translation: markup without data-i18n* and + * string literals that reach the page without t(), tn() or msg(). + * Opt out with translate="no" in markup or an `// i18n-ignore` comment on + * (or just above) the line in code. + * @returns {{where: string, text: string}[]} + */ +export function findUnmarked() { + const found = []; + for (const file of trackedFiles('index.html', 'src/ui/templates/*.html')) + found.push( + ...unmarkedMarkup(file, readFileSync(path.join(ROOT, file), 'utf8')), + ); + for (const file of sourceFiles().filter( + (name) => !NOT_UI.some((pattern) => pattern.test(name)), + )) + found.push( + ...unmarkedScript(file, readFileSync(path.join(ROOT, file), 'utf8')), + ); + const seen = new Set(); + return found.filter(({ where, text }) => { + const key = `${where}\0${text}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +// Text that never reaches the page as UI: tool results and descriptions for +// AI clients (MCP and the voice model). +const NOT_UI = [/^src\/tools\//]; + +function sourceFiles() { + return trackedFiles( + 'src/*.js', + 'src/*.mjs', + 'src/**/*.js', + 'src/**/*.mjs', + ).filter( + (file) => + !file.startsWith('src/i18n/') && + !/\.test\.mjs$|\/testSupport\//.test(file), + ); +} + const placeholders = (text) => [...String(text).matchAll(/\{([A-Za-z0-9_]+)\}/g)] .map((m) => m[1]) @@ -249,6 +511,15 @@ function main(args) { console.log(`${locale}: ${messages.size} messages, complete`); } } + const unmarked = findUnmarked(); + if (unmarked.length) { + failed = true; + console.log( + `${unmarked.length} UI strings not marked for translation (see docs/I18N.md):`, + ); + for (const { where, text } of unmarked) + console.log(` ${where} ${JSON.stringify(text)}`); + } return failed ? 1 : 0; } diff --git a/src/i18n/catalog.test.mjs b/src/i18n/catalog.test.mjs index 3d0272b24..4ac3bd0a1 100644 --- a/src/i18n/catalog.test.mjs +++ b/src/i18n/catalog.test.mjs @@ -6,6 +6,8 @@ import { extractMessages, readCatalog, shippedLocales, + unmarkedMarkup, + unmarkedScript, } from '../../scripts/i18n.mjs'; import { LOCALES } from './locale.js'; @@ -55,3 +57,52 @@ test('checkCatalog reports missing, stale, placeholder and plural problems', () assert.deepEqual(result.stale, ['Removed']); assert.equal(result.errors.length, 2); }); + +test('unmarkedMarkup finds text and attributes without data-i18n*', () => { + const texts = (html) => unmarkedMarkup('f.html', html).map((f) => f.text); + assert.deepEqual(texts('Close'), ['Close']); + assert.deepEqual(texts('Close'), []); + assert.deepEqual(texts(''), [ + 'title="Pin it"', + ]); + assert.deepEqual( + texts(''), + [], + ); + assert.deepEqual( + texts(''), + [], + ); + assert.deepEqual(texts('OFF'), []); + assert.deepEqual( + texts('public'), + [], + ); + assert.deepEqual(texts('— 42 —'), []); +}); + +test('unmarkedScript finds literals that reach the page without t/tn/msg', () => { + const texts = (js) => unmarkedScript('f.js', js).map((f) => f.text); + assert.deepEqual(texts("el.textContent = 'Saved';"), ['Saved']); + assert.deepEqual(texts("el.textContent = t('Saved');"), []); + assert.deepEqual(texts("el.textContent = on ? 'On air' : 'Off air';"), [ + 'On air', + 'Off air', + ]); + assert.deepEqual(texts("el.setAttribute('aria-label', `Open ${name}`);"), [ + 'Open', + ]); + assert.deepEqual( + texts("const row = { label: msg('Wind'), id: 'wind' };"), + [], + ); + assert.deepEqual(texts("const row = { label: 'Wind' };"), ['Wind']); + assert.deepEqual(texts("el.textContent = labelFor('flights');"), []); + assert.deepEqual(texts("el.className = 'is-active';"), []); + assert.deepEqual(texts("el.title = mode === 'Auto mode' ? a : b;"), []); + assert.deepEqual(texts("// el.textContent = 'Old copy';"), []); + assert.deepEqual( + texts("// i18n-ignore: product name\nel.textContent = 'OpenSky';"), + [], + ); +});