diff --git a/changelog.d/1274-import-issue-fix.md b/changelog.d/1274-import-issue-fix.md new file mode 100644 index 000000000..99bbc1429 --- /dev/null +++ b/changelog.d/1274-import-issue-fix.md @@ -0,0 +1 @@ +- Scan the kml root delimiter in bounded time (asturksever, #1274). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9adb71f3c..b10f43f3b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -9,7 +9,16 @@ broader suffix variants. Follow-ups retain boundary provenance, and cancelled lookups preserve the previous analyst cohort and boundary source. Navigation search keeps a successful geocode, with no search or view-based region recovery. -The standalone military source no longer requests the unavailable, undocumented ADSB.lol trace endpoint. Tracking retains its locally accumulated trail, and custom sources may still supply optional getTrack history. The old HTTP route validates the identifier then reports 503 history_unavailable without any upstream request, rather than forwarding a misleading 403. The `get_aircraft_track` tool reports `unsupported` for a military feed without `getTrack` instead of a retry-later failure. This does not restore historical coverage or download bulk daily archives. +KML namespace repair locates the opening token once and scans its quoted tag +delimiter once, skipping prolog and DOCTYPE comments and processing +instructions whole. The scanner is a boundary check only: an opening KML tag +that never closes, or one with raw markup starting inside it, fails before DOM +parsing, so a file of repeated incomplete `', end + 4); + if (skip < 0) return text; + end = skip + 2; + } else if (char === '<' && text.startsWith('', end + 2); + if (skip < 0) return text; + end = skip + 1; + } else if (char === '[') subsetDepth++; + else if (char === ']') subsetDepth--; + else if (char === '>' && subsetDepth === 0) break; + } + if (end === text.length) return text; + start = end + 1; + continue; + } + const close = text.startsWith('' + : text.startsWith('' + : null; + if (!close) break; + const end = text.indexOf(close, start + 2); + if (end < 0) return text; // The XML parser reports an unterminated preamble. + start = end + close.length; + } + const match = /^') { + end = index + 1; + break; + } + } + if (end < 0) + throw new GeoImportError( + `That ${what} file is not valid XML.`, + msg('That {format} file is not valid XML.'), + { format: what }, + ); + // Walk the root's attributes in order so quoted values are never read as + // declarations. Anything the walk cannot tokenize is kept verbatim for the + // XML parser to judge. + const seen = new Set(); + const attribute = /(\s+)([^\s=/>"']+)\s*=\s*("[^"]*"|'[^']*')/y; + let position = start + match[0].length; + let root = text.slice(start, position); + for (;;) { + attribute.lastIndex = position; + const found = attribute.exec(text); + if (!found || attribute.lastIndex > end) break; + position = attribute.lastIndex; + const prefix = /^xmlns:([\w.-]+)$/.exec(found[2])?.[1]; + if (prefix && seen.has(prefix)) continue; + if (prefix) seen.add(prefix); + root += found[0]; + } + root += text.slice(position, end); + if (!seen.has('xsi') && /[<\s]xsi:/.test(text)) + root = root.replace( + /(\/?)>$/, + ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"$1>', + ); + return text.slice(0, start) + root + text.slice(end); +} + function parseXml(text, what) { // Match the namespace repairs Cesium applies to ordinary exported KML. - if (what !== 'GPX') - text = String(text).replace(/]*>/i, (root) => { - const seen = new Set(); - root = root.replace( - /\s+xmlns:([\w.-]+)\s*=\s*(["'])[^"']*\2/g, - (declaration, prefix) => { - if (seen.has(prefix)) return ''; - seen.add(prefix); - return declaration; - }, - ); - if (!seen.has('xsi') && /[<\s]xsi:/.test(text)) - root = root.replace( - />$/, - ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">', - ); - return root; - }); + if (what !== 'GPX') text = repairKmlRootNamespaces(text, what); const doc = new DOMParser().parseFromString( String(text).replace(/^/, ''), 'application/xml', diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs index af02ec222..9b147e1d2 100644 --- a/src/data/geoFileImport.test.mjs +++ b/src/data/geoFileImport.test.mjs @@ -1,9 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { deflateRawSync } from 'node:zlib'; +import { spawnSync } from 'node:child_process'; import * as Cesium from 'cesium'; import { GeoImportError, + repairKmlRootNamespaces, loadGeoFile, MAX_IMPORT_POSITIONS, MAX_IMPORT_ENTITIES, @@ -1171,3 +1173,92 @@ test('a dated import exposes no clock and draws for all time', () => { assert.equal(recorded.getValue(Cesium.JulianDate.now()).length, 2); assert.ok(track.polyline.clampToGround.getValue()); }); + +test('KML root namespace repair respects quoted delimiters and preserves the document tail', () => { + const text = + ''; + const result = repairKmlRootNamespaces(text); + assert.match(result, /note="a > b"/); + assert.match(result, /xmlns:x="first"/); + assert.doesNotMatch(result, /second/); + assert.match( + result, + /xmlns:xsi="http:\/\/www.w3.org\/2001\/XMLSchema-instance"/, + ); + assert.ok(result.endsWith('')); + assert.match( + repairKmlRootNamespaces(''), + /XMLSchema-instance"\/>$/, + ); + for (const malformed of ['")}.repeat(100000) + ']>'; + if (repairKmlRootNamespaces(wide).split('xmlns:gx="a"').length !== 2) process.exit(1);`; + const child = spawnSync( + process.execPath, + ['--input-type=module', '-e', source], + { timeout: 5000, encoding: 'utf8' }, + ); + assert.ifError(child.error); + assert.equal(child.status, 0, child.stderr); +}); + +test('KML root repair ignores incomplete tag text inside preamble comments', () => { + const input = + '\n'; + assert.equal( + repairKmlRootNamespaces(input), + input.replace(' xmlns:gx="b"', ''), + ); +}); + +test('KML namespace repair follows a doctype with a quoted root-like entity', () => { + const input = + '">]>'; + assert.equal( + repairKmlRootNamespaces(input), + input.replace(' xmlns:gx="b"', ''), + ); +}); + +test('KML namespace repair skips doctype comments and processing instructions whole', () => { + for (const prolog of [ + "]>", + ' here -->]>', + " fine?>]>", + '">]>', + ]) { + const input = `${prolog}`; + assert.equal( + repairKmlRootNamespaces(input), + input.replace( + ' xmlns:gx="b">', + ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">', + ), + prolog, + ); + } + const unterminated = ''; + assert.equal(repairKmlRootNamespaces(unterminated), unterminated); +}); + +test('KML namespace repair ignores declaration text inside quoted attribute values', () => { + const input = + ' b xmlns:gx="fake"\' xmlns:gx="http://www.google.com/kml/ext/2.2" xmlns:gx="dup">'; + const result = repairKmlRootNamespaces(input); + assert.equal(result, input.replace(' xmlns:gx="dup"', '')); + assert.match(result, /\sxmlns:gx="http:\/\/www.google.com\/kml\/ext\/2.2"/); + const xsi = ''; + assert.match( + repairKmlRootNamespaces(xsi), + /note="xmlns:xsi='x'" xmlns:xsi="http:\/\/www.w3.org\/2001\/XMLSchema-instance">/, + ); +});