From e4fda9d4e6d7f0d5c8cc8bfba82dd33d959dbc64 Mon Sep 17 00:00:00 2001 From: Mert Kaya <47145541+mertkayacs@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:39:34 +0300 Subject: [PATCH 1/2] fix(geocode): preserve non-Latin names in keyless search --- changelog.d/1585-keyless-unicode-search.md | 3 + docs/CURRENT-STATE.md | 6 +- scripts/package-boundaries.json | 1 + src/data/fixtures/README.md | 5 + src/data/fixtures/photon-moscow-search.json | 110 ++++++++++++++++++++ src/keylessGeocoder.js | 16 ++- src/keylessGeocoder.test.mjs | 37 ++++++- 7 files changed, 166 insertions(+), 12 deletions(-) create mode 100644 changelog.d/1585-keyless-unicode-search.md create mode 100644 src/data/fixtures/photon-moscow-search.json diff --git a/changelog.d/1585-keyless-unicode-search.md b/changelog.d/1585-keyless-unicode-search.md new file mode 100644 index 000000000..276ade93f --- /dev/null +++ b/changelog.d/1585-keyless-unicode-search.md @@ -0,0 +1,3 @@ +- Keep keyless place search on a nearby exact match for names written in + non-Latin scripts instead of discarding the match and retrying without its + proximity ranking (Mert Kaya, @mertkayacs, #PRNUM). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index f1244947d..45fe2999e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5720,8 +5720,10 @@ HTTP refusals and outages remain retryable. Searches share a 12-second total deadline, with Photon requests capped at six seconds each. Caller/application cancellation stops retries and late cache writes. Replacing a location search cancels the previous lookup; disposing its controls cancels the active lookup. Photon uses a soft proximity bias, up to five -candidates, and an unbiased retry for name mismatches. A "place, region" query -prefers a candidate named exactly for the place whose city, county, state, +candidates, and an unbiased retry for name mismatches. Name comparison folds +Latin accents while preserving letters and marks in other scripts, so an exact +non-Latin name keeps its proximity ranking. A "place, region" query prefers a +candidate named exactly for the place whose city, county, state, country or country code (or USA/UK) contains every word of the region, and a settlement or administrative area over a POI when both fit. Nearby results that fail that check are retried unbiased. After the region checks of both passes, a diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index d008ceaaa..72652d8da 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -552,6 +552,7 @@ "src/search/placeSearch.js", "src/search/google.js", "src/keylessGeocoder.js", + "src/data/placeNames.js", "src/search/geospatial.js", "src/search/http.js", "src/search/defaults.js", diff --git a/src/data/fixtures/README.md b/src/data/fixtures/README.md index e9dd55f50..257eaf8a5 100644 --- a/src/data/fixtures/README.md +++ b/src/data/fixtures/README.md @@ -1,5 +1,10 @@ # Test fixtures +- `photon-moscow-search.json` - Photon responses for `Москва`, captured + 2026-11-05 with and without a proximity bias near the Tajikistan namesake. + The fixture is trimmed to the fields the keyless geocoder consumes and is + used only by offline search tests. Data from OpenStreetMap contributors, + ODbL 1.0 (https://www.openstreetmap.org/copyright). - `tomtom-flow-austin-12-935-1686.pbf` — one real TomTom traffic-flow vector tile (Mapbox Vector Tile protobuf, layer `"Traffic flow"`), downtown Austin z12 x935 y1686, captured 2026-07-16 from diff --git a/src/data/fixtures/photon-moscow-search.json b/src/data/fixtures/photon-moscow-search.json new file mode 100644 index 000000000..7b1a4d25a --- /dev/null +++ b/src/data/fixtures/photon-moscow-search.json @@ -0,0 +1,110 @@ +{ + "biased": { + "features": [ + { + "geometry": { + "type": "Point", + "coordinates": [71.8339931, 38.948412] + }, + "properties": { + "name": "Москва", + "country": "Тоҷикистон" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.6063916, 55.625578] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.6174782, 55.7505412] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.0536908, 55.7098009] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [32.163334, 56.917778] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + } + ] + }, + "unbiased": { + "features": [ + { + "geometry": { + "type": "Point", + "coordinates": [37.6063916, 55.625578] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.6174782, 55.7505412] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.0536908, 55.7098009] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [32.163334, 56.917778] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [71.8339931, 38.948412] + }, + "properties": { + "name": "Москва", + "country": "Тоҷикистон" + } + } + ] + } +} diff --git a/src/keylessGeocoder.js b/src/keylessGeocoder.js index 4e01f1c74..eb881f970 100644 --- a/src/keylessGeocoder.js +++ b/src/keylessGeocoder.js @@ -1,3 +1,5 @@ +import { foldPlaceName } from './data/placeNames.js'; + /** Photon/OpenStreetMap place-search adapter. */ const PHOTON_ENDPOINT = 'https://photon.komoot.io/api/'; @@ -233,19 +235,15 @@ export function photonSearchUrl( } /** - * Place name reduced to comparable form: unaccented, lowercased, punctuation - * collapsed to single spaces. "Huế" and "Hue" have to compare equal, because a - * user typing either means the same city. + * Place name reduced to comparable form: lowercased, Latin accents removed, + * other scripts preserved, and punctuation collapsed to single spaces. "Huế" + * and "Hue" have to compare equal, because a user typing either means the same + * city. * @param {string} text * @returns {string} */ export function normalizeToponym(text) { - return String(text ?? '') - .normalize('NFD') - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() - .replace(/[^a-z0-9]+/g, ' ') - .trim(); + return foldPlaceName(text); } /** diff --git a/src/keylessGeocoder.test.mjs b/src/keylessGeocoder.test.mjs index 95098a10c..e4c762f62 100644 --- a/src/keylessGeocoder.test.mjs +++ b/src/keylessGeocoder.test.mjs @@ -35,6 +35,12 @@ import { import { geocodeNavigationMode } from './locations.js'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const MOSCOW_RESPONSES = JSON.parse( + fs.readFileSync( + path.join(ROOT, 'src/data/fixtures/photon-moscow-search.json'), + 'utf8', + ), +); /** Photon's real answer for "Hanoi", trimmed to the fields we consume. */ const HANOI = { @@ -223,7 +229,6 @@ const HUE = { extent: [107.3763, 16.7739, 108.0518, 16.1276], }, }; - /** Answers biased requests with `nearby`, unbiased ones with `anywhere`. */ function photonPair(nearby, anywhere) { const urls = []; @@ -249,6 +254,36 @@ test('a biased near-miss is refused, and the unbiased answer is taken instead', assert.equal(fetchImpl.urls.length, 2, 'the refused bias costs exactly one extra request'); }); +test('a non-Latin exact match keeps the nearby namesake', async () => { + const fetchImpl = photonPair( + MOSCOW_RESPONSES.biased.features, + MOSCOW_RESPONSES.unbiased.features, + ); + + const result = await geocodeKeyless('Москва', { + bias: '38.90,71.80|39.00,71.90', + fetchImpl, + }); + + assert.equal(result.label, 'Москва, Тоҷикистон'); + assert.equal(fetchImpl.urls.length, 1); +}); + +test('toponym folding preserves non-Latin scripts', () => { + const cases = [ + ['Huế', 'hue'], + ['Москва', 'москва'], + ['Αθήνα', 'αθήνα'], + ['القاهرة', 'القاهرة'], + ['東京', '東京'], + ['São 東京', 'sao 東京'], + ]; + + for (const [input, expected] of cases) { + assert.equal(normalizeToponym(input), expected, input); + } +}); + test('a biased match that leads with the name is taken without a second request', async () => { // The other half of the rule, and the reason bias exists at all: "Sixth // Street" over Austin must stay Austin's. From bc65c274da4418edc80df8d3b4fa3b1ed4bb42aa Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 5 Nov 2026 17:13:00 +0000 Subject: [PATCH 2/2] fix(geocode): keep a leading "the" in keyless name matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit foldPlaceName drops a leading "the", so "The Hague" over upstate New York accepted Hague, NY as an exact nearby match. Split the script-aware fold into foldPlaceText and use it for keyless toponyms; foldPlaceName keeps its behaviour for the packs and street matchers. Also: changelog fragment credits #1585, folding test pins the Latin letters NFD does not decompose (ß, ø, ł), and CURRENT-STATE notes that non-Latin names match only when spelled with their marks. --- changelog.d/1585-keyless-unicode-search.md | 2 +- docs/CURRENT-STATE.md | 5 ++-- src/data/placeNames.js | 28 +++++++++++++++------- src/keylessGeocoder.js | 4 ++-- src/keylessGeocoder.test.mjs | 23 ++++++++++++++++++ 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/changelog.d/1585-keyless-unicode-search.md b/changelog.d/1585-keyless-unicode-search.md index 276ade93f..1f8b3af41 100644 --- a/changelog.d/1585-keyless-unicode-search.md +++ b/changelog.d/1585-keyless-unicode-search.md @@ -1,3 +1,3 @@ - Keep keyless place search on a nearby exact match for names written in non-Latin scripts instead of discarding the match and retrying without its - proximity ranking (Mert Kaya, @mertkayacs, #PRNUM). + proximity ranking (mertkayacs, #1585). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 45fe2999e..d60b351cc 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5721,8 +5721,9 @@ deadline, with Photon requests capped at six seconds each. Caller/application cancellation stops retries and late cache writes. Replacing a location search cancels the previous lookup; disposing its controls cancels the active lookup. Photon uses a soft proximity bias, up to five candidates, and an unbiased retry for name mismatches. Name comparison folds -Latin accents while preserving letters and marks in other scripts, so an exact -non-Latin name keeps its proximity ranking. A "place, region" query prefers a +Latin accents while preserving letters and marks in other scripts, so a +non-Latin name spelled exactly, marks included, keeps its proximity ranking. A +leading "the" counts: "The Hague" does not take a nearby "Hague". A "place, region" query prefers a candidate named exactly for the place whose city, county, state, country or country code (or USA/UK) contains every word of the region, and a settlement or administrative area over a POI when both fit. Nearby results that diff --git a/src/data/placeNames.js b/src/data/placeNames.js index 300d09209..90b458dd3 100644 --- a/src/data/placeNames.js +++ b/src/data/placeNames.js @@ -11,17 +11,16 @@ const FOLD = { }; /** - * Comparable form of a place name, shared by the bundled packs, the - * OpenFreeMap street and grounds matchers and the Nominatim server: case - * folded, accents removed from Latin letters only, punctuation and symbols - * collapsed to single spaces, and a leading "the" dropped. Other scripts keep - * their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while "São Paulo", - * "sao paulo" and "SÃO PAULO!" meet. + * Script-aware fold of a place name: case folded, accents removed from Latin + * letters only, punctuation and symbols collapsed to single spaces. Other + * scripts keep their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while + * "São Paulo", "sao paulo" and "SÃO PAULO!" meet. Every word is kept: "The + * Hague" is not "Hague". * * @param {unknown} value * @returns {string} */ -export function foldPlaceName(value) { +export function foldPlaceText(value) { return String(value ?? '') .toLowerCase() .replace(/[ßøæłđıœþ]/g, (c) => FOLD[c]) @@ -29,8 +28,19 @@ export function foldPlaceName(value) { .replace(/(\p{Script=Latin})\p{M}+/gu, '$1') .normalize('NFKC') .replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ') - .trim() - .replace(/^the /, ''); + .trim(); +} + +/** + * Comparable form of a place name, shared by the bundled packs, the + * OpenFreeMap street and grounds matchers and the Nominatim server: + * `foldPlaceText` with a leading "the" dropped. + * + * @param {unknown} value + * @returns {string} + */ +export function foldPlaceName(value) { + return foldPlaceText(value).replace(/^the /, ''); } /** diff --git a/src/keylessGeocoder.js b/src/keylessGeocoder.js index eb881f970..3a5cd0035 100644 --- a/src/keylessGeocoder.js +++ b/src/keylessGeocoder.js @@ -1,4 +1,4 @@ -import { foldPlaceName } from './data/placeNames.js'; +import { foldPlaceText } from './data/placeNames.js'; /** Photon/OpenStreetMap place-search adapter. */ const PHOTON_ENDPOINT = 'https://photon.komoot.io/api/'; @@ -243,7 +243,7 @@ export function photonSearchUrl( * @returns {string} */ export function normalizeToponym(text) { - return foldPlaceName(text); + return foldPlaceText(text); } /** diff --git a/src/keylessGeocoder.test.mjs b/src/keylessGeocoder.test.mjs index e4c762f62..7f50ebc2e 100644 --- a/src/keylessGeocoder.test.mjs +++ b/src/keylessGeocoder.test.mjs @@ -229,6 +229,7 @@ const HUE = { extent: [107.3763, 16.7739, 108.0518, 16.1276], }, }; + /** Answers biased requests with `nearby`, unbiased ones with `anywhere`. */ function photonPair(nearby, anywhere) { const urls = []; @@ -277,6 +278,10 @@ test('toponym folding preserves non-Latin scripts', () => { ['القاهرة', 'القاهرة'], ['東京', '東京'], ['São 東京', 'sao 東京'], + ['Straße', 'strasse'], + ['Tromsø', 'tromso'], + ['Łódź', 'lodz'], + ['The Hague', 'the hague'], ]; for (const [input, expected] of cases) { @@ -284,6 +289,24 @@ test('toponym folding preserves non-Latin scripts', () => { } }); +test('a leading "the" is part of the name a nearby match must equal', async () => { + // "The Hague" over upstate New York means the Dutch city, not Hague, NY. + const hagueNy = { + geometry: { type: 'Point', coordinates: [-73.5004, 43.6967] }, + properties: { osm_key: 'place', osm_value: 'town', type: 'city', name: 'Hague', country: 'United States' }, + }; + const theHague = { + geometry: { type: 'Point', coordinates: [4.3113, 52.0799] }, + properties: { osm_key: 'place', osm_value: 'city', type: 'city', name: 'The Hague', country: 'Nederland' }, + }; + const fetchImpl = photonPair([hagueNy], [theHague]); + + const result = await geocodeKeyless('The Hague', { bias: '43.65,-73.55|43.75,-73.45', fetchImpl }); + + assert.equal(Math.round(result.lat), 52, `got ${result.label}`); + assert.equal(fetchImpl.urls.length, 2); +}); + test('a biased match that leads with the name is taken without a second request', async () => { // The other half of the rule, and the reason bias exists at all: "Sixth // Street" over Austin must stay Austin's.