Merge pull request #1585 from mertkayacs/fix/keyless-unicode-search

fix(geocode): preserve non-Latin names in keyless search

changelog.d/1585-keyless-unicode-search.md

@@ -0,0 +1,3 @@
+- Keep keyless place search on a nearby exact match for names written in
+  non-Latin scripts instead of discarding the match and retrying without its
+  proximity ranking (mertkayacs, #1585).

docs/CURRENT-STATE.md

@@ -5881,8 +5881,11 @@ HTTP refusals and outages remain retryable. Searches share a 12-second total
 deadline, with Photon requests capped at six seconds each. Caller/application
 cancellation stops retries and late cache writes. Replacing a location search
 cancels the previous lookup; disposing its controls cancels the active lookup. Photon uses a soft proximity bias, up to five
-candidates, and an unbiased retry for name mismatches. A "place, region" query
+candidates, and an unbiased retry for name mismatches. Name comparison folds
-prefers a candidate named exactly for the place whose city, county, state,
+Latin accents while preserving letters and marks in other scripts, so a
+non-Latin name spelled exactly, marks included, keeps its proximity ranking. A
+leading "the" counts: "The Hague" does not take a nearby "Hague". A "place, region" query prefers a
+candidate named exactly for the place whose city, county, state,
 country or country code (or USA/UK) contains every word of the region, and a
 settlement or administrative area over a POI when both fit. Nearby results that
 fail that check are retried unbiased. After the region checks of both passes, a

scripts/package-boundaries.json

@@ -572,6 +572,7 @@
       "src/search/placeSearch.js",
       "src/search/google.js",
       "src/keylessGeocoder.js",
+      "src/data/placeNames.js",
       "src/search/geospatial.js",
       "src/search/http.js",
       "src/search/defaults.js",

src/data/fixtures/README.md

@@ -1,5 +1,10 @@
 # Test fixtures
 
+- `photon-moscow-search.json` - Photon responses for `Москва`, captured
+  2026-11-05 with and without a proximity bias near the Tajikistan namesake.
+  The fixture is trimmed to the fields the keyless geocoder consumes and is
+  used only by offline search tests. Data from OpenStreetMap contributors,
+  ODbL 1.0 (https://www.openstreetmap.org/copyright).
 - `tomtom-flow-austin-12-935-1686.pbf` — one real TomTom traffic-flow vector
   tile (Mapbox Vector Tile protobuf, layer `"Traffic flow"`), downtown Austin
   z12 x935 y1686, captured 2026-07-16 from

src/data/fixtures/photon-moscow-search.json

@@ -0,0 +1,110 @@
+{
+  "biased": {
+    "features": [
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [71.8339931, 38.948412]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Тоҷикистон"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [37.6063916, 55.625578]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [37.6174782, 55.7505412]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [37.0536908, 55.7098009]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [32.163334, 56.917778]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      }
+    ]
+  },
+  "unbiased": {
+    "features": [
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [37.6063916, 55.625578]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [37.6174782, 55.7505412]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [37.0536908, 55.7098009]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [32.163334, 56.917778]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Россия"
+        }
+      },
+      {
+        "geometry": {
+          "type": "Point",
+          "coordinates": [71.8339931, 38.948412]
+        },
+        "properties": {
+          "name": "Москва",
+          "country": "Тоҷикистон"
+        }
+      }
+    ]
+  }
+}

src/data/placeNames.js

@@ -11,17 +11,16 @@ const FOLD = {
 };
 
 /**
- * Comparable form of a place name, shared by the bundled packs, the
+ * Script-aware fold of a place name: case folded, accents removed from Latin
- * OpenFreeMap street and grounds matchers and the Nominatim server: case
+ * letters only, punctuation and symbols collapsed to single spaces. Other
- * folded, accents removed from Latin letters only, punctuation and symbols
+ * scripts keep their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while
- * collapsed to single spaces, and a leading "the" dropped. Other scripts keep
+ * "São Paulo", "sao paulo" and "SÃO PAULO!" meet. Every word is kept: "The
- * their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while "São Paulo",
+ * Hague" is not "Hague".
- * "sao paulo" and "SÃO PAULO!" meet.
  *
  * @param {unknown} value
  * @returns {string}
  */
-export function foldPlaceName(value) {
+export function foldPlaceText(value) {
   return String(value ?? '')
     .toLowerCase()
     .replace(/[ßøæłđıœþ]/g, (c) => FOLD[c])
@@ -29,8 +28,19 @@ export function foldPlaceName(value) {
     .replace(/(\p{Script=Latin})\p{M}+/gu, '$1')
     .normalize('NFKC')
     .replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
-    .trim()
+    .trim();
-    .replace(/^the /, '');
+}
+
+/**
+ * Comparable form of a place name, shared by the bundled packs, the
+ * OpenFreeMap street and grounds matchers and the Nominatim server:
+ * `foldPlaceText` with a leading "the" dropped.
+ *
+ * @param {unknown} value
+ * @returns {string}
+ */
+export function foldPlaceName(value) {
+  return foldPlaceText(value).replace(/^the /, '');
 }
 
 /**

src/keylessGeocoder.js

@@ -1,3 +1,5 @@
+import { foldPlaceText } from './data/placeNames.js';
+
 /** Photon/OpenStreetMap place-search adapter. */
 const PHOTON_ENDPOINT = 'https://photon.komoot.io/api/';
 
@@ -233,19 +235,15 @@ export function photonSearchUrl(
 }
 
 /**
- * Place name reduced to comparable form: unaccented, lowercased, punctuation
+ * Place name reduced to comparable form: lowercased, Latin accents removed,
- * collapsed to single spaces. "Huế" and "Hue" have to compare equal, because a
+ * other scripts preserved, and punctuation collapsed to single spaces. "Huế"
- * user typing either means the same city.
+ * and "Hue" have to compare equal, because a user typing either means the same
+ * city.
  * @param {string} text
  * @returns {string}
  */
 export function normalizeToponym(text) {
-  return String(text ?? '')
+  return foldPlaceText(text);
-    .normalize('NFD')
-    .replace(/[\u0300-\u036f]/g, '')
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, ' ')
-    .trim();
 }
 
 /**

src/keylessGeocoder.test.mjs

@@ -35,6 +35,12 @@ import {
 import { geocodeNavigationMode } from './locations.js';
 
 const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const MOSCOW_RESPONSES = JSON.parse(
+  fs.readFileSync(
+    path.join(ROOT, 'src/data/fixtures/photon-moscow-search.json'),
+    'utf8',
+  ),
+);
 
 /** Photon's real answer for "Hanoi", trimmed to the fields we consume. */
 const HANOI = {
@@ -249,6 +255,58 @@ test('a biased near-miss is refused, and the unbiased answer is taken instead',
   assert.equal(fetchImpl.urls.length, 2, 'the refused bias costs exactly one extra request');
 });
 
+test('a non-Latin exact match keeps the nearby namesake', async () => {
+  const fetchImpl = photonPair(
+    MOSCOW_RESPONSES.biased.features,
+    MOSCOW_RESPONSES.unbiased.features,
+  );
+
+  const result = await geocodeKeyless('Москва', {
+    bias: '38.90,71.80|39.00,71.90',
+    fetchImpl,
+  });
+
+  assert.equal(result.label, 'Москва, Тоҷикистон');
+  assert.equal(fetchImpl.urls.length, 1);
+});
+
+test('toponym folding preserves non-Latin scripts', () => {
+  const cases = [
+    ['Huế', 'hue'],
+    ['Москва', 'москва'],
+    ['Αθήνα', 'αθήνα'],
+    ['القاهرة', 'القاهرة'],
+    ['東京', '東京'],
+    ['São 東京', 'sao 東京'],
+    ['Straße', 'strasse'],
+    ['Tromsø', 'tromso'],
+    ['Łódź', 'lodz'],
+    ['The Hague', 'the hague'],
+  ];
+
+  for (const [input, expected] of cases) {
+    assert.equal(normalizeToponym(input), expected, input);
+  }
+});
+
+test('a leading "the" is part of the name a nearby match must equal', async () => {
+  // "The Hague" over upstate New York means the Dutch city, not Hague, NY.
+  const hagueNy = {
+    geometry: { type: 'Point', coordinates: [-73.5004, 43.6967] },
+    properties: { osm_key: 'place', osm_value: 'town', type: 'city', name: 'Hague', country: 'United States' },
+  };
+  const theHague = {
+    geometry: { type: 'Point', coordinates: [4.3113, 52.0799] },
+    properties: { osm_key: 'place', osm_value: 'city', type: 'city', name: 'The Hague', country: 'Nederland' },
+  };
+  const fetchImpl = photonPair([hagueNy], [theHague]);
+
+  const result = await geocodeKeyless('The Hague', { bias: '43.65,-73.55|43.75,-73.45', fetchImpl });
+
+  assert.equal(Math.round(result.lat), 52, `got ${result.label}`);
+  assert.equal(fetchImpl.urls.length, 2);
+});
+
 test('a biased match that leads with the name is taken without a second request', async () => {
   // The other half of the rule, and the reason bias exists at all: "Sixth
   // Street" over Austin must stay Austin's.