fix(geocode): keep a leading "the" in keyless name matching

foldPlaceName drops a leading "the", so "The Hague" over upstate New
York accepted Hague, NY as an exact nearby match. Split the script-aware
fold into foldPlaceText and use it for keyless toponyms; foldPlaceName
keeps its behaviour for the packs and street matchers.

Also: changelog fragment credits #1585, folding test pins the Latin
letters NFD does not decompose (ß, ø, ł), and CURRENT-STATE notes that
non-Latin names match only when spelled with their marks.

changelog.d/1585-keyless-unicode-search.md

@@ -1,3 +1,3 @@
 - Keep keyless place search on a nearby exact match for names written in
   non-Latin scripts instead of discarding the match and retrying without its
-  proximity ranking (Mert Kaya, @mertkayacs, #PRNUM).
+  proximity ranking (mertkayacs, #1585).

docs/CURRENT-STATE.md

@@ -5721,8 +5721,9 @@ deadline, with Photon requests capped at six seconds each. Caller/application
 cancellation stops retries and late cache writes. Replacing a location search
 cancels the previous lookup; disposing its controls cancels the active lookup. Photon uses a soft proximity bias, up to five
 candidates, and an unbiased retry for name mismatches. Name comparison folds
-Latin accents while preserving letters and marks in other scripts, so an exact
+Latin accents while preserving letters and marks in other scripts, so a
-non-Latin name keeps its proximity ranking. A "place, region" query prefers a
+non-Latin name spelled exactly, marks included, keeps its proximity ranking. A
+leading "the" counts: "The Hague" does not take a nearby "Hague". A "place, region" query prefers a
 candidate named exactly for the place whose city, county, state,
 country or country code (or USA/UK) contains every word of the region, and a
 settlement or administrative area over a POI when both fit. Nearby results that

src/data/placeNames.js

@@ -11,17 +11,16 @@ const FOLD = {
 };
 
 /**
- * Comparable form of a place name, shared by the bundled packs, the
+ * Script-aware fold of a place name: case folded, accents removed from Latin
- * OpenFreeMap street and grounds matchers and the Nominatim server: case
+ * letters only, punctuation and symbols collapsed to single spaces. Other
- * folded, accents removed from Latin letters only, punctuation and symbols
+ * scripts keep their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while
- * collapsed to single spaces, and a leading "the" dropped. Other scripts keep
+ * "São Paulo", "sao paulo" and "SÃO PAULO!" meet. Every word is kept: "The
- * their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while "São Paulo",
+ * Hague" is not "Hague".
- * "sao paulo" and "SÃO PAULO!" meet.
  *
  * @param {unknown} value
  * @returns {string}
  */
-export function foldPlaceName(value) {
+export function foldPlaceText(value) {
   return String(value ?? '')
     .toLowerCase()
     .replace(/[ßøæłđıœþ]/g, (c) => FOLD[c])
@@ -29,8 +28,19 @@ export function foldPlaceName(value) {
     .replace(/(\p{Script=Latin})\p{M}+/gu, '$1')
     .normalize('NFKC')
     .replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
-    .trim()
+    .trim();
-    .replace(/^the /, '');
+}
+
+/**
+ * Comparable form of a place name, shared by the bundled packs, the
+ * OpenFreeMap street and grounds matchers and the Nominatim server:
+ * `foldPlaceText` with a leading "the" dropped.
+ *
+ * @param {unknown} value
+ * @returns {string}
+ */
+export function foldPlaceName(value) {
+  return foldPlaceText(value).replace(/^the /, '');
 }
 
 /**

src/keylessGeocoder.js

@@ -1,4 +1,4 @@
-import { foldPlaceName } from './data/placeNames.js';
+import { foldPlaceText } from './data/placeNames.js';
 
 /** Photon/OpenStreetMap place-search adapter. */
 const PHOTON_ENDPOINT = 'https://photon.komoot.io/api/';
@@ -243,7 +243,7 @@ export function photonSearchUrl(
  * @returns {string}
  */
 export function normalizeToponym(text) {
-  return foldPlaceName(text);
+  return foldPlaceText(text);
 }
 
 /**

src/keylessGeocoder.test.mjs

@@ -229,6 +229,7 @@ const HUE = {
     extent: [107.3763, 16.7739, 108.0518, 16.1276],
   },
 };
+
 /** Answers biased requests with `nearby`, unbiased ones with `anywhere`. */
 function photonPair(nearby, anywhere) {
   const urls = [];
@@ -277,6 +278,10 @@ test('toponym folding preserves non-Latin scripts', () => {
     ['القاهرة', 'القاهرة'],
     ['東京', '東京'],
     ['São 東京', 'sao 東京'],
+    ['Straße', 'strasse'],
+    ['Tromsø', 'tromso'],
+    ['Łódź', 'lodz'],
+    ['The Hague', 'the hague'],
   ];
 
   for (const [input, expected] of cases) {
@@ -284,6 +289,24 @@ test('toponym folding preserves non-Latin scripts', () => {
   }
 });
 
+test('a leading "the" is part of the name a nearby match must equal', async () => {
+  // "The Hague" over upstate New York means the Dutch city, not Hague, NY.
+  const hagueNy = {
+    geometry: { type: 'Point', coordinates: [-73.5004, 43.6967] },
+    properties: { osm_key: 'place', osm_value: 'town', type: 'city', name: 'Hague', country: 'United States' },
+  };
+  const theHague = {
+    geometry: { type: 'Point', coordinates: [4.3113, 52.0799] },
+    properties: { osm_key: 'place', osm_value: 'city', type: 'city', name: 'The Hague', country: 'Nederland' },
+  };
+  const fetchImpl = photonPair([hagueNy], [theHague]);
+
+  const result = await geocodeKeyless('The Hague', { bias: '43.65,-73.55|43.75,-73.45', fetchImpl });
+
+  assert.equal(Math.round(result.lat), 52, `got ${result.label}`);
+  assert.equal(fetchImpl.urls.length, 2);
+});
+
 test('a biased match that leads with the name is taken without a second request', async () => {
   // The other half of the rule, and the reason bias exists at all: "Sixth
   // Street" over Austin must stay Austin's.