diff --git a/src/main.js b/src/main.js index 545cc800fa..99de9cca06 100644 --- a/src/main.js +++ b/src/main.js @@ -1,9 +1,7 @@ import { createStandaloneApplication } from './standalone/application.js'; import { describeError } from './standalone/errors.js'; import { t } from './i18n/index.js'; - -// The Vite boundary exposes only undefined, null, or a validated URL. -const photonSetting = import.meta.env.PHOTON_URL; +import { PHOTON_SETTING as photonSetting } from './search/photonSetting.js'; const application = createStandaloneApplication({ googleApiKey: import.meta.env.GOOGLE_MAPS_API_KEY, diff --git a/src/search/photonSetting.js b/src/search/photonSetting.js new file mode 100644 index 0000000000..371e6fa9b7 --- /dev/null +++ b/src/search/photonSetting.js @@ -0,0 +1,6 @@ +/** + * The browser's Photon endpoint setting, read in one place. The Vite boundary + * (build/vite.js) exposes only undefined (use the public service), null (off, + * from an empty PHOTON_URL) or a validated https URL. + */ +export const PHOTON_SETTING = import.meta.env?.PHOTON_URL; diff --git a/src/search/placeSuggestions.js b/src/search/placeSuggestions.js new file mode 100644 index 0000000000..f00f847dab --- /dev/null +++ b/src/search/placeSuggestions.js @@ -0,0 +1,362 @@ +import { normalizePhotonFeature, photonSearchUrl } from '../keylessGeocoder.js'; +import { parseCoordinateQuery } from './coordinateParser.js'; +import { + requestWithDeadline, + waitForSignal, +} from '../sources/requestDeadline.js'; + +const MAX_QUERY_CHARS = 200; +const CJK = + /[\p{Script=Han}\p{Script=Hangul}\p{Script=Hiragana}\p{Script=Katakana}]/u; + +/** Han, Hangul or Kana text, whose place names are often two characters. */ +export const isCjk = (text) => CJK.test(String(text || '')); +/** The fewest characters a place name is looked up from. */ +export const minPlaceChars = (text) => (isCjk(text) ? 2 : 3); + +/** + * Text that may hold something private, which is never looked up, offered + * as a place or sent anywhere: + * - an "@" with something right on both sides (an email or a handle); + * - seven digits or more (in any script, full-width too) with nothing but + * spaces, punctuation, symbols or format characters between them + * (phone, card, SSN and account numbers, and their starts); + * - a phone number written with dots ("415.555.0100"); + * - two letters, two digits and ten letters or digits more (an IBAN's + * shape), spaces allowed; + * - "password", "passcode", "passwd", "ssn", "pw", "pwd" or "bearer" with + * anything after it, "token", "secret", "api key" or "key" with ":" or + * "=" after it, "api key", "access token" or "secret key" with a word of + * six characters or more after it, the word for "password" in the + * interface's languages with anything after it, or "pin" (also "pin + * code", "pin number", "pin is") with a digit after it; + * - a word starting "sk_", "sk-", "rk_", "pk_live", "ghp_" or a Slack token's "xoxb-" and the like (any + * case), or "AIza", "eyJ" or "AKIA"; + * - a word that may be a password or key (see mayBeSecret). + * The text is read after NFKC, so full-width letters and "@" count too. + * Text that parses as coordinates or a grid reference is a spot, not a + * secret. + */ +export function mayBePrivate(text) { + return privacyReason(text) !== null; +} + +/** + * Whether the only thing private-looking in a line is a long run of digits, + * as an address's house number and postcode can be ("Москва, ул. Тверская, + * д. 13, 125009"): such a line may be searched as a place when the user + * picks that on purpose. Never for an email, handle, password, key, code, + * IBAN or anything else. + * @param {string} text + */ +export function onlyNumbersLookPrivate(text) { + // Read as privacyReason reads it: "⁴¹¹¹", "①②", "card" and words split by + // zero-width spaces count as the digits and words they fold to. + const line = foldLine(text); + const groups = line.match(/\d+/g) || []; + return ( + privacyReason(text) === 'number' && + /\p{L}{2,}/u.test(line) && + // Address-sized: no group over six digits, nine digits at most in all + // (a house number and a postcode), and no word that says what a number + // is ("card 4111 …", "phone +1 415 …", "token 12345678"). + groups.every((group) => group.length <= 6) && + groups.join('').length <= 9 && + !NUMBER_CONTEXT.test(line) + ); +} + +/** + * Words that say a number is an account, a card, a phone or a secret, in the + * interface's languages where they differ ("Karte", "Telefon", "tarjeta"). + */ +const NUMBER_CONTEXT = + /(? + /^(?:sk[-_]|rk_|pk_live|ghp_|xox[abposr]-)/i.test(token) || + /^(?:AIza|eyJ|AKIA)/.test(token) || + mayBeSecret(token), + ) + ) + return 'other'; + return longNumber ? 'number' : null; +} + +/** + * A word that may be a password or key, with letters and digits mixed + * within one run: eight characters or more, four letters or more with a digit ("hunter22", + * "letmein1"), or letters, digits and a symbol together ("Tr0ub4dor&3"). + * Callsigns (UAL2110, BAW15K, N123AB) and postcodes are shorter or have + * fewer letters. + */ +function mayBeSecret(token) { + // Letters and digits must mix within one run ("hunter22", "w0rd"): a + // number beside a word ("Yanino-1", "2-chōme", "656/G.B.", "Route-66") + // is a name. + // Latin letters only: CJK addresses are written without spaces + // ("東京都港区六本木6丁目", "北京市朝阳区建国路88号"). + const mixed = token + .split(/[^\p{L}\p{N}]+/u) + .some((run) => /\p{Script=Latin}/u.test(run) && /\d/.test(run)); + if (!mixed) return false; + const letters = (token.match(/\p{L}/gu) || []).length; + const digits = (token.match(/\d/g) || []).length; + if (!letters || !digits) return false; + if (token.length >= 8) return true; + if (token.length >= 6 && /[^\p{L}\p{N}'.,\-/]/u.test(token)) return true; + return token.length >= 6 && letters >= 4; +} +const MIN_GAP_MS = 500; +const TIMEOUT_MS = 6000; +const PAUSE_AFTER_REFUSAL_MS = 60_000; +const CACHE_LIMIT = 200; + +/** List Photon candidates while typing. The submitted search owns navigation. */ +export function createPlaceSuggester({ + endpoint, + fetchImpl = (...args) => globalThis.fetch(...args), + limit = 4, +} = {}) { + const enabled = endpoint === undefined || Boolean(endpoint); + const cache = new Map(); + let nextRequestAt = 0; + let pausedUntil = 0; + + return { + enabled, + async suggest(query, { signal } = {}) { + signal?.throwIfAborted(); + const text = String(query ?? '') + .trim() + .slice(0, MAX_QUERY_CHARS); + if (!enabled || text.length < minPlaceChars(text) || mayBePrivate(text)) + return []; + const language = (globalThis.document?.documentElement?.lang || 'en') + .slice(0, 2) + .toLowerCase(); + const key = `${language}|${text.toLowerCase()}`; + if (cache.has(key)) return cache.get(key); + + // Concurrent callers recheck the gap after waking before sending. + while (nextRequestAt > Date.now()) { + let timer; + const pause = new Promise((resolve) => { + timer = setTimeout(resolve, nextRequestAt - Date.now()); + }); + try { + await waitForSignal(pause, signal); + } finally { + clearTimeout(timer); + } + } + signal?.throwIfAborted(); + if (Date.now() < pausedUntil) return []; + const url = new URL(photonSearchUrl(text, { endpoint, limit })); + if (['en', 'de', 'fr', 'it'].includes(language)) + url.searchParams.set('lang', language); + + nextRequestAt = Date.now() + MIN_GAP_MS; + const features = await requestWithDeadline( + async (requestSignal) => { + const response = await fetchImpl(url, { signal: requestSignal }); + requestSignal.throwIfAborted(); + if (!response.ok) { + pausedUntil = Date.now() + PAUSE_AFTER_REFUSAL_MS; + return null; + } + const data = await response.json(); + return data?.features; + }, + { signal, timeoutMs: TIMEOUT_MS }, + ); + signal?.throwIfAborted(); + if (!Array.isArray(features)) return []; + + const seen = new Set(); + const places = []; + for (const feature of features) { + const normalized = normalizePhotonFeature(feature); + if (!normalized) continue; + const { + city, + state, + country, + countrycode: countryCode, + osm_value: osmType, + type, + } = feature.properties ?? {}; + const name = normalized.name || city || state || country; + if (!name) continue; + const detail = [...new Set([city, state, country])] + .filter((part) => part && part !== name) + .join(', '); + const id = `${name}|${detail}`; + if (seen.has(id)) continue; + seen.add(id); + const { southwest, northeast } = normalized.viewport ?? {}; + places.push({ + name, + detail, + latitude: normalized.lat, + longitude: normalized.lng, + type: (osmType === 'administrative' ? type : osmType || type) || '', + state: state || null, + country: country || null, + countryCode: /^[a-z]{2}$/i.test(countryCode ?? '') + ? countryCode.toUpperCase() + : null, + extent: southwest + ? [southwest.lng, northeast.lat, northeast.lng, southwest.lat] + : null, + }); + if (places.length >= limit) break; + } + + cache.set(key, places); + if (cache.size > CACHE_LIMIT) cache.delete(cache.keys().next().value); + return places; + }, + }; +} + +/** + * The cities that lie in a region: inside its extent (`[west, north, east, + * south]`, as Photon gives it) and, when the region names a country by its + * ISO code, in that country. Without either there is nothing to tell a + * region's cities from its neighbours', so none are listed. + * @param {Array<{latitude: number, longitude: number, country?: string}>} cities + * Bundled cities, `country` an ISO 3166-1 code. + * @param {{extent?: ?Array, countryCode?: ?string}} region + */ +export function citiesInRegion(cities, { extent, countryCode } = {}) { + const box = Array.isArray(extent) && extent.length === 4 ? extent : null; + if (!box && !countryCode) return []; + return cities.filter( + (city) => + (!countryCode || city.country === countryCode) && + (!box || inExtent(box, city)), + ); +} + +/** + * Whether a place lies in an extent `[west, north, east, south]`. An extent + * across the antimeridian (west east of east) runs from west round to east. + * @param {Array} extent + * @param {{latitude: number, longitude: number}} place + */ +export function inExtent([west, north, east, south], place) { + const lon = place.longitude; + const inLongitude = + west <= east ? lon >= west && lon <= east : lon >= west || lon <= east; + return place.latitude <= north && place.latitude >= south && inLongitude; +} diff --git a/src/search/placeSuggestions.test.mjs b/src/search/placeSuggestions.test.mjs new file mode 100644 index 0000000000..a549202f2c --- /dev/null +++ b/src/search/placeSuggestions.test.mjs @@ -0,0 +1,280 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + citiesInRegion, + createPlaceSuggester, + inExtent, + mayBePrivate, +} from './placeSuggestions.js'; + +test('Photon suggestions keep distinct valid candidates and their region bounds', async () => { + const texas = { + geometry: { type: 'Point', coordinates: [-99, 31] }, + properties: { + name: 'Texas', + country: 'United States', + countrycode: 'us', + type: 'state', + osm_value: 'administrative', + extent: [-107, 37, -93, 25], + }, + }; + const requested = []; + const service = createPlaceSuggester({ + endpoint: 'https://fixture.test/api/', + fetchImpl: async (url) => { + requested.push(url); + return Response.json({ + features: [ + texas, + texas, + { + ...texas, + geometry: { type: 'Point', coordinates: [-99, 131] }, + }, + ], + }); + }, + }); + const expected = [ + { + name: 'Texas', + detail: 'United States', + type: 'state', + latitude: 31, + longitude: -99, + state: null, + country: 'United States', + countryCode: 'US', + extent: [-107, 37, -93, 25], + }, + ]; + assert.deepEqual(await service.suggest(' Texas '), expected); + assert.deepEqual(await service.suggest('texas'), expected); + assert.deepEqual(await service.suggest('Tx'), []); + assert.equal(requested.length, 1); + assert.equal(requested[0].origin, 'https://fixture.test'); + assert.equal(requested[0].searchParams.get('q'), 'Texas'); +}); + +test('a disabled or cancelled suggestion sends no request', async () => { + const sent = []; + const fetchImpl = async (url) => { + sent.push(url.searchParams.get('q')); + return Response.json({ features: [] }); + }; + const off = createPlaceSuggester({ endpoint: null, fetchImpl }); + assert.equal(off.enabled, false); + assert.deepEqual(await off.suggest('Texas'), []); + + const service = createPlaceSuggester({ fetchImpl }); + await service.suggest('Texas'); + const abort = new AbortController(); + const waiting = service.suggest('Austin', { signal: abort.signal }); + abort.abort(); + await assert.rejects(waiting, { name: 'AbortError' }); + assert.deepEqual(sent, ['Texas']); +}); + +test('suggestions while typing keep their spacing to the place service', async () => { + const sentAt = []; + const service = createPlaceSuggester({ + endpoint: 'https://fixture.test/api/', + fetchImpl: async () => { + sentAt.push(performance.now()); + return Response.json({ features: [] }); + }, + }); + // Three different texts at once, as fast typing past the debounce sends. + await Promise.all(['Aus', 'Aust', 'Austin'].map((q) => service.suggest(q))); + assert.equal(sentAt.length, 3); + for (let i = 1; i < sentAt.length; i++) + assert.ok( + sentAt[i] - sentAt[i - 1] >= 490, + `request ${i} came ${Math.round(sentAt[i] - sentAt[i - 1])} ms after the one before`, + ); +}); + +test('a state lists only its own cities: Texas has no Mexican one', () => { + const texas = { extent: [-106.65, 36.5, -93.51, 25.84], countryCode: 'US' }; + const cities = [ + { name: 'Houston', latitude: 29.76, longitude: -95.37, country: 'US' }, + { name: 'El Paso', latitude: 31.76, longitude: -106.49, country: 'US' }, + // Inside the Texas box, across the border. + { + name: 'Ciudad Juárez', + latitude: 31.74, + longitude: -106.49, + country: 'MX', + }, + { name: 'Reynosa', latitude: 26.09, longitude: -98.28, country: 'MX' }, + // In the US, outside Texas. + { name: 'Denver', latitude: 39.74, longitude: -104.99, country: 'US' }, + ]; + assert.deepEqual( + citiesInRegion(cities, texas).map((city) => city.name), + ['Houston', 'El Paso'], + ); + // A country without bounds still keeps only its own. + assert.deepEqual( + citiesInRegion(cities, { countryCode: 'MX' }).map((city) => city.name), + ['Ciudad Juárez', 'Reynosa'], + ); + // Fiji's extent runs across the antimeridian. + assert.deepEqual( + citiesInRegion( + [ + { name: 'Labasa', latitude: -16.43, longitude: 179.38, country: 'FJ' }, + { name: 'Lakeba', latitude: -18.2, longitude: -178.8, country: 'FJ' }, + ], + { extent: [177, -12, -178, -21], countryCode: 'FJ' }, + ).length, + 2, + ); + assert.deepEqual(citiesInRegion(cities, {}), []); +}); + +test('two-character Han, Hangul and Kana names are looked up', async () => { + const sent = []; + const service = createPlaceSuggester({ + endpoint: 'https://fixture.test/api/', + fetchImpl: async (url) => { + sent.push(url.searchParams.get('q')); + return Response.json({ features: [] }); + }, + }); + await service.suggest('東'); + await service.suggest('NY'); + await service.suggest('東京'); + assert.deepEqual(sent, ['東京']); +}); + +test('nothing that may be private is sent to the place service', async () => { + const sent = []; + const service = createPlaceSuggester({ + endpoint: 'https://fixture.test/api/', + fetchImpl: async (url) => { + sent.push(url.searchParams.get('q')); + return Response.json({ features: [] }); + }, + }); + for (const text of [ + 'me@example.com', + 'sk-abc123', + 'AKIAIOSFODNN7EXAMPLE', + 'ghp_0123abcd', + 'xoxb-1234', + 'token a1b2c3d4e5f6g7', + ]) + assert.deepEqual(await service.suggest(text), [], text); + assert.equal(mayBePrivate('Saint Petersburg 2'), false); + assert.equal(mayBePrivate('Bahnhofstrasse'), false); + assert.deepEqual(sent, []); +}); + +test('an extent across the antimeridian holds both of its sides', () => { + const fiji = [177, -12, -178, -21]; + assert.equal(inExtent(fiji, { latitude: -16.4, longitude: 179.4 }), true); + assert.equal(inExtent(fiji, { latitude: -18.2, longitude: -178.8 }), true); + assert.equal(inExtent(fiji, { latitude: -18.2, longitude: 170 }), false); + assert.equal( + inExtent([-107, 37, -93, 25], { latitude: 31, longitude: -99 }), + true, + ); +}); + +test('cards, phones, IBANs, keys and passwords are private', () => { + for (const text of [ + '4111 1111 1111 1111', + '+1 (415) 555-0100', + 'SSN 123-45-6789', + 'GB29 NWBK 6016 1331 9268 19', + 'sk_live_51Habc', + 'rk_live_x', + 'pk_live_abc', + 'AIzaSyA0', + 'eyJhbGciOi', + 'password hunter2', + // Digits of other scripts that would read as a spot once folded, and + // password words split by format characters or curly apostrophes. + '١٢٣٤, ٥٦٧٨', + '१२३४ ५६७८', + 'parola d’ordine viola', + 'pass\u200bword: violet', + 'mot\u200b de passe violet', + 'me\u200b@example.com', + '@elonmusk', + 'john.smith@', + 'my number is 555 0100 thanks', + 'john @ gmail.com', + 'door code 4421', + 'otp 482913', + 'verification code 482913', + '2fa 482913', + 'card ending 4242', + 'my secret is swordfish', + 'passphrase correct horse battery', + 'mnemonic: abandon ability able', + 'john at example dot com', + 'the access code is 8842', + 'ping @jdoe', + 'pass: correcthorse', + 'pin 1234', + // Full-width forms and digits of any script, and digits joined by + // dots, underscores or commas. + '4111 1111 1111 1111', + 'john@example.com', + 'sk_live_abc', + 'my password is hunter', + '123-45-6789', + '٤١١١ ١١١١ ١١١١ ١١١١', + '४१११ ११११ ११११ ११११', + '4111.1111.1111.1111', + '4111_1111_1111_1111', + '4111,1111,1111,1111', + '123.45.6789', + 'what is 4111.1111.1111.1111', + ]) + assert.equal(mayBePrivate(text), true, text); + for (const text of [ + 'Pin Oak, Texas', + // An address with a spaced "@", and names the old rules caught. + 'flat 2 @ 10 Downing St', + 'Nanjampeta @ Chandambylu', + 'Sachsenhausen', + 'Eisenhammer', + 'Xoxocotla', + 'Santa Cruz Xoxocotlán', + 'Yanino-1', + 'Meguro Honmachi 2-chome', + 'Chak No. 656/G.B.', + 'Route-66', + // Street addresses: a house number beside a postcode, CJK addresses + // written without spaces, and postcodes as they are written. + '北京市朝阳区建国路88号', + '東京都港区六本木6丁目', + '台北市信義區市府路45號', + '渋谷区道玄坂2-1', + '〒100-0001 東京都千代田区千代田1-1', + 'Washington DC 20500-0003', + '1600 Pennsylvania Ave NW, Washington, DC 20500-0003', + 'São Paulo 01310-200', + 'Calle 100 # 19-61 Bogota', + 'ul. Nowy Świat 10, 00-950 Warszawa', + 'Drottninggatan 53, 111 21 Stockholm', + 'Mariahilfer Straße 120, 1070 Wien', + 'Rua Augusta 274, 1100-053 Lisboa', + 'Tverskaya 13 125009 Moscow', + 'Hauptstrasse 12 10115 Berlin', + 'Calle Mayor 10, 28013 Madrid', + 'Av Paulista 1578 01310-200', + 'Eyjafjallajökull', + 'Route 66', + '30.27, -97.74', + '30.2672, -97.7431', + '14R PU 2090 4906', + 'New York 10001', + 'Passau', + ]) + assert.equal(mayBePrivate(text), false, text); +});