diff --git a/changelog.d/1819-place-privacy.md b/changelog.d/1819-place-privacy.md new file mode 100644 index 0000000000..dfb8e30112 --- /dev/null +++ b/changelog.d/1819-place-privacy.md @@ -0,0 +1,4 @@ +- Place suggestions keep more private text to the browser (a capital + after "@", keys after "=" or in quotes, letter codes after a code label, + phone numbers that looked like a postcode), and a house is suggested by + its number and street (manjunath22466, samehkhamis, #1819). diff --git a/scripts/wikidataPlaceTiles.mjs b/scripts/wikidataPlaceTiles.mjs index b874734472..15f0c89e3a 100644 --- a/scripts/wikidataPlaceTiles.mjs +++ b/scripts/wikidataPlaceTiles.mjs @@ -100,22 +100,35 @@ const kmBetween = (a, b) => * @param {Array} rows Largest first. */ export function withoutNearTwins(rows) { - // Cells of 0.05 degrees (about 5 km), searched with their neighbours. - const cell = (lat, lon) => `${Math.floor(lat * 20)}_${Math.floor(lon * 20)}`; + // Cells of 0.05 degrees (about 5 km of latitude), searched with their + // neighbours. A cell narrows toward the poles, so as many longitude cells + // are searched as TWIN_KM spans there, wrapping at the antimeridian. + const CELLS = 20; + const AROUND = 360 * CELLS; const kept = new Map(); return rows.filter((row) => { const name = foldName(row[0]); - const [la, lo] = cell(row[1], row[2]).split('_').map(Number); + const la = Math.floor(row[1] * CELLS); + const lo = Math.floor(row[2] * CELLS); + const cos = Math.cos( + (Math.min(89.9, Math.abs(row[1]) + 0.1) * Math.PI) / 180, + ); + const span = Math.min( + AROUND / 2, + Math.ceil((TWIN_KM / (111.32 * cos)) * CELLS) + 1, + ); for (let i = -1; i <= 1; i++) - for (let j = -1; j <= 1; j++) - for (const other of kept.get(`${la + i}_${lo + j}`) || []) + for (let j = -span; j <= span; j++) + for (const other of kept.get( + `${la + i}_${(((lo + j) % AROUND) + AROUND) % AROUND}`, + ) || []) if ( other.name === name && other.row[4] === row[4] && kmBetween(other.row, row) <= TWIN_KM ) return false; - const key = `${la}_${lo}`; + const key = `${la}_${((lo % AROUND) + AROUND) % AROUND}`; if (!kept.has(key)) kept.set(key, []); kept.get(key).push({ name, row }); return true; diff --git a/src/search/placeSuggestions.js b/src/search/placeSuggestions.js index e552fbd97f..6fa20b11b5 100644 --- a/src/search/placeSuggestions.js +++ b/src/search/placeSuggestions.js @@ -87,6 +87,12 @@ function foldLine(text) { .replace(/(?![0-9])\p{Nd}/gu, '0'); } +/** A US ZIP+4 after a state's code or name, as an address writes it. */ +const US_ZIP4 = new RegExp( + String.raw`(?<=(?:^|\s)(?:[a-z]{2}|[a-z]\.[a-z]|Alabama|Alaska|Arizona|Arkansas|California|Colorado|Connecticut|Delaware|Florida|Georgia|Hawaii|Idaho|Illinois|Indiana|Iowa|Kansas|Kentucky|Louisiana|Maine|Maryland|Massachusetts|Mass|Michigan|Minnesota|Mississippi|Missouri|Montana|Nebraska|Nevada|New Hampshire|New Jersey|New Mexico|New York|North Carolina|North Dakota|Ohio|Oklahoma|Oregon|Pennsylvania|Rhode Island|South Carolina|South Dakota|Tennessee|Texas|Utah|Vermont|Virginia|Washington|West Virginia|Wisconsin|Wyoming|Puerto Rico)\.?,?\s)\d{5}-\d{4}\b`, + 'giu', +); + /** * Why a line may be private: 'number' when only a long run of digits makes * it so, 'other' for anything else, or null. @@ -108,7 +114,7 @@ function privacyReason(text) { // Chandambylu"). // Also while it is typed ("john.smith@") or spaced ("john @ gmail.com"). if ( - /[^\s@]@[^\s@]|(?:^|\s)@[a-z0-9_]{2,}(?![\p{L}\p{N}])|[^\s@]@(?:\s|$)|\S\s+@\s+[\p{L}\p{N}-]{2,}\.\p{L}{2,}/u.test( + /[^\s@]@[^\s@]|(?:^|\s)@[a-z0-9_]{2,}(?![\p{L}\p{N}])|[^\s@]@(?:\s|$)|\S\s+@\s+[\p{L}\p{N}-]{2,}\.\p{L}{2,}/iu.test( line, ) ) @@ -138,8 +144,10 @@ function privacyReason(text) { ? line : line .replace( - /(^|\s)\d{1,4}[a-z]?,?\s+\d{5}(?:-\d{3})?(?=\s+\p{L}|$)/giu, - '$1 ', + // After a street word, or before the city ("12, 10115 Berlin"); + // "030 12345" alone is a phone number. + /(?<=\p{L}\.?\s)\d{1,4}[a-z]?,?\s+\d{5}(?:-\d{3})?(?=\s+\p{L}|$)|(?<=^|\s)\d{1,4}[a-z]?,?\s+\d{5}(?:-\d{3})?(?=\s+\p{L})/giu, + ' ', ) // Other postcodes after a street's house number and before the city (a // four-digit one after a comma, so "is 555 0100 thanks" stays a number): @@ -150,7 +158,10 @@ function privacyReason(text) { /(?<=\p{L}\.?\s)\d{1,4}[a-z]?(?:,\s*\d{4}(?:-\d{3})?|,?\s+(?:\d{2}-\d{3}|\d{3} \d{2}|\d{6}))(?=\s+\p{L})/giu, ' x ', ) - .replace(/(?<=\p{L}\.?,?\s)\d{5}-\d{4}\b/gu, ' x ') + // ZIP+4 after a state ("DC 20500-0003", "MA. 02139-4307", + // "California 90210-1234"), not after any word ("Alice + // 12345-6789"). + .replace(US_ZIP4, ' x ') .replace(/\b\d{5}-\d{3}\b/g, ' x ') .replace( /〒\s?\d{3}-\d{4}|\b\d{3}-\d{4}(?=\s*[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}])/gu, @@ -173,7 +184,9 @@ function privacyReason(text) { /(? token.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '')) .some( (token) => /^(?:sk[-_]|rk_|pk_live|ghp_|xox[abposr]-)/i.test(token) || @@ -201,6 +224,23 @@ function privacyReason(text) { return longNumber ? 'number' : null; } +/** + * A code after a label that can only be a code, without a colon ("PIN + * A7B2", "verification code ABCDEF", "otp 7GH2K"): holding a digit, or in + * capitals after a label that is not, and ending the line. "Gate Code Lane", + * "GATE CODE LANE", "OTP Bank" and "Auth Code ABCD Street" stay places. + */ +function codeAfterLabel(line) { + const match = + /\b((?:pin|cvv|cvc|otp|2fa|mfa|(?:door|access|gate|alarm|entry|security|verification|one-time|auth|login) code)(?:\s+is)?)\s+([a-z0-9][a-z0-9-]{3,})\s*[.!]?\s*$/i.exec( + line, + ); + if (!match) return false; + const [, label, value] = match; + if (/\d/.test(value)) return true; + return value === value.toUpperCase() && label !== label.toUpperCase(); +} + /** * A word that may be a password or key, with letters and digits mixed * within one run: eight characters or more, four letters or more with a digit ("hunter22", @@ -236,13 +276,20 @@ const CACHE_LIMIT = 200; * The spacing, the pause and the cache, shared by every suggester in the tab: * a search box that is put up again starts from where the last one left off. */ -const shared = { nextRequestAt: 0, pausedUntil: 0, cache: new Map() }; +const shared = { + nextRequestAt: 0, + pausedUntil: 0, + cache: new Map(), + /** Requests on their way, by cache key: the same text asked again waits. */ + pending: new Map(), +}; /** Forget the shared spacing, pause and cache (tests). */ export function resetPlaceSuggestions() { shared.nextRequestAt = 0; shared.pausedUntil = 0; shared.cache.clear(); + shared.pending.clear(); } /** List Photon candidates while typing. The submitted search owns navigation. */ @@ -269,91 +316,117 @@ export function createPlaceSuggester({ const key = `${endpoint ?? ''}|${limit}|${language}|${text.toLowerCase()}`; if (cache.has(key)) return cache.get(key); - // Concurrent callers recheck the gap after waking before sending. - while (shared.nextRequestAt > Date.now()) { - let timer; - const pause = new Promise((resolve) => { - timer = setTimeout(resolve, shared.nextRequestAt - Date.now()); - }); + // The same text already on its way is that request's answer, unless + // its caller cancels it: then this one asks for itself. + const inFlight = shared.pending.get(key); + if (inFlight) try { - await waitForSignal(pause, signal); - } finally { - clearTimeout(timer); + return await waitForSignal(inFlight, signal); + } catch { + signal?.throwIfAborted(); + } + const ask = async () => { + // Concurrent callers recheck the gap after waking before sending. + while (shared.nextRequestAt > Date.now()) { + let timer; + const pause = new Promise((resolve) => { + timer = setTimeout(resolve, shared.nextRequestAt - Date.now()); + }); + try { + await waitForSignal(pause, signal); + } finally { + clearTimeout(timer); + } } - } - signal?.throwIfAborted(); - if (Date.now() < shared.pausedUntil) return []; - const url = new URL(photonSearchUrl(text, { endpoint, limit })); - if (['en', 'de', 'fr', 'it'].includes(language)) - url.searchParams.set('lang', language); - - shared.nextRequestAt = Date.now() + MIN_GAP_MS; - let features; - try { - features = await requestWithDeadline( - async (requestSignal) => { - const response = await fetchImpl(url, { signal: requestSignal }); - requestSignal.throwIfAborted(); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const data = await response.json(); - return data?.features; - }, - { signal, timeoutMs: TIMEOUT_MS }, - ); - } catch (error) { signal?.throwIfAborted(); - // A refusal, a timeout or a failed request (a refusal sent without - // CORS headers arrives as one) all hold suggestions off a while. - void error; - shared.pausedUntil = Date.now() + PAUSE_AFTER_REFUSAL_MS; - return []; - } - signal?.throwIfAborted(); - if (!Array.isArray(features)) return []; + // The same text asked again while this one waited is answered now. + if (cache.has(key)) return cache.get(key); + if (Date.now() < shared.pausedUntil) return []; + const url = new URL(photonSearchUrl(text, { endpoint, limit })); + if (['en', 'de', 'fr', 'it'].includes(language)) + url.searchParams.set('lang', language); - const seen = new Set(); - const places = []; - for (const feature of features) { - const normalized = normalizePhotonFeature(feature); - if (!normalized) continue; - const { - city, - state, - country, - countrycode: countryCode, - osm_value: osmType, - type, - } = feature.properties ?? {}; - const name = normalized.name || city || state || country; - if (!name) continue; - const detail = [...new Set([city, state, country])] - .filter((part) => part && part !== name) - .join(', '); - const id = `${name}|${detail}`; - if (seen.has(id)) continue; - seen.add(id); - const { southwest, northeast } = normalized.viewport ?? {}; - places.push({ - name, - detail, - latitude: normalized.lat, - longitude: normalized.lng, - type: (osmType === 'administrative' ? type : osmType || type) || '', - state: state || null, - country: country || null, - countryCode: /^[a-z]{2}$/i.test(countryCode ?? '') - ? countryCode.toUpperCase() - : null, - extent: southwest - ? [southwest.lng, northeast.lat, northeast.lng, southwest.lat] - : null, - }); - if (places.length >= limit) break; - } + shared.nextRequestAt = Date.now() + MIN_GAP_MS; + let features; + try { + features = await requestWithDeadline( + async (requestSignal) => { + const response = await fetchImpl(url, { signal: requestSignal }); + requestSignal.throwIfAborted(); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + return data?.features; + }, + { signal, timeoutMs: TIMEOUT_MS }, + ); + } catch (error) { + signal?.throwIfAborted(); + // A refusal, a timeout or a failed request (a refusal sent without + // CORS headers arrives as one) all hold suggestions off a while. + void error; + shared.pausedUntil = Date.now() + PAUSE_AFTER_REFUSAL_MS; + return []; + } + signal?.throwIfAborted(); + if (!Array.isArray(features)) return []; - cache.set(key, places); - if (cache.size > CACHE_LIMIT) cache.delete(cache.keys().next().value); - return places; + const seen = new Set(); + const places = []; + for (const feature of features) { + const normalized = normalizePhotonFeature(feature); + if (!normalized) continue; + const { + city, + state, + country, + countrycode: countryCode, + osm_value: osmType, + type, + street, + housenumber: houseNumber, + } = feature.properties ?? {}; + // A house has no name of its own: its street and number are its + // name, so two houses on one street stay two places. + const address = + street && houseNumber ? `${houseNumber} ${street}` : ''; + const name = normalized.name || address || city || state || country; + if (!name) continue; + const detail = [...new Set([city, state, country])] + .filter((part) => part && part !== name) + .join(', '); + const id = `${name}|${detail}`; + if (seen.has(id)) continue; + seen.add(id); + const { southwest, northeast } = normalized.viewport ?? {}; + places.push({ + name, + detail, + latitude: normalized.lat, + longitude: normalized.lng, + type: (osmType === 'administrative' ? type : osmType || type) || '', + state: state || null, + country: country || null, + countryCode: /^[a-z]{2}$/i.test(countryCode ?? '') + ? countryCode.toUpperCase() + : null, + extent: southwest + ? [southwest.lng, northeast.lat, northeast.lng, southwest.lat] + : null, + }); + if (places.length >= limit) break; + } + + cache.set(key, places); + if (cache.size > CACHE_LIMIT) cache.delete(cache.keys().next().value); + return places; + }; + const asking = ask(); + shared.pending.set(key, asking); + try { + return await asking; + } finally { + if (shared.pending.get(key) === asking) shared.pending.delete(key); + } }, }; } diff --git a/src/search/placeSuggestions.test.mjs b/src/search/placeSuggestions.test.mjs index f5e2ac6f51..7063008116 100644 --- a/src/search/placeSuggestions.test.mjs +++ b/src/search/placeSuggestions.test.mjs @@ -327,3 +327,103 @@ test('a UK postcode without its space is a place', () => { for (const line of ['EC1A1BB', 'W1A0AX', 'BT487NN']) assert.equal(mayBePrivate(line), false, line); }); + +test('emails, keys and codes in other shapes stay private', () => { + for (const line of [ + 'john @Gmail.com', + 'John @EXAMPLE.COM', + '030 12345', + 'Alice 12345-6789', + 'access_token=abcdef', + 'secret_key=abcdef', + '"sk-abcdefghijklmnop"', + '(ghp_abcdef)', + 'OTP: ABCDEF', + 'verification code ABCDEF', + 'PIN: A7B2', + ]) + assert.equal(mayBePrivate(line), true, line); + for (const line of [ + '10 Downing Street SW1A2AA, London', + 'Washington DC 20500-0003', + 'Pin Oak', + 'Gate Code Lane', + ]) + assert.equal(mayBePrivate(line), false, line); +}); + +test('two houses on one street are two suggestions, asked once', async () => { + let sent = 0; + const house = (number, lon) => ({ + geometry: { type: 'Point', coordinates: [lon, 51.5034] }, + properties: { + street: 'Downing Street', + housenumber: number, + city: 'London', + country: 'United Kingdom', + countrycode: 'gb', + osm_value: 'house', + }, + }); + const service = createPlaceSuggester({ + endpoint: 'https://fixture.test/api/', + fetchImpl: async () => { + sent++; + return Response.json({ + features: [house('10', -0.1276), house('12', -0.1277)], + }); + }, + }); + const [first, second] = await Promise.all([ + service.suggest('Downing Street'), + service.suggest('Downing Street'), + ]); + assert.deepEqual( + first.map((place) => place.name), + ['10 Downing Street', '12 Downing Street'], + ); + assert.deepEqual(second, first); + assert.equal(sent, 1); +}); + +test('streets named for keys, tokens and codes, and ZIP+4 after a state, are places', () => { + for (const line of [ + 'Secret Key Lake', + 'Access Token Road', + 'Api Key Lane', + 'GATE CODE LANE', + 'SECURITY CODE PLAZA', + 'OTP Bank', + 'MFA BOSTON', + 'Auth Code ABCD Street', + 'California 90210-1234', + 'Austin Texas 78701-1234', + 'Cambridge MA. 02139-4307', + '12, 10115 Berlin', + ]) + assert.equal(mayBePrivate(line), false, line); + for (const line of [ + 'gate code: abcd', + 'PIN A7B2', + 'PIN is A7B2', + 'OTP-ABCDEF', + 'OTP – ABCDEF', + 'login code ab-12-cd', + 'auth_token=abc', + ]) + assert.equal(mayBePrivate(line), true, line); +}); + +test('the same text asked again while its request is out is sent once', async () => { + let sent = 0; + const service = createPlaceSuggester({ + endpoint: 'https://fixture.test/api/', + fetchImpl: async () => { + sent++; + await new Promise((resolve) => setTimeout(resolve, 700)); + return Response.json({ features: [] }); + }, + }); + await Promise.all([service.suggest('Paris'), service.suggest('Paris')]); + assert.equal(sent, 1); +}); diff --git a/src/tooling/wikidataPlaceTiles.test.mjs b/src/tooling/wikidataPlaceTiles.test.mjs index 007c15ed15..9e9cf54179 100644 --- a/src/tooling/wikidataPlaceTiles.test.mjs +++ b/src/tooling/wikidataPlaceTiles.test.mjs @@ -88,3 +88,18 @@ test('one place listed twice nearby keeps its larger listing', () => { ['Q1', 'Q3', 'Q4', 'Q5'], ); }); + +test('twins are found where longitude cells narrow, and across 180°', () => { + const rows = [ + // 1.98 km apart at 70° N, two longitude cells apart. + ['Nord', 70, 10.049, 9000, 'NO', 'Q1'], + ['Nord', 70, 10.101, 2000, 'NO', 'Q2'], + // Either side of the antimeridian, under 2 km apart. + ['Kai', -16.5, 179.995, 5000, 'FJ', 'Q3'], + ['Kai', -16.5, -179.995, 1000, 'FJ', 'Q4'], + ]; + assert.deepEqual( + withoutNearTwins(rows).map((row) => row[5]), + ['Q1', 'Q3'], + ); +});