From c06aa37ce2c829aea4e3efe9f688a4fb74cdae46 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 12:05:16 -0800 Subject: [PATCH] Keep the place suggester's fair-use and privacy promises - One spacing, pause and cache for the whole tab, so a search box put up again does not start afresh. - A failed request or a timeout pauses suggestions for a minute, as a refusal does. - A number named as a card, a phone or an account keeps its digits together, so no address is read out of them; a UK postcode written without its space is a place, not a password. - The docs say exactly what is sent. Co-authored-by: Manjunath Paparaju --- DATA_SOURCES.md | 2 +- changelog.d/1814-place-suggestions.md | 10 +-- scripts/format-scope.json | 3 +- src/search/placeSuggestions.js | 109 ++++++++++++++++---------- src/search/placeSuggestions.test.mjs | 49 ++++++++++++ 5 files changed, 126 insertions(+), 47 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 441552dd89..ce24d709cd 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -287,7 +287,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Acartia.** Keyless. `/api/acartia/sightings` reads the current 7-day marine-mammal feed for ALL, MAMMALS and WHALES views intersecting 45.5–50.5° N, 126–121° W in the Salish Sea. WHALES accepts recognized cetaceans; other identified marine mammals appear under ALL/MAMMALS. The server rounds positions to 0.01° and drops comments before caching projected records (10 min fresh, stale ≤24 h); raw responses are not retained. Selected text cards credit the reporting provider under CC BY 4.0 and show Be Whale Wise guidance. `/sightings/current` answers without a key today, but Acartia's README asks API users to create an account and generate an API key on the [Acartia dashboard](https://acartia.io/) and to follow its [community guidelines](https://acartia.io/register). Attribution also appears in the layer row and Data attribution popover. Opportunistic sightings, not a census. - **Mapped installations.** OpenFreeMap military polygons use a bundled Overture/OSM name index; wide views show named points. Optional operator-configured Overpass and explicit Google Places searches keep their own source paths. Mapped data does not establish current activity or complete coverage. - **Place search.** Named-place fly-to uses Google Geocoding when a Maps key is configured, then Photon (`PHOTON_URL`; unset uses the public instance, empty disables it), then `/api/geocode` (Nominatim search) when neither answers. `/api/geocode` and the voice outline route `/api/geocode/outline` share one Nominatim queue (`NOMINATIM_URL`; unset, empty and invalid values disable it; an operator must explicitly select an authorized endpoint): one request at a time at least 1.1 s apart, an identifying `User-Agent` and `Referer`, a short queue that refuses a burst, a pause after 429/403/5xx answers (honouring `Retry-After` in full), at most one retry after a network failure, and a per-install daily cap on public-instance requests (`NOMINATIM_DAILY_CAP`, default 50). Processes using the same installation state file hold one lock through the actual fetch invocation and recording the next public start, with the daily slot persisted before invocation; restarts share that count, spacing and any pause. An unfinished dispatch marker fails closed until operator recovery; it is never automatically reclaimed. Public access fails closed if the state file cannot be read, locked, written or semantically validated. This coordination does not span independent installations or state directories and does not establish an application-wide public-service allowance or authorize public use. An operator-selected non-public endpoint is outside the public cap, spacing and durable-state requirement; an upstream cooldown is kept with the endpoint that returned it rather than leaking across operator URLs. Answers are cached on disk (outlines 90 days, places 30 days, no-results 24 hours); failures are not cached. Outlines are looked up only for an explicit ask, after bundled data and any configured Overpass, and only results whose type suits the ask are used. Administrative outline responses expose country/admin-1/admin-2 identity only when the selected Nominatim row has a recognized, unambiguous address type consistent with its own name; numeric rank, ambiguous types, missing fields and contradictory names do not manufacture identity. It is not used for systematic or bulk queries. -- **Place suggestions.** The search box's suggester (`src/search/placeSuggestions.js`) asks Photon for a few candidates while a place is typed: at most one request every 500 ms per tab, a six-second limit, none for a minute after a refusal, and text capped at 200 characters. Text that may be private (an email address, seven or more digits however separated, an IBAN shape, a password, PIN, token or key) is never sent. An empty `PHOTON_URL` turns suggestions off along with the place search. Answers are cached in the tab. +- **Place suggestions.** The search box's suggester (`src/search/placeSuggestions.js`) asks Photon for a few candidates while a place is typed: at most one request every 500 ms per tab (shared by every search box), a six-second limit for the whole answer, none for a minute after a refusal, a failed request or a timeout, and text capped at 200 characters. Text that may be private is never sent: an email address, seven or more digits however separated (a street address's house number and postcode excepted, unless a word such as "card" or "phone" names the number), an IBAN shape, a password, PIN, token or key. An empty `PHOTON_URL` turns suggestions off along with the place search. Answers are cached in the tab. - **Street and building outlines.** Voice street highlights, pointed-at buildings and grounds use the OpenFreeMap vector tiles the traffic layer already loads. Traffic and outlines share one bounded pending fetch/decode owner per immutable XYZ tile; materialized outline geometry is charged to the retained-byte ceiling and capped by coordinate count. Consumer result caches and request lifetimes remain independent, while eviction or explicit teardown cancels queued idle projections and releases their decoded owner. No Overpass request is made unless `OVERPASS_UPSTREAMS` is configured. Named grounds require a returned name matching the selected place, except that an ask whose head noun names a kind of grounds (a school, university or college, hospital, cemetery, stadium or zoo) may use the part of the smallest unnamed tile area of that class that holds the resolved place; a differently named area never answers. Other anonymous landcover polygons cannot establish identity, so unsupported grounds stay unavailable rather than borrowing a nearby POI name. - **Voice areas and place search.** Voice areas (`resolve_area`) use the same outline sources as voice outlines. Voice place search (`osm_query`, `/api/osm/features`) runs only on an operator-configured Overpass: a fixed list of kinds, a bounded box and result count, one query at a time, answers cached for a day. It never uses Nominatim. - **Cockpit regional briefing.** `/api/regional-brief` rounds aircraft coordinates into 0.1° cache cells, caches results for five minutes, and resolves physical regions from bundled public-domain Natural Earth polygons without reverse-geocoding requests. Google News RSS is queried with the resolved locality/region first; GDELT is used only when that RSS query fails or is empty. Google's published Google News terms restrict that source to personal, noncommercial use, so commercial deployments must disable/replace it or obtain separate permission; GDELT permits commercial dataset use with citation. The Data attribution popover identifies the active headline sources; article links retain publisher attribution. Headlines are location-query matches, not verified incidents, risk rankings, or evidence that a location is safe. Empty, partial, stale, and unavailable source states remain distinct. Open-Meteo supplies current conditions independently of the news source. `WX OFF` disables cockpit weather rendering only; the Local Info briefing still fetches its source-backed weather values and displays the required linked Open-Meteo credit. diff --git a/changelog.d/1814-place-suggestions.md b/changelog.d/1814-place-suggestions.md index 48f67fc721..1c02bcb55c 100644 --- a/changelog.d/1814-place-suggestions.md +++ b/changelog.d/1814-place-suggestions.md @@ -1,6 +1,6 @@ - The place suggester asks Photon for candidates while a place is typed: at - most one request every 500 ms per tab, none for a minute after a refusal, - and never text that may be private (an email address, long numbers, an - IBAN shape, a password, PIN, token or key). An empty `PHOTON_URL` turns it - off along with the place search. The search box that uses it follows - (manjunath22466, samehkhamis, #1814). + most one request every 500 ms per tab, none for a minute after a refusal + or a failed request, and never text that may be private (an email address, + long numbers, an IBAN shape, a password, PIN, token or key). An empty + `PHOTON_URL` turns it off along with the place search. The search box that + uses it follows (manjunath22466, samehkhamis, #1814). diff --git a/scripts/format-scope.json b/scripts/format-scope.json index aa51f28c05..2472b69114 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -1583,5 +1583,6 @@ "src/ui/layerFinder.test.mjs", "src/data/wikidataCities.js", "src/data/wikidataCities.test.mjs", - "scripts/build-wikidata-cities.mjs" + "scripts/build-wikidata-cities.mjs", + "src/search/placeSuggestions.test.mjs" ] diff --git a/src/search/placeSuggestions.js b/src/search/placeSuggestions.js index f00f847dab..e552fbd97f 100644 --- a/src/search/placeSuggestions.js +++ b/src/search/placeSuggestions.js @@ -130,25 +130,34 @@ function privacyReason(text) { // ("DC 20500-0003"), a Brazilian CEP ("01310-200") and a Japanese one // after "〒" or before the address ("〒100-0001", "100-0001 東京都"). // "#" (a street number's sign, "Calle 100 # 19-61") splits digits too. - const digits = line - .replace(/(^|\s)\d{1,4}[a-z]?,?\s+\d{5}(?:-\d{3})?(?=\s+\p{L}|$)/giu, '$1 ') - // Other postcodes after a street's house number and before the city (a - // four-digit one after a comma, so "is 555 0100 thanks" stays a number): - // "Nowy Świat 10, 00-950 Warszawa", "Drottninggatan 53, 111 21 - // Stockholm", "Mariahilfer Straße 120, 1070 Wien", "Rua Augusta 274, - // 1100-053 Lisboa", "Tverskaya 13 125009 Moscow". - .replace( - /(?<=\p{L}\.?\s)\d{1,4}[a-z]?(?:,\s*\d{4}(?:-\d{3})?|,?\s+(?:\d{2}-\d{3}|\d{3} \d{2}|\d{6}))(?=\s+\p{L})/giu, - ' x ', - ) - .replace(/(?<=\p{L}\.?,?\s)\d{5}-\d{4}\b/gu, ' x ') - .replace(/\b\d{5}-\d{3}\b/g, ' x ') - .replace( - /〒\s?\d{3}-\d{4}|\b\d{3}-\d{4}(?=\s*[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}])/gu, - ' x ', - ) - .replace(/#/g, ' x ') - .replace(/[^\p{L}\p{N}]/gu, ''); + // A word that says what the number is ("card 4111 111111", "call 0207 + // 946095") keeps the digits together: no address is read out of them. + const named = NUMBER_CONTEXT.test(line); + const digits = ( + named + ? line + : line + .replace( + /(^|\s)\d{1,4}[a-z]?,?\s+\d{5}(?:-\d{3})?(?=\s+\p{L}|$)/giu, + '$1 ', + ) + // Other postcodes after a street's house number and before the city (a + // four-digit one after a comma, so "is 555 0100 thanks" stays a number): + // "Nowy Świat 10, 00-950 Warszawa", "Drottninggatan 53, 111 21 + // Stockholm", "Mariahilfer Straße 120, 1070 Wien", "Rua Augusta 274, + // 1100-053 Lisboa", "Tverskaya 13 125009 Moscow". + .replace( + /(?<=\p{L}\.?\s)\d{1,4}[a-z]?(?:,\s*\d{4}(?:-\d{3})?|,?\s+(?:\d{2}-\d{3}|\d{3} \d{2}|\d{6}))(?=\s+\p{L})/giu, + ' x ', + ) + .replace(/(?<=\p{L}\.?,?\s)\d{5}-\d{4}\b/gu, ' x ') + .replace(/\b\d{5}-\d{3}\b/g, ' x ') + .replace( + /〒\s?\d{3}-\d{4}|\b\d{3}-\d{4}(?=\s*[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}])/gu, + ' x ', + ) + .replace(/#/g, ' x ') + ).replace(/[^\p{L}\p{N}]/gu, ''); const longNumber = /\d{7,}/.test(digits) || // A phone number written with dots ("415.555.0100"). @@ -205,6 +214,8 @@ function mayBeSecret(token) { // is a name. // Latin letters only: CJK addresses are written without spaces // ("東京都港区六本木6丁目", "北京市朝阳区建国路88号"). + // A UK postcode written without its space ("EC1A1BB", "BT487NN"). + if (/^[a-z]{1,2}\d[a-z\d]?\d[a-z]{2}$/i.test(token)) return false; const mixed = token .split(/[^\p{L}\p{N}]+/u) .some((run) => /\p{Script=Latin}/u.test(run) && /\d/.test(run)); @@ -221,6 +232,19 @@ const TIMEOUT_MS = 6000; const PAUSE_AFTER_REFUSAL_MS = 60_000; const CACHE_LIMIT = 200; +/** + * The spacing, the pause and the cache, shared by every suggester in the tab: + * a search box that is put up again starts from where the last one left off. + */ +const shared = { nextRequestAt: 0, pausedUntil: 0, cache: new Map() }; + +/** Forget the shared spacing, pause and cache (tests). */ +export function resetPlaceSuggestions() { + shared.nextRequestAt = 0; + shared.pausedUntil = 0; + shared.cache.clear(); +} + /** List Photon candidates while typing. The submitted search owns navigation. */ export function createPlaceSuggester({ endpoint, @@ -228,9 +252,7 @@ export function createPlaceSuggester({ limit = 4, } = {}) { const enabled = endpoint === undefined || Boolean(endpoint); - const cache = new Map(); - let nextRequestAt = 0; - let pausedUntil = 0; + const { cache } = shared; return { enabled, @@ -244,14 +266,14 @@ export function createPlaceSuggester({ const language = (globalThis.document?.documentElement?.lang || 'en') .slice(0, 2) .toLowerCase(); - const key = `${language}|${text.toLowerCase()}`; + const key = `${endpoint ?? ''}|${limit}|${language}|${text.toLowerCase()}`; if (cache.has(key)) return cache.get(key); // Concurrent callers recheck the gap after waking before sending. - while (nextRequestAt > Date.now()) { + while (shared.nextRequestAt > Date.now()) { let timer; const pause = new Promise((resolve) => { - timer = setTimeout(resolve, nextRequestAt - Date.now()); + timer = setTimeout(resolve, shared.nextRequestAt - Date.now()); }); try { await waitForSignal(pause, signal); @@ -260,25 +282,32 @@ export function createPlaceSuggester({ } } signal?.throwIfAborted(); - if (Date.now() < pausedUntil) return []; + if (Date.now() < shared.pausedUntil) return []; const url = new URL(photonSearchUrl(text, { endpoint, limit })); if (['en', 'de', 'fr', 'it'].includes(language)) url.searchParams.set('lang', language); - nextRequestAt = Date.now() + MIN_GAP_MS; - const features = await requestWithDeadline( - async (requestSignal) => { - const response = await fetchImpl(url, { signal: requestSignal }); - requestSignal.throwIfAborted(); - if (!response.ok) { - pausedUntil = Date.now() + PAUSE_AFTER_REFUSAL_MS; - return null; - } - const data = await response.json(); - return data?.features; - }, - { signal, timeoutMs: TIMEOUT_MS }, - ); + shared.nextRequestAt = Date.now() + MIN_GAP_MS; + let features; + try { + features = await requestWithDeadline( + async (requestSignal) => { + const response = await fetchImpl(url, { signal: requestSignal }); + requestSignal.throwIfAborted(); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + return data?.features; + }, + { signal, timeoutMs: TIMEOUT_MS }, + ); + } catch (error) { + signal?.throwIfAborted(); + // A refusal, a timeout or a failed request (a refusal sent without + // CORS headers arrives as one) all hold suggestions off a while. + void error; + shared.pausedUntil = Date.now() + PAUSE_AFTER_REFUSAL_MS; + return []; + } signal?.throwIfAborted(); if (!Array.isArray(features)) return []; diff --git a/src/search/placeSuggestions.test.mjs b/src/search/placeSuggestions.test.mjs index a549202f2c..f5e2ac6f51 100644 --- a/src/search/placeSuggestions.test.mjs +++ b/src/search/placeSuggestions.test.mjs @@ -5,8 +5,12 @@ import { createPlaceSuggester, inExtent, mayBePrivate, + resetPlaceSuggestions, } from './placeSuggestions.js'; +// The spacing, pause and cache are the tab's: each test starts clean. +test.beforeEach(() => resetPlaceSuggestions()); + test('Photon suggestions keep distinct valid candidates and their region bounds', async () => { const texas = { geometry: { type: 'Point', coordinates: [-99, 31] }, @@ -278,3 +282,48 @@ test('cards, phones, IBANs, keys and passwords are private', () => { ]) assert.equal(mayBePrivate(text), false, text); }); + +const answering = (calls, answer) => async (url) => { + calls.push(String(url)); + if (answer instanceof Error) throw answer; + return { ok: true, json: async () => ({ features: [] }) }; +}; + +test('a failed or timed-out request pauses suggestions as a refusal does', async () => { + const calls = []; + const suggester = createPlaceSuggester({ + endpoint: 'https://photon.test/api/', + fetchImpl: answering(calls, new TypeError('Failed to fetch')), + }); + assert.deepEqual(await suggester.suggest('Paris'), []); + assert.deepEqual(await suggester.suggest('Berlin'), []); + assert.equal(calls.length, 1); +}); + +test('every suggester in the tab keeps one spacing and one pause', async () => { + const calls = []; + const options = { + endpoint: 'https://photon.test/api/', + fetchImpl: answering(calls), + }; + const first = createPlaceSuggester(options); + const second = createPlaceSuggester(options); + const started = Date.now(); + await Promise.all([first.suggest('Paris'), second.suggest('Berlin')]); + assert.equal(calls.length, 2); + assert.ok(Date.now() - started >= 490); +}); + +test('a number named as a card or a phone stays private', () => { + for (const line of [ + 'card 4111 111111 thanks', + 'call 0207 946095 now', + 'phone 12 345678 home', + ]) + assert.equal(mayBePrivate(line), true, line); +}); + +test('a UK postcode without its space is a place', () => { + for (const line of ['EC1A1BB', 'W1A0AX', 'BT487NN']) + assert.equal(mayBePrivate(line), false, line); +});