From 66eb6d6a7c4fccd44f156dd62f8c2a84ded1742b Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Mon, 14 Sep 2026 22:08:16 -0700 Subject: [PATCH] refactor: add configurable Nominatim adapter and portable source helpers --- CHANGELOG.md | 2 + docs/CODE-BOUNDARIES.md | 50 ++++++++ docs/CURRENT-STATE.md | 2 + package.json | 5 +- scripts/format-scope.json | 6 +- scripts/package-boundaries.json | 73 +++++++++-- server/providers/common/http.js | 58 +-------- server/providers/overpass/query.js | 49 +------- src/search/defaults.js | 93 ++++++++------ src/search/index.js | 6 + src/search/nominatim.js | 158 +++++++++++++++++++++++ src/search/nominatim.test.mjs | 194 +++++++++++++++++++++++++++++ src/sources/httpBody.js | 54 ++++++++ src/sources/overpass.js | 48 +++++++ 14 files changed, 646 insertions(+), 152 deletions(-) create mode 100644 src/search/nominatim.js create mode 100644 src/search/nominatim.test.mjs create mode 100644 src/sources/httpBody.js create mode 100644 src/sources/overpass.js diff --git a/CHANGELOG.md b/CHANGELOG.md index ef58e2e65..515c20845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +- Add an optional Nominatim geocoding adapter with configurable search/reverse endpoints, cancellation, bounded responses and retryable upstream errors. Extract portable response-reading and Overpass lexical helpers while retaining existing server exports. + - Expose reference feed factories independently of standalone catalog construction; preserve source choices and asset attribution. - Add source-only layer exports and enforce source, browser, standalone and voice import directions. Move plain record/feed helpers and settings filesystem hardening to their owners while preserving compatibility and behavior. diff --git a/docs/CODE-BOUNDARIES.md b/docs/CODE-BOUNDARIES.md index 6d6d4d55f..7bb89d80a 100644 --- a/docs/CODE-BOUNDARIES.md +++ b/docs/CODE-BOUNDARIES.md @@ -545,3 +545,53 @@ geometry and billboard resources used by cards, picking and trails. source instances independently of standalone setup. Individual sources remain available through `layers/earthquakes/source` and `layers/submarine-cables/source`. The latter retains the bundled dataset’s attribution and licensing requirements. + +## Geocoding and feature-query providers + +`createDefaultPlaceSearch` accepts an explicit Nominatim selection: + +```js +const places = createDefaultPlaceSearch({ + geocoding: { + provider: 'nominatim', + searchEndpoint: '/places/search', + reverseEndpoint: '/places/reverse', + }, + fetchImpl, + signal, +}); +``` + +The endpoints must return Nominatim JSONv2. They are supplied by application +composition; no instance is selected automatically and this does not install +those routes. For a remote instance, configure its full URLs and transport +requirements. The caller owns request scheduling, identifying headers and any +instance-specific usage policy. The existing public fallback retains its shared +queue, pacing and cache. `endpoints.nominatim` still names the legacy +Google-shaped `/api/geocode` fallback; it is not a raw Nominatim endpoint. + +Coordinates and supplied presets remain offline. Explicit Nominatim selection +replaces the network forward-geocoder chain; it does not silently fall back to +another instance. Reverse geocoding is available only when a reverse endpoint +is configured. Other operations (Places text/nearby search and routing) retain +their independent providers. For custom compositions, `createNominatimProvider` +from `./search/nominatim` can be passed to `createPlaceSearch` and +`createGeospatialServices`; it exposes no route or nearby-search capability. +An empty search array is a definitive miss. HTTP failures, malformed results +and oversized responses remain retryable. Attribution identifies OpenStreetMap / +Nominatim. + +Overpass currently supplies traffic road geometry, ALPR camera records, military +installation footprints, and annotation geometry (administrative boundaries, +neighborhoods, streets, building/grounds outlines and monument candidates). +Nominatim forward/reverse lookup does not replace those queries. Layer source +interfaces select traffic, camera and installation ingestion separately; +annotation `boundaries.query` still uses Overpass QL and OSM-shaped elements. +Replacing that service with a different query language requires operation-level +feature/geometry adapters, not just changing its URL. Rendering and geometry +selection remain consumers of those results. + +`./sources/http-body` exports bounded text/JSON readers using web primitives; +`./sources/overpass` exports the quoted-string/comment lexer. The lexer alone is +not a query validator: spatial bounds, timeouts and other policy remain in the +server sanitizer. Existing server imports keep their compatibility exports. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index e0d94a6ae..5e533f4e5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,7 @@ # God's Eye View Current State +Place search accepts an explicit Nominatim provider with independently configured search and reverse endpoints. The default offline/Google/Photon/local-fallback order is unchanged when no provider is selected. Provider adapters share normalized coordinates, viewport framing and reverse labels; roads and boundary geometry remain separate services. Portable capped-response and Overpass lexical helpers are exported independently of the Node server. + Reference feed construction is exported through `sources/reference`; the cable source also has a dedicated `layers/submarine-cables/source` entry. Standalone catalog compatibility remains available. Source choices, data and attribution are unchanged. Source factories have dedicated `layers//source` exports. ALPR and earthquake record normalization and CCTV source policy no longer pull rendering into source consumers. Catalog construction and voice feed reads use their focused owners; compatibility entries remain available. Settings filesystem hardening lives under `server/standalone/`. Import-direction checks complement export ownership checks; source behavior, settings policy and rendering are unchanged. diff --git a/package.json b/package.json index c9ad7e86c..3171375d4 100644 --- a/package.json +++ b/package.json @@ -226,6 +226,9 @@ "./layers/satellites/source": "./src/layers/satellites/source.js", "./layers/traffic/source": "./src/layers/traffic/source.js", "./sources/reference": "./src/sources/reference.js", - "./layers/submarine-cables/source": "./src/layers/submarineCables/bundledSource.js" + "./layers/submarine-cables/source": "./src/layers/submarineCables/bundledSource.js", + "./sources/http-body": "./src/sources/httpBody.js", + "./sources/overpass": "./src/sources/overpass.js", + "./search/nominatim": "./src/search/nominatim.js" } } diff --git a/scripts/format-scope.json b/scripts/format-scope.json index bf3be68ff..4018a9498 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -670,5 +670,9 @@ "scripts/check-import-directions.mjs", "src/tooling/importDirections.test.mjs", "src/sources/reference.js", - "src/sources/reference.test.mjs" + "src/sources/reference.test.mjs", + "src/search/nominatim.js", + "src/search/nominatim.test.mjs", + "src/sources/httpBody.js", + "src/sources/overpass.js" ] diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 8ee1cfaa8..cc5561a89 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -44,7 +44,8 @@ "server/providers/vessels/ais-live.js", "src/data/adsbLolFallback.js", "src/data/aisStreamAdapter.js", - "src/data/aisWatchdog.js" + "src/data/aisWatchdog.js", + "src/sources/httpBody.js" ], "external": ["ws"] }, @@ -66,7 +67,8 @@ "server/providers/common/http.js", "scripts/google-server-key.mjs", "src/data/placeProviderPayloads.js", - "src/data/routeSteps.js" + "src/data/routeSteps.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -83,7 +85,8 @@ "server/providers/space/celestrak.js", "server/providers/space/launch-library.js", "server/providers/common/http.js", - "src/data/spaceProviderRequests.js" + "src/data/spaceProviderRequests.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -134,7 +137,8 @@ "modules": [ "server/providers/gbfs.js", "server/providers/common/http.js", - "src/data/gbfsSource.js" + "src/data/gbfsSource.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -162,7 +166,8 @@ "server/providers/places/google-key.js", "scripts/google-server-key.mjs", "src/data/cctvFootprint.js", - "src/sources/cctvTypes.js" + "src/sources/cctvTypes.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -177,7 +182,8 @@ "server/providers/radio/transport.js", "src/data/radioCountry.js", "server/providers/common/http.js", - "src/sources/radioBrowser.js" + "src/sources/radioBrowser.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -197,7 +203,9 @@ "server/providers/overpass/transport.js", "server/providers/places/routes.js", "src/data/placeProviderPayloads.js", - "src/data/routeSteps.js" + "src/data/routeSteps.js", + "src/sources/httpBody.js", + "src/sources/overpass.js" ], "external": [] }, @@ -214,7 +222,8 @@ "server/providers/military-installations/query.js", "server/providers/overpass/constants.js", "server/providers/overpass/geometry.js", - "server/providers/overpass/transport.js" + "server/providers/overpass/transport.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -234,7 +243,8 @@ "server/providers/regional/weather-effects.js", "server/providers/regional/weather.js", "src/data/regionalModel.js", - "src/nominatimGeocode.js" + "src/nominatimGeocode.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -284,7 +294,11 @@ "src/search/defaults.js", "src/search/coordinateGeocoder.js", "src/search/coordinateParser.js", - "src/search/presetGeocoder.js" + "src/search/presetGeocoder.js", + "src/search/nominatim.js", + "src/nominatimGeocode.js", + "src/data/regionalModel.js", + "src/sources/httpBody.js" ], "external": [] }, @@ -1619,7 +1633,10 @@ "src/layers/alpr/records.js", "src/layers/earthquakes/records.js", "src/layers/cctv/sourcePolicy.js", - "src/sources/reference.js" + "src/sources/reference.js", + "src/search/nominatim.js", + "src/nominatimGeocode.js", + "src/sources/httpBody.js" ], "external": [ "@mapbox/vector-tile", @@ -1971,7 +1988,11 @@ "src/app/localGeojsonServices.js", "src/layers/alpr/records.js", "src/layers/earthquakes/records.js", - "src/layers/cctv/sourcePolicy.js" + "src/layers/cctv/sourcePolicy.js", + "src/search/nominatim.js", + "src/nominatimGeocode.js", + "src/data/regionalModel.js", + "src/sources/httpBody.js" ], "external": [ "cesium", @@ -2015,7 +2036,11 @@ "src/services/terrainHeights.js", "src/search/coordinateGeocoder.js", "src/search/coordinateParser.js", - "src/search/presetGeocoder.js" + "src/search/presetGeocoder.js", + "src/search/nominatim.js", + "src/nominatimGeocode.js", + "src/data/regionalModel.js", + "src/sources/httpBody.js" ], "external": [ "cesium", @@ -2253,5 +2278,27 @@ "src/layers/submarineCables/bundledSource.js" ], "external": [] + }, + "nominatim-provider": { + "exports": ["./search/nominatim"], + "modules": [ + "src/search/nominatim.js", + "src/nominatimGeocode.js", + "src/search/google.js", + "src/search/geospatial.js", + "src/data/regionalModel.js", + "src/sources/httpBody.js" + ], + "external": [] + }, + "source-http-body": { + "exports": ["./sources/http-body"], + "modules": ["src/sources/httpBody.js"], + "external": [] + }, + "overpass-lexer": { + "exports": ["./sources/overpass"], + "modules": ["src/sources/overpass.js"], + "external": [] } } diff --git a/server/providers/common/http.js b/server/providers/common/http.js index b3a902949..0e530bb29 100644 --- a/server/providers/common/http.js +++ b/server/providers/common/http.js @@ -1,57 +1,7 @@ -/** - * Read a fetch() Response body as text with a hard byte cap. Rejects early on an - * oversized Content-Length, then streams with a running cap so a chunked or - * length-omitted response cannot blow past the limit. Throws { code:'RESPONSE_TOO_LARGE' }. - */ -export async function readResponseTextCapped(response, maxBytes, signal) { - const tooLarge = () => - Object.assign(new Error('Upstream response too large'), { - code: 'RESPONSE_TOO_LARGE', - }); - const declared = Number(response.headers.get('content-length')); - if (Number.isFinite(declared) && declared > maxBytes) { - void response.body?.cancel().catch(() => {}); - throw tooLarge(); - } - signal?.throwIfAborted(); - const reader = response.body?.getReader?.(); - if (!reader) { - const text = await response.text(); - signal?.throwIfAborted(); - if (Buffer.byteLength(text) > maxBytes) throw tooLarge(); - return text; - } - const cancel = () => { - void reader.cancel().catch(() => {}); - }; - signal?.addEventListener('abort', cancel, { once: true }); - const decoder = new TextDecoder(); - let out = ''; - let total = 0; - try { - for (;;) { - signal?.throwIfAborted(); - const { done, value } = await reader.read(); - signal?.throwIfAborted(); - if (done) break; - total += value.byteLength; - if (total > maxBytes) throw tooLarge(); - out += decoder.decode(value, { stream: true }); - } - return out + decoder.decode(); - } catch (error) { - cancel(); - throw error; - } finally { - signal?.removeEventListener('abort', cancel); - reader.releaseLock(); - } -} - -/** Parse a fetch() JSON response only after enforcing a hard byte cap. */ -export async function readResponseJsonCapped(response, maxBytes, signal) { - return JSON.parse(await readResponseTextCapped(response, maxBytes, signal)); -} +export { + readResponseTextCapped, + readResponseJsonCapped, +} from '../../../src/sources/httpBody.js'; /** * Return the existing promise for a cache key, or create one and remove it diff --git a/server/providers/overpass/query.js b/server/providers/overpass/query.js index b40266e22..cee444d3a 100644 --- a/server/providers/overpass/query.js +++ b/server/providers/overpass/query.js @@ -1,3 +1,4 @@ +import { stripOverpassNoise } from '../../../src/sources/overpass.js'; import { OVERPASS_MAX_AROUND_M, OVERPASS_MAX_BBOX_DEG, @@ -31,54 +32,6 @@ function isOverpassBoundaryQuery(cacheKey) { * * @returns {{ok:true, body:string} | {ok:false, error:string}} */ -/** - * Single-pass lexer: blank out quoted literals (→ empty quotes) and strip line - * and block comments — recognizing each in one walk so a comment marker INSIDE a - * quoted string is treated as string content, not a comment (and vice versa). - * Chained regex replaces get the ordering wrong (a quoted slash-slash would hide - * the rest of the line), which is exactly the bypass this avoids. - */ -function stripOverpassNoise(src) { - let out = ''; - let i = 0; - const n = src.length; - while (i < n) { - const c = src[i]; - if (c === '"' || c === "'") { - const quote = c; - i += 1; - while (i < n) { - if (src[i] === '\\') { - i += 2; - continue; - } // escaped char - if (src[i] === quote) { - i += 1; - break; - } // closing quote - i += 1; - } - out += quote + quote; // collapse the literal to empty quotes - continue; - } - if (c === '/' && src[i + 1] === '*') { - i += 2; - while (i < n && !(src[i] === '*' && src[i + 1] === '/')) i += 1; - i += 2; - out += ' '; - continue; - } - if (c === '/' && src[i + 1] === '/') { - i += 2; - while (i < n && src[i] !== '\n') i += 1; - out += ' '; - continue; - } - out += c; - i += 1; - } - return out; -} function sanitizeOverpassBody(rawBody) { let params; diff --git a/src/search/defaults.js b/src/search/defaults.js index f97ddbf17..2b0428025 100644 --- a/src/search/defaults.js +++ b/src/search/defaults.js @@ -1,3 +1,4 @@ +import { createNominatimProvider } from './nominatim.js'; import { createGeospatialServices } from './geospatial.js'; import { createHttpGeospatialProvider } from './http.js'; import { createPlaceSearch } from './placeSearch.js'; @@ -22,54 +23,76 @@ export function createDefaultPlaceSearch({ endpoints = {}, providers = {}, presets = null, + geocoding = null, } = {}) { + if (geocoding && geocoding.provider !== 'nominatim') + throw new TypeError('Unsupported geocoding provider'); + const selected = geocoding + ? createNominatimProvider({ ...geocoding, fetchImpl }) + : null; + if (selected && !selected.geocode) + throw new TypeError('Nominatim searchEndpoint is required'); const forward = createPlaceSearch({ signal, providers: providers.geocode || [ createCoordinateGeocoder(), ...(presets ? [createPresetGeocoder({ presets })] : []), - createGoogleGeocoder({ - request(query, { bias, signal }) { - const key = resolveApiKey?.(); - if (!key) return null; - const url = new URL( - endpoints.geocode || - 'https://maps.googleapis.com/maps/api/geocode/json', - ); - url.searchParams.set('address', query); - url.searchParams.set('key', key); - if (bias) url.searchParams.set('bounds', bias); - return fetchImpl(url.toString(), { signal }); - }, - }), - createPhotonGeocoder({ fetchImpl, endpoint: endpoints.photon }), - // Last resort: the local Nominatim route, which answers with no key when - // neither of the two above did. It speaks the same result shape, so it - // rides the existing Google adapter rather than needing its own. - createGoogleGeocoder({ - request(query, { bias, signal }) { - const params = new URLSearchParams({ q: query }); - if (bias) params.set('bounds', bias); - return fetchImpl( - `${endpoints.nominatim || '/api/geocode'}?${params}`, - { - signal, - }, - ); - }, - }), + ...(selected + ? [selected] + : [ + createGoogleGeocoder({ + request(query, { bias, signal }) { + const key = resolveApiKey?.(); + if (!key) return null; + const url = new URL( + endpoints.geocode || + 'https://maps.googleapis.com/maps/api/geocode/json', + ); + url.searchParams.set('address', query); + url.searchParams.set('key', key); + if (bias) url.searchParams.set('bounds', bias); + return fetchImpl(url.toString(), { signal }); + }, + }), + createPhotonGeocoder({ fetchImpl, endpoint: endpoints.photon }), + // Last resort: the local Nominatim route, which answers with no key when + // neither of the two above did. It speaks the same result shape, so it + // rides the existing Google adapter rather than needing its own. + createGoogleGeocoder({ + request(query, { bias, signal }) { + const params = new URLSearchParams({ q: query }); + if (bias) params.set('bounds', bias); + return fetchImpl( + `${endpoints.nominatim || '/api/geocode'}?${params}`, + { + signal, + }, + ); + }, + }), + ]), ], }); + const operations = createHttpGeospatialProvider({ + fetchImpl, + resolveApiKey, + endpoints, + }); return { ...forward, ...createGeospatialServices({ signal, providers: { - ...createHttpGeospatialProvider({ - fetchImpl, - resolveApiKey, - endpoints, - }), + ...operations, + ...(selected + ? { + reverseGeocode: selected.reverseGeocode, + attribution: { + ...operations.attribution, + ...selected.attribution, + }, + } + : {}), ...providers, }, }), diff --git a/src/search/index.js b/src/search/index.js index 119d95160..eb0e727cc 100644 --- a/src/search/index.js +++ b/src/search/index.js @@ -13,3 +13,9 @@ export { parseCoordinateQuery, formatCoordinateLabel, } from './coordinateParser.js'; +export { + createNominatimProvider, + createNominatimClient, + normalizeNominatimResult, + normalizeNominatimReverse, +} from './nominatim.js'; diff --git a/src/search/nominatim.js b/src/search/nominatim.js new file mode 100644 index 000000000..029d68224 --- /dev/null +++ b/src/search/nominatim.js @@ -0,0 +1,158 @@ +import { + nominatimToGeocodeResult, + nominatimViewboxFromBounds, +} from '../nominatimGeocode.js'; +import { normalizeGooglePlace } from './google.js'; +import { validCoordinate } from './geospatial.js'; +import { normalizeRegionalPlace } from '../data/regionalModel.js'; +import { readResponseJsonCapped } from '../sources/httpBody.js'; + +/** Retain the established framing/type rules for JSON and JSONv2 results. */ +export function normalizeNominatimResult(hit) { + return nominatimToGeocodeResult( + hit && { ...hit, class: hit.class || hit.category }, + ); +} + +export function normalizeNominatimReverse(hit) { + const result = normalizeNominatimResult(hit); + const region = normalizeRegionalPlace(hit); + if (!result || !region) return null; + const clean = (value) => + String(value || '') + .replace(/[\x00-\x1f\x7f]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 120); + const street = clean(hit.address?.road || hit.address?.pedestrian); + return { + formattedAddress: result.formatted_address, + locality: region.locality, + region: region.region, + country: region.country, + types: result.types, + labels: [clean(result.formatted_address)].filter(Boolean), + streetLabels: street ? [street] : [], + }; +} + +/** + * Nominatim JSONv2 transport for explicitly configured endpoints. Callers own + * scheduling and request headers; this does not select a public instance. + */ +export function createNominatimClient({ + searchEndpoint, + reverseEndpoint, + fetchImpl = (...args) => fetch(...args), +} = {}) { + for (const endpoint of [searchEndpoint, reverseEndpoint]) { + if (endpoint == null) continue; + if (typeof endpoint !== 'string' || !endpoint || /[?#]/.test(endpoint)) + throw new TypeError('A query-free Nominatim endpoint is required'); + const url = new URL(endpoint, 'http://localhost'); + if ( + !['http:', 'https:'].includes(url.protocol) || + url.username || + url.password || + !( + (endpoint.startsWith('/') && !endpoint.startsWith('//')) || + /^https?:\/\//.test(endpoint) + ) + ) + throw new TypeError('Invalid Nominatim endpoint'); + } + async function request(endpoint, params, signal) { + signal?.throwIfAborted(); + const response = await fetchImpl( + `${endpoint}?${new URLSearchParams({ + format: 'jsonv2', + addressdetails: '1', + ...params, + })}`, + { signal, redirect: 'error' }, + ); + signal?.throwIfAborted(); + if (!response.ok) { + await response.body?.cancel?.().catch(() => {}); + throw new Error('Geocoding service unavailable'); + } + return readResponseJsonCapped(response, 512 * 1024, signal); + } + return { + ...(searchEndpoint + ? { + async search(query, { bias, signal } = {}) { + const viewbox = nominatimViewboxFromBounds(bias); + const rows = await request( + searchEndpoint, + { + q: query, + limit: '1', + ...(viewbox ? { viewbox, bounded: '0' } : {}), + }, + signal, + ); + if (!Array.isArray(rows)) + throw new Error('Invalid geocoding response'); + if (rows.length && !normalizeNominatimResult(rows[0])) + throw new Error('Invalid geocoding result'); + return rows; + }, + } + : {}), + ...(reverseEndpoint + ? { + async reverse(latitude, longitude, { signal } = {}) { + if (!validCoordinate([longitude, latitude])) return null; + const row = await request( + reverseEndpoint, + { lat: latitude, lon: longitude }, + signal, + ); + // A successful, explicit no-result response is distinct from malformed data. + if (row?.error === 'Unable to geocode') return null; + if (!normalizeNominatimReverse(row)) + throw new Error('Invalid reverse geocoding result'); + return row; + }, + } + : {}), + }; +} + +/** Forward and reverse operations only; feature queries and routing are separate. */ +export function createNominatimProvider(options = {}) { + const client = createNominatimClient(options); + return { + attribution: { + geocode: 'OpenStreetMap / Nominatim', + reverseGeocode: 'OpenStreetMap / Nominatim', + }, + ...(client.search + ? { + async geocode(query, options = {}) { + try { + const rows = await client.search(query, options); + return { + place: rows.length + ? normalizeGooglePlace(normalizeNominatimResult(rows[0])) + : null, + answered: true, + }; + } catch { + options.signal?.throwIfAborted(); + return { place: null, answered: false }; + } + }, + } + : {}), + ...(client.reverse + ? { + async reverseGeocode(latitude, longitude, options) { + const row = await client.reverse(latitude, longitude, options); + return row ? normalizeNominatimReverse(row) : null; + }, + } + : {}), + }; +} diff --git a/src/search/nominatim.test.mjs b/src/search/nominatim.test.mjs new file mode 100644 index 000000000..a2667f129 --- /dev/null +++ b/src/search/nominatim.test.mjs @@ -0,0 +1,194 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createNominatimProvider } from './nominatim.js'; +import { createDefaultPlaceSearch } from './defaults.js'; +import { createPlaceSearch } from './placeSearch.js'; +import { createGeospatialServices } from './geospatial.js'; +import { readResponseTextCapped } from '../sources/httpBody.js'; + +const hit = { + lat: '51.5', + lon: '-0.12', + name: 'London', + display_name: 'London, England', + category: 'place', + type: 'city', + addresstype: 'city', + boundingbox: ['51', '52', '-1', '0'], + address: { + city: 'London', + state: 'England', + country: 'United Kingdom', + road: 'Whitehall', + }, +}; +const endpoints = { + searchEndpoint: 'https://places.example/search', + reverseEndpoint: 'https://places.example/reverse', +}; + +test('Nominatim normalizes framing and reverse context through independent configured endpoints', async () => { + const calls = []; + const adapter = createNominatimProvider({ + ...endpoints, + fetchImpl: async (url, init) => { + calls.push({ url: new URL(url), init }); + return Response.json(url.includes('/search?') ? [hit] : hit); + }, + }); + const answer = await adapter.geocode('London', { bias: '50,-2|53,1' }); + assert.equal(answer.answered, true); + assert.equal(answer.place.name, 'London'); + assert.deepEqual(answer.place.types, ['locality', 'political']); + assert.deepEqual(answer.place.viewport, { + southwest: { lat: 51, lng: -1 }, + northeast: { lat: 52, lng: 0 }, + }); + const reverse = await adapter.reverseGeocode(51.5, -0.12); + assert.equal(reverse.locality, 'London'); + assert.deepEqual(reverse.streetLabels, ['Whitehall']); + assert.equal(calls[0].url.searchParams.get('viewbox'), '-2,53,1,50'); + assert.equal(calls[0].url.searchParams.get('bounded'), '0'); + assert.equal(calls[0].url.searchParams.get('format'), 'jsonv2'); + assert.equal(calls[1].url.pathname, '/reverse'); + assert.equal(calls[1].init.redirect, 'error'); + assert.equal(calls[1].init.headers, undefined); + assert.deepEqual( + createGeospatialServices({ providers: adapter }).capabilities, + { reverseGeocode: true, textSearch: false, nearby: false, route: false }, + ); +}); + +test('explicit Nominatim choice retains offline coordinates and bypasses the default geocoder chain', async () => { + const calls = []; + const search = createDefaultPlaceSearch({ + geocoding: { provider: 'nominatim', ...endpoints }, + resolveApiKey: () => { + throw new Error('must not request a Google key'); + }, + fetchImpl: async (url) => { + calls.push(url); + return Response.json(url.includes('/search?') ? [hit] : hit); + }, + }); + assert.equal((await search.geocode('London')).place.name, 'London'); + await search.reverseGeocode(51.5, -0.12); + assert.equal(calls.length, 2); + assert(calls.every((url) => new URL(url).hostname === 'places.example')); + assert.equal(search.attribution.reverseGeocode, 'OpenStreetMap / Nominatim'); + assert.equal(search.attribution.route, 'OpenStreetMap / OSRM'); + await search.geocode('51.5, -0.12'); + assert.equal(calls.length, 2); + assert.throws(() => + createDefaultPlaceSearch({ geocoding: { provider: 'unknown' } }), + ); + assert.throws(() => + createDefaultPlaceSearch({ geocoding: { provider: 'nominatim' } }), + ); +}); + +test('outages and malformed answers are retryable; only an explicit empty array is a cached miss', async () => { + for (const makeResponse of [ + () => Response.json({}, { status: 429 }), + () => Response.json({ error: 'bad' }), + () => Response.json([{ ...hit, lat: '' }]), + () => new Response('{'), + () => + Response.json([hit], { headers: { 'content-length': String(600000) } }), + ]) { + let count = 0; + const adapter = createNominatimProvider({ + ...endpoints, + fetchImpl: async () => { + count++; + return makeResponse(); + }, + }); + const service = createPlaceSearch({ providers: [adapter] }); + assert.equal((await service.geocode('missing')).place, null); + assert.equal((await service.geocode('missing')).place, null); + assert.equal(count, 2); + } + let count = 0; + const adapter = createNominatimProvider({ + ...endpoints, + fetchImpl: async () => { + count++; + return Response.json([]); + }, + }); + const service = createPlaceSearch({ providers: [adapter] }); + await service.geocode('missing'); + await service.geocode('missing'); + assert.equal(count, 1); +}); + +test('aborting a pending Nominatim body cancels its stream and rejects instead of caching a miss', async () => { + const controller = new AbortController(); + let cancelled = false; + const adapter = createNominatimProvider({ + ...endpoints, + fetchImpl: async () => + new Response( + new ReadableStream({ + start() { + setTimeout(() => controller.abort(), 10); + }, + cancel() { + cancelled = true; + }, + }), + ), + }); + await assert.rejects( + adapter.geocode('London', { signal: controller.signal }), + { name: 'AbortError' }, + ); + assert.equal(cancelled, true); +}); + +test('portable response cap counts UTF-8 bytes and cancels oversized streams', async () => { + await assert.rejects( + readResponseTextCapped( + { headers: new Headers(), text: async () => 'éé' }, + 3, + ), + { code: 'RESPONSE_TOO_LARGE' }, + ); + let cancelled = false; + const response = new Response( + new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array(4)); + }, + cancel() { + cancelled = true; + }, + }), + ); + await assert.rejects(readResponseTextCapped(response, 3), { + code: 'RESPONSE_TOO_LARGE', + }); + assert.equal(cancelled, true); +}); + +test('unconfigured operations stay absent and invalid coordinates do not make requests', async () => { + assert.equal(createNominatimProvider().geocode, undefined); + let calls = 0; + const adapter = createNominatimProvider({ + ...endpoints, + fetchImpl: async () => { + calls++; + return Response.json(hit); + }, + }); + assert.equal(await adapter.reverseGeocode(100, 0), null); + assert.equal(calls, 0); + for (const searchEndpoint of [ + 'https://u:p@example.com/search', + 'https://example.com/search?key=x', + '//example.com/search', + 'file:///search', + ]) + assert.throws(() => createNominatimProvider({ searchEndpoint })); +}); diff --git a/src/sources/httpBody.js b/src/sources/httpBody.js new file mode 100644 index 000000000..a3e5cf736 --- /dev/null +++ b/src/sources/httpBody.js @@ -0,0 +1,54 @@ +/** + * Read a fetch() Response body as text with a hard byte cap. Rejects early on an + * oversized Content-Length, then streams with a running cap so a chunked or + * length-omitted response cannot blow past the limit. Throws { code:'RESPONSE_TOO_LARGE' }. + */ +export async function readResponseTextCapped(response, maxBytes, signal) { + const tooLarge = () => + Object.assign(new Error('Upstream response too large'), { + code: 'RESPONSE_TOO_LARGE', + }); + const declared = Number(response.headers.get('content-length')); + if (Number.isFinite(declared) && declared > maxBytes) { + void response.body?.cancel().catch(() => {}); + throw tooLarge(); + } + signal?.throwIfAborted(); + const reader = response.body?.getReader?.(); + if (!reader) { + const text = await response.text(); + signal?.throwIfAborted(); + if (new TextEncoder().encode(text).byteLength > maxBytes) throw tooLarge(); + return text; + } + const cancel = () => { + void reader.cancel().catch(() => {}); + }; + signal?.addEventListener('abort', cancel, { once: true }); + const decoder = new TextDecoder(); + let out = ''; + let total = 0; + try { + for (;;) { + signal?.throwIfAborted(); + const { done, value } = await reader.read(); + signal?.throwIfAborted(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) throw tooLarge(); + out += decoder.decode(value, { stream: true }); + } + return out + decoder.decode(); + } catch (error) { + cancel(); + throw error; + } finally { + signal?.removeEventListener('abort', cancel); + reader.releaseLock(); + } +} + +/** Parse a fetch() JSON response only after enforcing a hard byte cap. */ +export async function readResponseJsonCapped(response, maxBytes, signal) { + return JSON.parse(await readResponseTextCapped(response, maxBytes, signal)); +} diff --git a/src/sources/overpass.js b/src/sources/overpass.js new file mode 100644 index 000000000..a2a292e57 --- /dev/null +++ b/src/sources/overpass.js @@ -0,0 +1,48 @@ +/** + * Single-pass lexer: blank out quoted literals (→ empty quotes) and strip line + * and block comments — recognizing each in one walk so a comment marker INSIDE a + * quoted string is treated as string content, not a comment (and vice versa). + * Chained regex replaces get the ordering wrong (a quoted slash-slash would hide + * the rest of the line), which is exactly the bypass this avoids. + */ +export function stripOverpassNoise(src) { + let out = ''; + let i = 0; + const n = src.length; + while (i < n) { + const c = src[i]; + if (c === '"' || c === "'") { + const quote = c; + i += 1; + while (i < n) { + if (src[i] === '\\') { + i += 2; + continue; + } // escaped char + if (src[i] === quote) { + i += 1; + break; + } // closing quote + i += 1; + } + out += quote + quote; // collapse the literal to empty quotes + continue; + } + if (c === '/' && src[i + 1] === '*') { + i += 2; + while (i < n && !(src[i] === '*' && src[i + 1] === '/')) i += 1; + i += 2; + out += ' '; + continue; + } + if (c === '/' && src[i + 1] === '/') { + i += 2; + while (i < n && src[i] !== '\n') i += 1; + out += ' '; + continue; + } + out += c; + i += 1; + } + return out; +}