diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d6c42632..335a5b9b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ - Director assets cancelled during the final stream read are discarded with reader cleanup. +- Analyst queries preserve useful rows when an upstream timestamp is outside the supported date range. + +- Fire detections with missing or impossible positions are skipped while genuine zero coordinates remain valid. + - Frequently reused terrain batches retain their cached heights ahead of unused cells. - Recent Imagery’s exported `setBox` and `searchHls` reject missing, blank or diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index de4ca26ca..323bb104e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -75,6 +75,8 @@ Windows/Claude Desktop Cowork confirmation remains outstanding (#927). Tool argument string lengths follow JSON Schema Unicode code-point counts, so supplementary characters count once rather than as two UTF-16 units. +Query timestamps outside the representable Date range are reported as null, preserving the rest of the result row. + `gods-eye-view/tools` defines queries that answer questions from the app's data, and `gods-eye-view/tools/mcp` exposes a composed catalog over the Model Context Protocol. `npm run mcp` serves Core's tools over stdio to a local MCP client, @@ -1433,6 +1435,8 @@ accessible name remains "Don't show this again". Control behavior is unchanged. ## FIRMS source status +FIRMS ingestion and adaptation require explicit finite coordinates within WGS84 ranges; genuine zero coordinates remain valid and admitted detection indices stay sequential. + The FIRMS proxy records source success after appending its rows. If aggregation throws, that source reports failure without a contradictory success entry. The existing per-record append continues to support large feeds; sequential diff --git a/src/data/firms-coordinate-evidence.test.mjs b/src/data/firms-coordinate-evidence.test.mjs new file mode 100644 index 000000000..d60a7ee37 --- /dev/null +++ b/src/data/firms-coordinate-evidence.test.mjs @@ -0,0 +1,69 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseFirmsCsv } from './firmsCsv.js'; +import { adaptFirmsRecords } from './firmsAdapt.js'; +test('missing positions do not become fire detections at zero coordinates', () => { + const csv = + 'latitude,longitude,acq_date,acq_time,confidence,frp\n,1,2026-07-16,1200,n,1\n1, ,2026-07-16,1200,n,1\n95,1,2026-07-16,1200,n,1\n1,181,2026-07-16,1200,n,1\n0,0,2026-07-16,1200,n,1\n-90,-180,2026-07-16,1200,n,1'; + assert.deepEqual( + parseFirmsCsv(csv).map((r) => [r.lat, r.lon]), + [ + [0, 0], + [-90, -180], + ], + ); + const records = [ + { lat: null, lon: 1 }, + { lat: '', lon: 1 }, + { lat: false, lon: 1 }, + { lat: 95, lon: 1 }, + { lat: 1, lon: 181 }, + { lat: '0', lon: 0 }, + { lat: -90, lon: -180 }, + ]; + assert.deepEqual( + adaptFirmsRecords(records).map((r) => [r.index, r.lat, r.lon]), + [ + [0, 0, 0], + [1, -90, -180], + ], + ); +}); + +test('both axes keep their exact bounds and drop values just past them', () => { + const row = (lat, lon) => `${lat},${lon},2026-07-16,1200,n,1`; + const csv = [ + 'latitude,longitude,acq_date,acq_time,confidence,frp', + row(90, 180), + row(-90, -180), + row(90.000001, 0), + row(-90.000001, 0), + row(0, 180.000001), + row(0, -180.000001), + row(' ', 5), + row(5, '\t'), + ].join('\n'); + assert.deepEqual( + parseFirmsCsv(csv).map((r) => [r.lat, r.lon]), + [ + [90, 180], + [-90, -180], + ], + ); + const kept = adaptFirmsRecords([ + { lat: 90, lon: 180 }, + { lat: '-90', lon: '-180' }, + { lat: -90.000001, lon: 0 }, + { lat: 0, lon: -180.000001 }, + { lat: 5, lon: ' ' }, + { lat: 5, lon: undefined }, + { lat: [5], lon: 5 }, + ]); + assert.deepEqual( + kept.map((r) => [r.lat, r.lon]), + [ + [90, 180], + [-90, -180], + ], + ); +}); diff --git a/src/data/firmsAdapt.js b/src/data/firmsAdapt.js index 2fc909bce..312838e80 100644 --- a/src/data/firmsAdapt.js +++ b/src/data/firmsAdapt.js @@ -7,8 +7,8 @@ */ /** - * Map proxy fire records into internal fire records. Records with - * non-finite coordinates are skipped; `index` is the post-skip position + * Map proxy fire records into internal fire records. Records with missing, + * non-numeric or out-of-range (WGS84) coordinates are skipped; `index` is the post-skip position * (it keys pick ids and context-store ids, so it must stay sequential). * `contextEntity`/`position` start null and are lazily filled by the layer. * @param {?Array} records - /api/firms `fires` array. @@ -21,7 +21,15 @@ export function adaptFirmsRecords(records) { for (const record of records) { const lat = Number(record?.lat); const lon = Number(record?.lon); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + if ( + !hasCoordinateValue(record?.lat) || + !hasCoordinateValue(record?.lon) || + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + continue; fires.push({ index: fires.length, lat, @@ -110,3 +118,11 @@ function finiteNumber(value) { const number = Number(value); return Number.isFinite(number) ? number : 0; } + +// Same rule as parseFirmsCsv: a blank or non-numeric value is not a zero. +function hasCoordinateValue(value) { + return ( + typeof value === 'number' || + (typeof value === 'string' && value.trim() !== '') + ); +} diff --git a/src/data/firmsCsv.js b/src/data/firmsCsv.js index 7f9019e18..aafeac50b 100644 --- a/src/data/firmsCsv.js +++ b/src/data/firmsCsv.js @@ -105,7 +105,17 @@ export function parseFirmsCsv(text) { if (parts.length < header.length) continue; // malformed row — skip const lat = Number(parts[iLat]); const lon = Number(parts[iLon]); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + // Blank or out-of-range positions are dropped, never read as 0 (see + // hasCoordinateValue in firmsAdapt.js for the proxy-record side). + if ( + !parts[iLat].trim() || + !parts[iLon].trim() || + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + continue; records.push({ lat, diff --git a/src/tools/results.js b/src/tools/results.js index 51ed82bbb..f9af7af3d 100644 --- a/src/tools/results.js +++ b/src/tools/results.js @@ -24,7 +24,9 @@ export function capRows(rows, limit = DEFAULT_LIMIT) { /** Format an epoch millisecond time as ISO 8601, or null. */ export function isoTime(ms) { - return Number.isFinite(ms) ? new Date(ms).toISOString() : null; + if (!Number.isFinite(ms)) return null; + const date = new Date(ms); + return Number.isFinite(date.getTime()) ? date.toISOString() : null; } /** "1 earthquake", "3 earthquakes". */ diff --git a/src/tools/tool-timestamp-range.test.mjs b/src/tools/tool-timestamp-range.test.mjs new file mode 100644 index 000000000..7aeee737b --- /dev/null +++ b/src/tools/tool-timestamp-range.test.mjs @@ -0,0 +1,59 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { isoTime } from './results.js'; +import { getBikeShare } from './queries/mobility.js'; +test('out-of-range finite timestamps are unknown rather than throwing', async () => { + for (const value of [1e30, -1e30, NaN, Infinity, null]) + assert.equal(isoTime(value), null); + assert.equal(isoTime(0), '1970-01-01T00:00:00.000Z'); + assert.equal(isoTime(8640000000000000), '+275760-09-13T00:00:00.000Z'); + assert.equal(isoTime(-8640000000000000), '-271821-04-20T00:00:00.000Z'); + for (const value of [8640000000000001, -8640000000000001, -Infinity]) + assert.equal(isoTime(value), null); + assert.equal(isoTime(-1), '1969-12-31T23:59:59.999Z'); + const station = { + station_id: 's1', + name: 'Fixture', + lat: 0, + lon: 0, + capacity: 4, + }; + const result = await getBikeShare.run( + { area: { bbox: [-1, -1, 1, 1] } }, + { + services: { + bikeshare: { + systems: [ + { + id: 'fixture', + city: 'Fixture', + centerLat: 0, + centerLon: 0, + loadRadiusKm: 30, + stationInformationUrl: 'information', + stationStatusUrl: 'status', + }, + ], + async getStations(url) { + return { + data: { + stations: + url === 'information' + ? [station] + : [ + { + station_id: 's1', + last_reported: 1e30, + num_bikes_available: 2, + }, + ], + }, + }; + }, + }, + }, + }, + ); + assert.equal(result.data.returned, 1); + assert.equal(result.data.rows[0].last_reported, null); +}); diff --git a/src/voice/gevActions.js b/src/voice/gevActions.js index 9e1e183d0..267f633ff 100644 --- a/src/voice/gevActions.js +++ b/src/voice/gevActions.js @@ -4808,8 +4808,9 @@ function compactAnalystItem(r) { for (const [key, spec] of Object.entries(fields)) { if (!present(r[key]) || key in compact) continue; if (spec.type === 'time') { - const ms = Number(r[key]); - if (Number.isFinite(ms)) compact.time = new Date(ms).toISOString(); + // A finite value can still be past Date's range, where toISOString throws. + const time = new Date(Number(r[key])); + if (Number.isFinite(time.getTime())) compact.time = time.toISOString(); } else compact[key] = r[key]; } if (present(r.ageHours)) compact.ageHours = r.ageHours; diff --git a/src/voice/gevActions.test.mjs b/src/voice/gevActions.test.mjs index 3d0e35b3e..2a80b829c 100644 --- a/src/voice/gevActions.test.mjs +++ b/src/voice/gevActions.test.mjs @@ -3459,6 +3459,26 @@ test('analyst_query refusals carry their code and the allowed values', async () assert.match(ok.items[0].time, /^\d{4}-\d\d-\d\dT/); }); +test('analyst_query keeps its answer when one record has an out-of-range time', async () => { + globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null }; + const now = Date.now(); + const earthquakes = { + getStats: () => ({ count: 2, lastUpdate: now }), + getAnalystRecords: () => [ + { id: 'q', lat: 0, lon: 0, magnitude: 5, timeMs: now }, + { id: 'r', lat: 1, lon: 1, magnitude: 4, timeMs: 8.64e15 + 1 }, + ], + }; + const { runner } = manifestRunner({ earthquakes }, { enabled: ['earthquakes'] }); + const result = await runner('analyst_query', { layers: ['earthquakes'], scope: { kind: 'anywhere' } }); + assert.equal(result.ok, true); + const q = result.items.find((item) => item.id === 'q'); + assert.match(q.time, /^\d{4}-\d\d-\d\dT/); + const r = result.items.find((item) => item.id === 'r'); + assert.ok(r, 'the record is still listed'); + assert.equal(r.time, undefined, 'an unrepresentable time is omitted'); +}); + test('spoken layer names from the manifest reach the newer layers', async () => { globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null }; const stub = () => ({ getStats: () => ({ count: 0 }) });