Compare commits

..
Author SHA1 Message Date
Mazeyar Moeini FeizabadiandClaude Fable 5.2 829d1c1f7e docs(changelog): add the #1711 fragment
Co-Authored-By: Claude Fable 5.2 <[email protected]>
2026-11-13 16:06:03 +01:00
Mazeyar Moeini FeizabadiandClaude Fable 5.2 e79d972389 perf(firms): serve a cached body and gzip instead of serialising per request
/api/firms ran JSON.stringify over the whole fire set for every request and
sent it uncompressed: about 20 MB per 100,000 detections.

The handler now keeps the last body as bytes and reuses it for the same
cache entry and stale flag while the clock stays inside the span over which
the trailing-24 h filter keeps the same detections, so the JSON a request
gets is byte-identical to a fresh serialisation. A client that accepts gzip
gets the body gzipped, compressed once per kept body.

The span ends when the oldest kept acquisition minute turns 24 h old, so
requests minutes apart are still serialised afresh.

Co-Authored-By: Claude Fable 5.2 <[email protected]>
2026-11-13 14:43:14 +01:00
15 changed files with 245 additions and 643 deletions

No files matched your search

-4
View File
@@ -1,4 +0,0 @@
- Cut the stutter while zooming and panning with Street Traffic on in dense
cities. Coarse roads are no longer clipped against every detail tile in one
task, roads already parsed in a load are not parsed again, and ground floors
are read once per cell (mazy1998, #1705).
+2
View File
@@ -0,0 +1,2 @@
- `/api/firms` keeps its response bytes and sends them gzipped instead of
serialising the whole fire set for every request (mazy1998, #1711).
+9
View File
@@ -3297,6 +3297,15 @@ The `get_active_fires` tool reports acquisition times as UTC ISO timestamps,
including zero-padded hours and minutes; missing or invalid dates and times
remain `null`, including impossible calendar dates from a custom source.
`/api/firms` keeps its last response body as bytes and reuses it for requests
that would get the same payload: the same cache entry and `stale` flag, while
the clock stays inside the span over which the trailing-24-hour filter keeps
the same detections. That span ends when the oldest kept acquisition minute
turns 24 hours old, so requests minutes apart are still serialised afresh. A
client that sends `Accept-Encoding: gzip` gets the body gzipped (compressed
once per kept body, `Content-Encoding: gzip`); every response carries
`Vary: Accept-Encoding`. The JSON is unchanged.
The FIRMS proxy records source success after appending its rows. If aggregation
throws, that source reports failure without a contradictory success entry.
The existing per-record append continues to support large feeds; sequential
+1
View File
@@ -376,6 +376,7 @@
"modules": [
"server/providers/firms.js",
"server/providers/common/http.js",
"server/providers/common/compress.js",
"src/sources/httpBody.js",
"src/i18n/marker.js",
"src/data/firmsCsv.js"
+58 -10
View File
@@ -3,7 +3,12 @@ import https from 'node:https';
import { promises as fsp } from 'node:fs';
import { Readable } from 'node:stream';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import {
filterTrailing24h,
parseFirmsCsv,
trailing24hWindow,
} from '../../src/data/firmsCsv.js';
import { acceptsGzip, encodedBody, gzipBody } from './common/compress.js';
import { readResponseTextCapped } from './common/http.js';
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
@@ -179,6 +184,11 @@ export function firmsProxy() {
let statusCache = null;
/** @type {?Promise<?{used: number, limit: number}>} */
let statusInflight = null;
/**
* The last /api/firms payload as bytes, with its gzip. One slot.
* @type {?{entry: object, stale: boolean, from: number, until: number, body: Buffer, gzip: ?Promise<?Buffer>}}
*/
let served = null;
const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim();
@@ -259,12 +269,8 @@ export function firmsProxy() {
return { at: now, sources, fires };
}
/**
* Cache entry → response payload. Fires are RE-filtered to the trailing
* 24 h at serve time so a stale cache never serves >24h-old detections.
*/
function buildPayload(entry, stale) {
const fires = filterTrailing24h(entry.fires, Date.now());
/** Cache entry and the fires to serve from it → response payload. */
function buildPayload(entry, stale, fires) {
return {
fetchedAt: entry.at,
stale,
@@ -275,6 +281,32 @@ export function firmsProxy() {
};
}
/**
* The payload as the bytes to send. Serialising the whole fire set is tens
* of MB in a heavy season, so it is done once and kept, not once per request.
*
* Fires are RE-filtered to the trailing 24 h at serve time so a stale cache
* never serves >24h-old detections. The kept bytes are therefore reused only
* for the same entry and stale flag while the clock is inside the span over
* which that filter keeps the same records: a request never gets a body that
* serialising for it would not have produced.
*/
function payloadBody(entry, stale) {
const now = Date.now();
if (
served?.entry === entry &&
served.stale === stale &&
now >= served.from &&
now <= served.until
) {
return served;
}
const { kept, from, until } = trailing24hWindow(entry.fires, now);
const body = Buffer.from(JSON.stringify(buildPayload(entry, stale, kept)));
served = { entry, stale, from, until, body, gzip: null };
return served;
}
/** mapkey_status transactions, cached 5 min, best-effort (null on failure). */
function getTransactions(key) {
const now = Date.now();
@@ -327,6 +359,22 @@ export function firmsProxy() {
});
res.end(JSON.stringify(obj));
};
const sendPayload = async (entry, stale) => {
const kept = payloadBody(entry, stale);
// Compressed once per kept body, on the first request that accepts
// gzip, so an identity response never pays or waits for it.
const gzip = acceptsGzip(req)
? await (kept.gzip ??= gzipBody(kept.body).catch(() => null))
: null;
const encoded = encodedBody(req, kept.body, gzip);
if (res.headersSent) return;
res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...encoded.headers,
});
res.end(encoded.body);
};
try {
const subPath = String(req.url || '').split('?')[0];
const key = mapKey();
@@ -363,7 +411,7 @@ export function firmsProxy() {
const entry = mem;
if (entry && Date.now() - entry.at < TTL_MS) {
sendJson(200, buildPayload(entry, false));
await sendPayload(entry, false);
return;
}
// Stale or missing → refresh, single-flight (concurrent requests
@@ -389,9 +437,9 @@ export function firmsProxy() {
const pending = inflight;
const fresh = await pending;
if (fresh) {
sendJson(200, buildPayload(fresh, false));
await sendPayload(fresh, false);
} else if (entry) {
sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty
await sendPayload(entry, true); // upstream down — stale beats empty
} else {
sendJson(502, {
error: 'firms fetch failed and no cache available',
+27 -2
View File
@@ -182,19 +182,44 @@ export function acquisitionMsUtc(acqDate, acqTime) {
* @returns {Array<Object>} Filtered records (original objects, order preserved).
*/
export function filterTrailing24h(records, nowMs) {
if (!Array.isArray(records) || !Number.isFinite(nowMs)) return [];
return trailing24hWindow(records, nowMs).kept;
}
/**
* {@link filterTrailing24h} with the span of `nowMs` values, inclusive, over
* which it keeps exactly these records. A caller holding something built from
* `kept` can reuse it while its clock stays inside `from..until`.
* @param {Array<{acqDate: string, acqTime: string|number}>} records - Parser records.
* @param {number} nowMs - Reference epoch milliseconds.
* @returns {{kept: Array<Object>, from: number, until: number}} Span in epoch
* ms; NaN (never reusable) for unusable input.
*/
export function trailing24hWindow(records, nowMs) {
if (!Array.isArray(records) || !Number.isFinite(nowMs))
return { kept: [], from: NaN, until: NaN };
const oldest = nowMs - WINDOW_MS;
const newest = nowMs + FORWARD_SLACK_MS;
let from = -Infinity;
let until = Infinity;
const memo = new Map();
return records.filter((record) => {
const kept = records.filter((record) => {
const key = `${record.acqDate}:${record.acqTime}`;
let ms = memo.get(key);
if (ms === undefined) {
ms = acquisitionMsUtc(record.acqDate, record.acqTime);
memo.set(key, ms);
// Each distinct time narrows the span to the clock values that leave
// it on the same side of the window.
if (ms < oldest) from = Math.max(from, ms + WINDOW_MS + 1);
else if (ms > newest) until = Math.min(until, ms - FORWARD_SLACK_MS - 1);
else if (Number.isFinite(ms)) {
from = Math.max(from, ms - FORWARD_SLACK_MS);
until = Math.min(until, ms + WINDOW_MS);
}
}
return Number.isFinite(ms) && ms >= oldest && ms <= newest;
});
return { kept, from, until };
}
/** Trimmed string cell at index, '' for missing columns. */
+121 -1
View File
@@ -7,11 +7,15 @@ import dnsLookup from 'node:dns';
import https from 'node:https';
import net from 'node:net';
import { Readable } from 'node:stream';
import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js';
import os from 'node:os';
import path from 'node:path';
import { gunzipSync } from 'node:zlib';
import { filterTrailing24h, parseFirmsCsv, trailing24hWindow } from './firmsCsv.js';
import {
describeFetchError,
fetchFirms,
fetchIpv4,
firmsProxy,
isAddressSelectionFailure,
} from '../../server/providers/firms.js';
@@ -316,3 +320,119 @@ test('FIRMS requests only the satellites NASA still delivers', () => {
// a MAP_KEY transaction.
assert.ok(!SOURCES.includes('VIIRS_SNPP_NRT'));
});
// The /api/firms handler itself, over a disk cache seeded in a temp directory:
// no server, no upstream, and a placeholder MAP_KEY that is never sent anywhere.
const TTL_MS = 30 * 60_000;
const fire = (acqDate, acqTime, lat = 1) => ({ lat, lon: 2, frp: 4.5, confidence: 'n', brightness: 330.1,
brightnessTi5: 290.2, daynight: 'D', acqDate, acqTime, satellite: 'N20', instrument: 'VIIRS' });
const SERVED_SOURCES = SOURCES.map((source, index) => ({ source, count: index === 0 ? 20_001 : 0, ok: true }));
// 20,000 detections an hour old and one that turns 24 h old five minutes after NOW.
const SERVED_FIRES = [fire('2026-09-10', '1205'), ...Array.from({ length: 20_000 }, (_, i) => fire('2026-09-11', '1100', i / 1000))];
const LEAVES_AT = NOW + 5 * 60_000;
/** What the handler sent before it kept a body: one JSON.stringify of the payload per request. */
const unfixedBody = (at, stale, now) => {
const fires = filterTrailing24h(SERVED_FIRES, now);
return JSON.stringify({ fetchedAt: at, stale, ttlMs: TTL_MS, sources: SERVED_SOURCES, count: fires.length, fires });
};
function createFirmsHandler(t, at) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'firms-proxy-'));
fs.mkdirSync(path.join(dir, '.gev-cache'));
fs.writeFileSync(path.join(dir, '.gev-cache', 'firms.json'), JSON.stringify({ at, sources: SERVED_SOURCES, fires: SERVED_FIRES }));
const cwd = process.cwd();
const key = process.env.FIRMS_MAP_KEY;
const realFetch = globalThis.fetch;
process.env.FIRMS_MAP_KEY = 'unit-test-placeholder';
// A refresh must never reach NASA from a test: every upstream call fails.
globalThis.fetch = async () => { throw new Error('upstream down'); };
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['Date'], now: NOW });
t.after(() => {
globalThis.fetch = realFetch;
if (key === undefined) delete process.env.FIRMS_MAP_KEY; else process.env.FIRMS_MAP_KEY = key;
fs.rmSync(dir, { recursive: true, force: true });
});
let handler;
process.chdir(dir); // the proxy resolves its cache directory when created
try {
firmsProxy().configureServer({ middlewares: { use: (route, fn) => { handler = fn; } } });
} finally {
process.chdir(cwd);
}
return (acceptEncoding) => new Promise((resolve, reject) => {
const res = {
headersSent: false,
writeHead(status, headers) { this.headersSent = true; this.status = status; this.headers = headers; },
end(body) { resolve({ status: this.status, headers: this.headers, body }); },
};
handler({ url: '/', headers: acceptEncoding === undefined ? {} : { 'accept-encoding': acceptEncoding } }, res).catch(reject);
});
}
test('/api/firms serialises once for requests that share a payload', async (t) => {
const request = createFirmsHandler(t, NOW);
const first = await request();
t.mock.timers.setTime(LEAVES_AT); // exactly 24 h old is still inside the window
const second = await request();
assert.equal(first.status, 200);
assert.ok(Buffer.isBuffer(first.body), 'the body is kept as bytes');
assert.equal(second.body, first.body, 'the second request sends the kept bytes, not a new serialisation');
assert.equal(first.body.toString(), unfixedBody(NOW, false, NOW), 'byte-identical to the per-request serialisation');
assert.equal(JSON.parse(first.body).count, 20_001);
});
test('/api/firms sends gzip only to a client that accepts it', async (t) => {
const request = createFirmsHandler(t, NOW);
const identity = await request();
const gzipped = await request('gzip, deflate, br');
const refused = await request('gzip;q=0, br');
assert.deepEqual(identity.headers,
{ 'Content-Type': 'application/json', 'Cache-Control': 'no-store', Vary: 'Accept-Encoding' });
assert.deepEqual(gzipped.headers, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store',
'Content-Encoding': 'gzip', Vary: 'Accept-Encoding' });
assert.deepEqual(refused.headers, identity.headers);
assert.equal(refused.body, identity.body);
assert.ok(gzipped.body.byteLength < identity.body.byteLength / 4, 'the fire set compresses');
assert.equal(gunzipSync(gzipped.body).toString(), unfixedBody(NOW, false, NOW), 'decodes to the identity bytes');
assert.equal((await request('gzip')).body, gzipped.body, 'compressed once');
});
test('/api/firms never reuses a body the request would not have produced', async (t) => {
const request = createFirmsHandler(t, NOW - TTL_MS + 6 * 60_000);
const at = NOW - TTL_MS + 6 * 60_000;
const before = await request();
t.mock.timers.setTime(LEAVES_AT + 1); // one detection is now older than 24 h
const after = await request('gzip');
assert.equal(before.body.toString(), unfixedBody(at, false, NOW));
assert.equal(gunzipSync(after.body).toString(), unfixedBody(at, false, LEAVES_AT + 1));
assert.equal(JSON.parse(gunzipSync(after.body)).count, 20_000);
t.mock.timers.setTime(NOW + 6 * 60_000); // the entry reaches its TTL and the refresh fails
const stale = await request();
assert.equal(stale.body.toString(), unfixedBody(at, true, NOW + 6 * 60_000));
assert.equal((await request()).body, stale.body, 'the stale body is kept too');
});
test('trailing24hWindow spans the clock values that keep the same records', () => {
const records = [fire('2026-09-10', '1205'), fire('2026-09-10', '1300'), fire('2026-09-11', '1100'),
fire('2026-09-11', '1430'), fire('2026-09-11', '1600'), fire('bad-date', '1006')];
const edges = records.map((r) => Date.parse(`${r.acqDate}T${r.acqTime.slice(0, 2)}:${r.acqTime.slice(2)}Z`))
.filter(Number.isFinite).flatMap((ms) => [ms - 2 * 3600_000, ms + 24 * 3600_000]);
assert.equal(edges.length, 10);
const clocks = [NOW, ...edges.flatMap((ms) => [ms - 1, ms, ms + 1])];
for (const now of clocks) {
const { kept, from, until } = trailing24hWindow(records, now);
assert.deepEqual(kept, filterTrailing24h(records, now));
assert.ok(from <= now && now <= until, 'the span holds its own clock');
for (const other of clocks) {
const then = filterTrailing24h(records, other);
const same = then.length === kept.length && then.every((r, i) => r === kept[i]);
// Before the first record and after the last both keep nothing; the span covers one side.
const bothEmpty = then.length === 0 && kept.length === 0;
if (other >= from && other <= until) assert.ok(same, `${other} is inside the span of ${now}`);
else assert.ok(!same || bothEmpty, `${other} is outside the span of ${now}`);
}
}
assert.deepEqual(trailing24hWindow([], NOW), { kept: [], from: -Infinity, until: Infinity });
assert.deepEqual(trailing24hWindow(records, NaN), { kept: [], from: NaN, until: NaN });
});
+1 -3
View File
@@ -1,7 +1,6 @@
import { roadIdentity } from './retention.js';
import * as Cesium from 'cesium';
import {
floorCellReader,
prepareRoadSurfaces,
roadHeightStations,
trafficSurfaceReady,
@@ -302,7 +301,6 @@ export function createFlow({ state: layerState, services, parts, source }) {
}
}
if (depthHeight !== undefined) layerState._viewHeightEstimate = depthHeight;
const floorAt = floorCellReader(services.ground);
for (const road of roads) {
road.key ||= roadIdentity(road);
const prepared = !refining && preparedRoads.get(road.key);
@@ -321,7 +319,7 @@ export function createFlow({ state: layerState, services, parts, source }) {
);
for (let i = 0; i < road.coords.length; i++) {
const [lon, lat] = road.coords[i];
const floor = floorAt(lat, lon);
const floor = services.ground?.cachedGroundFloor?.(lat, lon);
const height =
Number.isFinite(floor) && Math.abs(floor) <= 9000
? Math.max(floor, layerState._viewHeightEstimate)
+3 -3
View File
@@ -39,10 +39,10 @@ export function createTrafficLayer({ services, source }) {
parts.controls = createControls(context);
parts.lifecycle = createLifecycle(context);
state._parseRoads = TRAFFIC_TIMING_ENABLED
? (data, trace, parsedBySource) =>
? (data, trace) =>
trace
? parts.timing.parseRoadsTimed(data, trace, parsedBySource)
: parts.model.parseRoads(data, trace, parsedBySource)
? parts.timing.parseRoadsTimed(data, trace)
: parts.model.parseRoads(data)
: parts.model.parseRoads;
state._loadRoadsForBounds = TRAFFIC_TIMING_ENABLED
+1 -7
View File
@@ -348,16 +348,11 @@ export function createIngestion({
layerState._retainedPreview = false;
layerState._detailRoadsReady = false;
let paintRevision = 0;
// The snapshot that ends a pass replaces the streamed roads with the
// composed set, which is mostly the tile records already parsed and
// drawn. Remembering those per pass means only the records new to that
// snapshot (the clipped coarse roads) are parsed, not the whole set again.
let parsedBySource = new WeakMap();
const onTile = (data, passSignal, preview) => {
if (generation !== layerState._loadGeneration || passSignal.aborted)
return;
const start = performance.now();
const parsed = layerState._parseRoads(data, trace, parsedBySource);
const parsed = layerState._parseRoads(data, trace);
phaseTiming('parse', start, { roads: parsed.length, incremental: true });
if (data.replace) streamed = parsed;
else streamed.push(...parsed);
@@ -536,7 +531,6 @@ export function createIngestion({
if (altitude > FAST_FETCH_ALTITUDE) return;
streamed = [];
parsedBySource = new WeakMap();
majorPass = false;
// Detailed pass: fetch the full road graph (tertiary, residential, etc.)
console.log(`[Data:Traffic] Full fetch local roads [${cacheKey}]`);
+4 -18
View File
@@ -1,5 +1,5 @@
import * as Cesium from 'cesium';
import { floorCellReader, roadSurfaceChunks } from './surface.js';
import { roadSurfaceChunks } from './surface.js';
import { t, translateKnown } from '../../i18n/index.js';
import {
flowDensityMult,
@@ -13,35 +13,22 @@ import {
} from './policy.js';
export function createModel({ state: layerState, services, parts, source }) {
/**
* Build scene waypoints from source records; thinning and terrain remain rendering policy.
*
* `parsedBySource` (source record → its parsed roads) is filled as records
* are parsed, and a record already in it is answered from it.
*/
function parseRoads(roadData, _trace, parsedBySource) {
/** Build scene waypoints from source records; thinning and terrain remain rendering policy. */
function parseRoads(roadData) {
if (!roadData || !roadData.roads) {
return [];
}
const roads = [];
const floorAt = floorCellReader(services.ground);
for (const road of roadData.roads) {
if (!road.coordinates || road.coordinates.length < 2) continue;
// A replacing snapshot repeats records this load already parsed.
const known = parsedBySource?.get(road);
if (known) {
for (const parsed of known) roads.push(parsed);
continue;
}
const first = roads.length;
for (const coords of roadSurfaceChunks(road.coordinates)) {
const type = road.type;
const oneway = road.oneway;
// Pre-compute Cartesian3 waypoints (lon, lat, height) for fast lerp animation
const waypoints = coords.map(([lng, lat]) => {
const floor = floorAt(lat, lng);
const floor = services.ground?.cachedGroundFloor?.(lat, lng);
const h =
(Number.isFinite(floor) && Math.abs(floor) <= 9000
? floor
@@ -73,7 +60,6 @@ export function createModel({ state: layerState, services, parts, source }) {
flow: road.flow || null,
});
}
parsedBySource?.set(road, roads.slice(first));
}
return roads;
-444
View File
@@ -1,444 +0,0 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import * as Cesium from 'cesium';
import { createTrafficSource, outsideDetailTiles } from './source.js';
import { createIngestion } from './ingestion.js';
import { createModel } from './model.js';
import { floorCellReader, roadSurfaceChunks } from './surface.js';
import { DOT_HEIGHT_OFFSET } from './policy.js';
import { clipTileLine } from '../../sources/openFreeMap.js';
import { tileToBBox } from '../../data/tomtomTiles.js';
// The clip as it was before the extent shortcut: every road against the four
// regions around every tile. The shortcut must return exactly this.
function clipEveryRoadAgainstEveryTile(roads, tiles) {
return roads.flatMap((road) => {
let lines = [road.coordinates];
for (const tile of tiles) {
const box = tileToBBox(tile.z, tile.x, tile.y);
const regions = [
{ west: -180, east: box.west, south: -85, north: 85 },
{ west: box.east, east: 180, south: -85, north: 85 },
{ west: box.west, east: box.east, south: -85, north: box.south },
{ west: box.west, east: box.east, south: box.north, north: 85 },
];
lines = lines.flatMap((line) =>
regions.flatMap((region) => clipTileLine(line, region)),
);
}
return lines.map((coordinates) => ({ ...road, coordinates }));
});
}
function seeded(seed) {
let state = seed >>> 0;
return () => {
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
return state / 2 ** 32;
};
}
function tileBlock(x, y, size = 4) {
const tiles = [];
for (let dx = 0; dx < size; dx++)
for (let dy = 0; dy < size; dy++)
tiles.push({ z: 14, x: x + dx, y: y + dy });
return tiles;
}
/** Roads in and around a tile block: wandering, straight, tiny, and on tile edges. */
function roadsAround(tiles, random, count = 400) {
const boxes = tiles.map((tile) => tileToBBox(tile.z, tile.x, tile.y));
const west = Math.min(...boxes.map((box) => box.west)),
east = Math.max(...boxes.map((box) => box.east)),
south = Math.min(...boxes.map((box) => box.south)),
north = Math.max(...boxes.map((box) => box.north));
const width = east - west,
height = north - south;
const roads = [];
for (let i = 0; i < count; i++) {
const step = i % 7 === 0 ? 0.00002 : i % 3 === 0 ? 0.01 : 0.002;
let lon = west - width / 2 + random() * width * 2,
lat = south - height / 2 + random() * height * 2;
const coordinates = [[lon, lat]];
const points = 1 + Math.floor(random() * 12);
for (let j = 0; j < points; j++) {
lon += (random() - 0.5) * step;
lat += (random() - 0.5) * step;
coordinates.push([lon, lat]);
}
roads.push({ type: 'primary', oneway: i % 2, id: i, coordinates });
}
const box = boxes[5];
const midLat = (box.south + box.north) / 2,
midLon = (box.west + box.east) / 2;
roads.push(
// Along a tile edge, ending on a corner, and touching an edge from outside.
{
id: 'edge',
coordinates: [
[box.west, box.south - 0.01],
[box.west, box.north + 0.01],
],
},
{
id: 'corner',
coordinates: [
[box.west - 0.01, box.south - 0.01],
[box.west, box.south],
[box.east, box.north],
],
},
{
id: 'touch',
coordinates: [
[box.west - 0.01, midLat],
[box.west, midLat],
],
},
// Across the whole block, and a repeated vertex.
{
id: 'across',
coordinates: [
[west - 0.01, midLat],
[east + 0.01, midLat + 0.0003],
],
},
{
id: 'repeat',
coordinates: [
[midLon, midLat],
[midLon, midLat],
[midLon + 0.001, midLat],
],
},
// Records the clip drops or leaves alone.
{ id: 'single', coordinates: [[west - 0.02, midLat]] },
{ id: 'empty', coordinates: [] },
{
id: 'negative-zero',
coordinates: [
[-0, south - 0.02],
[0.001, south - 0.02],
],
},
{
id: 'polar',
coordinates: [
[west - 0.02, 86],
[west - 0.02, 86.01],
],
},
);
return roads;
}
for (const [place, x, y] of [
['Manhattan', 4823, 6158],
['London, across longitude 0', 8190, 5446],
['longitude 0 and latitude 0', 8190, 8190],
['the southern and eastern hemispheres', 15070, 9826],
])
test(`coarse roads clipped around detail tiles match the every-tile clip: ${place}`, () => {
const tiles = tileBlock(x, y);
const roads = roadsAround(tiles, seeded(x * 31 + y));
for (const loaded of [tiles, tiles.slice(0, 5), tiles.slice(3, 4), []]) {
const expected = clipEveryRoadAgainstEveryTile(roads, loaded);
assert.ok(loaded.length === 0 || expected.length !== roads.length);
// deepStrictEqual separates -0 from 0 and compares every vertex exactly.
assert.deepStrictEqual(outsideDetailTiles(roads, loaded), expected);
}
});
test('only a road that reaches a detail tile is clipped against it', () => {
const tiles = tileBlock(4823, 6158);
const block = tiles.map((tile) => tileToBBox(tile.z, tile.x, tile.y));
const west = Math.min(...block.map((box) => box.west)),
north = Math.max(...block.map((box) => box.north));
// 300 roads beside the block and between its tile edges, none touching one.
const clear = [];
for (let i = 0; i < 300; i++) {
const box = block[i % block.length];
const lon = i % 2 ? west - 0.02 - i * 1e-5 : box.west + 0.004;
const lat = i % 2 ? box.south + 0.004 : north + 0.01 + i * 1e-5;
clear.push({
coordinates: [
[lon, lat],
[lon + 0.002, lat + 0.0005],
[lon + 0.003, lat + 0.001],
],
});
}
const first = block[0];
const crossing = {
coordinates: [
[first.west + 0.004, north + 0.01],
[first.west + 0.004, (first.south + first.north) / 2],
],
};
let clips = 0;
const counted = (line, region) => {
clips++;
return clipTileLine(line, region);
};
const roads = outsideDetailTiles([...clear, crossing], tiles, counted);
assert.deepStrictEqual(
roads,
clipEveryRoadAgainstEveryTile([...clear, crossing], tiles),
);
assert.equal(roads.length, 301);
// Four regions for each tile the crossing road reaches, never the 19,264
// clips of every road against every tile.
assert.ok(clips >= 4, `expected the crossing road to be clipped: ${clips}`);
assert.ok(clips <= 4 * tiles.length, `clipped ${clips} times`);
});
/** A DEM-regime ground service: one floor per toFixed(3) cell, counting reads. */
function cellGround({ mesh = false, warm = () => true } = {}) {
const ground = {
reads: 0,
meshFloorPreferred: () => mesh,
cachedGroundFloor(lat, lon) {
ground.reads++;
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null;
// Number() as the shared cache does it: "-0.000" and "0.000" are one cell.
const cell = `${Number(lat.toFixed(3))},${Number(lon.toFixed(3))}`;
if (!warm(cell)) return null;
let hash = 0;
for (const character of cell)
hash = (Math.imul(hash, 31) + character.charCodeAt(0)) | 0;
// The mesh regime answers per point, not per cell.
return (hash % 4000) / 10 + (mesh ? lat - Math.floor(lat * 1000) / 1000 : 0);
},
};
return ground;
}
test('floor reads are shared per cell and equal the per-coordinate reads', () => {
const random = seeded(7);
const points = [];
for (let i = 0; i < 4000; i++)
points.push([40.75 + random() * 0.01, -73.99 + random() * 0.01]);
// Cell edges, where toFixed(3) and Math.round can disagree, in each sign.
for (const lat of [40.7505, -40.7505, 0.0005, -0.0005, 1.0005, 8.5745])
for (const lon of [-73.9905, 73.9905, 0.0005, -0.0005, 1.0005, 2.3515])
points.push([lat, lon], [lat + 1e-9, lon - 1e-9], [lat - 1e-9, lon]);
points.push([Number.NaN, 0], [0, Infinity], [0, 0], [-0, -0]);
const direct = cellGround();
const expected = points.map(([lat, lon]) =>
direct.cachedGroundFloor(lat, lon),
);
const ground = cellGround();
const floorAt = floorCellReader(ground);
assert.deepStrictEqual(
points.map(([lat, lon]) => floorAt(lat, lon)),
expected,
);
// 4000 points in a 0.01 degree square share about 121 cells.
assert.ok(ground.reads < 600, `read the shared cache ${ground.reads} times`);
// A cold cell is remembered as cold for the pass, not asked again.
const cold = cellGround({ warm: () => false });
const coldAt = floorCellReader(cold);
assert.equal(coldAt(40.7512, -73.9812), null);
assert.equal(coldAt(40.7513, -73.9813), null);
assert.equal(cold.reads, 1);
});
test('mesh floors and unknown ground services are read per coordinate', () => {
const mesh = cellGround({ mesh: true });
const meshAt = floorCellReader(mesh);
const a = meshAt(40.7512, -73.9812),
b = meshAt(40.75121, -73.9812);
assert.equal(mesh.reads, 2);
assert.notEqual(a, b);
// No regime to ask: nothing says a floor is the same across a cell.
let reads = 0;
const bareAt = floorCellReader({ cachedGroundFloor: () => ++reads });
bareAt(40.7512, -73.9812);
bareAt(40.7512, -73.9812);
assert.equal(reads, 2);
assert.equal(floorCellReader(undefined)(1, 2), undefined);
assert.equal(floorCellReader({})(1, 2), undefined);
});
function road(lon, lat, points = 40) {
const coordinates = [];
for (let i = 0; i < points; i++)
coordinates.push([lon + i * 0.00004, lat + i * 0.00003]);
return { type: 'residential', oneway: 1, coordinates };
}
test('parsing reads each floor cell once and builds the same waypoints', () => {
const ground = cellGround({ warm: (cell) => !cell.endsWith('3') });
const state = { _viewHeightEstimate: 12 };
const model = createModel({
state,
services: { ground },
parts: {},
source: {},
});
const records = [road(-73.99, 40.75), road(-73.9899, 40.7501)];
const parsed = model.parseRoads({ roads: records });
const coordinates = records.reduce(
(count, record) => count + record.coordinates.length,
0,
);
assert.ok(
ground.reads < coordinates / 3,
`${ground.reads} floor reads for ${coordinates} coordinates`,
);
const direct = cellGround({ warm: (cell) => !cell.endsWith('3') });
const expected = records.flatMap((record) =>
roadSurfaceChunks(record.coordinates).map((coords) =>
coords.map(([lon, lat]) => {
const floor = direct.cachedGroundFloor(lat, lon);
return Cesium.Cartesian3.fromDegrees(
lon,
lat,
(floor ?? state._viewHeightEstimate) + DOT_HEIGHT_OFFSET,
);
}),
),
);
assert.ok(direct.reads >= coordinates);
assert.deepStrictEqual(
parsed.map((entry) => entry.waypoints),
expected,
);
});
test('a record parsed earlier in the pass is not parsed again', () => {
const ground = cellGround();
const model = createModel({
state: {},
services: { ground },
parts: {},
source: {},
});
const first = { ...road(-73.99, 40.75), oneway: 0 },
second = road(-73.98, 40.76),
short = { type: 'primary', coordinates: [[-73.97, 40.77]] };
const parsedBySource = new WeakMap();
const tile = model.parseRoads({ roads: [first] }, null, parsedBySource);
assert.equal(tile.length, 2, 'a two-way record parses to both directions');
const reads = ground.reads;
const replaced = model.parseRoads(
{ roads: [second, short, first], replace: true },
null,
parsedBySource,
);
assert.equal(replaced.length, 3);
// The same parsed roads, in the snapshot's order: dots stay attached.
assert.equal(replaced[1], tile[0]);
assert.equal(replaced[2], tile[1]);
const fresh = cellGround();
createModel({
state: {},
services: { ground: fresh },
parts: {},
source: {},
}).parseRoads({ roads: [second] });
assert.equal(ground.reads - reads, fresh.reads, 'only the new record read');
// Without the map every call parses, as before.
const again = model.parseRoads({ roads: [first] });
assert.notEqual(again[0], tile[0]);
assert.deepStrictEqual(again, tile);
});
test('the snapshot that ends a pass parses only the records it adds', async () => {
const detail = { z: 14, x: 3743, y: 6745 };
const core = tileToBBox(detail.z, detail.x, detail.y);
const local = {
type: 'residential',
oneway: 1,
coordinates: [
[core.west + 0.002, core.south + 0.002],
[core.west + 0.004, core.south + 0.004],
],
};
const distant = {
type: 'primary',
oneway: 1,
coordinates: [
[core.east + 0.01, core.south],
[core.east + 0.02, core.south],
],
};
const bounds = {
...core,
east: core.east + 0.03,
coverage: {
detail: [detail],
coarse: [{ z: 12, x: 935, y: 1686 }],
key: 'test',
},
};
const source = createTrafficSource({
mapTiles: {
async fetchBounds(box, { zoom, onTile }) {
const tile = { roads: zoom === 14 ? [local] : [distant] };
onTile?.(tile);
return {
tiles: [tile],
loadedTiles: zoom === 14 ? [detail] : bounds.coverage.coarse,
partial: false,
};
},
},
});
const parsedRecords = [];
const painted = [];
const state = {
_enabled: true,
_loadGeneration: 0,
_tileCache: new Map(),
_roadMode: null,
_liveMode: false,
// The model's contract: fill the map, answer known records from it.
_parseRoads: (data, trace, parsedBySource) =>
data.roads.flatMap((record) => {
const known = parsedBySource?.get(record);
if (known) return known;
parsedRecords.push(record);
const parsed = [{ record }];
parsedBySource?.set(record, parsed);
return parsed;
}),
};
const ingestion = createIngestion({
state,
source,
services: {},
parts: {
viewport: {
clampBounds: (b) => b,
getBoundsCenter: () => ({ lat: 30.26, lon: -97.74 }),
},
flow: {
warmFlow: () => Promise.resolve([]),
applyFlowThenRender: async (roads, box, generation, altitude, label) => {
if (label === 'Loaded full') painted.push(roads);
return true;
},
},
},
});
await ingestion.loadRoadsForBounds(bounds, 350);
assert.equal(
parsedRecords.filter((record) => record === local).length,
1,
'the detail tile record is parsed when its tile streams, not again',
);
// The full graph is still the detail record plus the coarse road outside it.
assert.equal(painted.at(-1).length, 2);
assert.equal(painted.at(-1)[0].record, local);
assert.deepStrictEqual(
painted.at(-1)[1].record.coordinates,
distant.coordinates,
);
});
+15 -100
View File
@@ -33,108 +33,23 @@ export function trafficDetailBounds(box) {
return detail;
}
/** Longitude/latitude extent of one line; NaN coordinates leave it unusable. */
function lineExtent(coordinates) {
let west = Infinity,
east = -Infinity,
south = Infinity,
north = -Infinity;
for (const [lon, lat] of coordinates) {
if (!(lon >= west)) west = lon;
if (!(lon <= east)) east = lon;
if (!(lat >= south)) south = lat;
if (!(lat <= north)) north = lat;
}
return { coordinates, west, east, south, north, settled: false };
}
/**
* What `clipTileLine` returns for a line lying wholly inside the clip region:
* the same line with each vertex recomputed from the one before it. Returns
* null for a line under the 12 m floor `clipTileLine` drops.
*/
function unclippedLine(coordinates) {
if (coordinates.length < 2) return null;
const out = [];
let metres = 0;
for (let i = 1; i < coordinates.length; i++) {
const a = coordinates[i - 1],
b = coordinates[i];
const dx = b[0] - a[0],
dy = b[1] - a[1];
if (i === 1) out.push([a[0] + 0 * dx, a[1] + 0 * dy]);
out.push([a[0] + dx, a[1] + dy]);
metres +=
Math.hypot(
(out[i][0] - out[i - 1][0]) * Math.cos((out[i][1] * Math.PI) / 180),
out[i][1] - out[i - 1][1],
) * 111320;
}
return metres >= 12 ? out : null;
}
const sameLine = (a, b) =>
a.every(
(point, i) => Object.is(point[0], b[i][0]) && Object.is(point[1], b[i][1]),
);
/**
* Keep coarse roads only outside the detailed tile cores.
*
* Each tile splits a line into the four regions around its core. Nearly every
* coarse road lies wholly inside one of those regions for nearly every tile,
* where the clip hands the line back unchanged, so that case is answered from
* the line's extent and only a line that reaches a tile's core or edge is
* clipped. Clipping every road against every tile froze the page for hundreds
* of milliseconds per road load at street level.
*
* `clip` is injectable for tests.
*/
export function outsideDetailTiles(roads, tiles, clip = clipTileLine) {
const boxes = tiles.map((tile) => tileToBBox(tile.z, tile.x, tile.y));
/** Keep coarse roads only outside the detailed tile cores. */
export function outsideDetailTiles(roads, tiles) {
return roads.flatMap((road) => {
let lines = [lineExtent(road.coordinates)];
for (const box of boxes) {
const next = [];
for (const line of lines) {
// Strictly inside one region, and inside the bounds every region
// shares: the other three regions return nothing.
const clear =
line.west > -180 &&
line.east < 180 &&
line.south > -85 &&
line.north < 85 &&
(line.east < box.west ||
line.west > box.east ||
(line.west > box.west &&
line.east < box.east &&
(line.north < box.south || line.south > box.north)));
if (!clear) {
const regions = [
{ west: -180, east: box.west, south: -85, north: 85 },
{ west: box.east, east: 180, south: -85, north: 85 },
{ west: box.west, east: box.east, south: -85, north: box.south },
{ west: box.west, east: box.east, south: box.north, north: 85 },
];
for (const region of regions)
for (const coordinates of clip(line.coordinates, region))
next.push(lineExtent(coordinates));
} else if (line.settled) {
next.push(line);
} else {
// The first unclipped pass can still move a vertex by a rounding
// step (a road crossing longitude or latitude 0); once a pass
// changes nothing, no later one will.
const coordinates = unclippedLine(line.coordinates);
if (!coordinates) continue;
const kept = lineExtent(coordinates);
kept.settled = sameLine(coordinates, line.coordinates);
next.push(kept);
}
}
lines = next;
let lines = [road.coordinates];
for (const tile of tiles) {
const box = tileToBBox(tile.z, tile.x, tile.y);
const regions = [
{ west: -180, east: box.west, south: -85, north: 85 },
{ west: box.east, east: 180, south: -85, north: 85 },
{ west: box.west, east: box.east, south: -85, north: box.south },
{ west: box.west, east: box.east, south: box.north, north: 85 },
];
lines = lines.flatMap((line) =>
regions.flatMap((region) => clipTileLine(line, region)),
);
}
return lines.map(({ coordinates }) => ({ ...road, coordinates }));
return lines.map((coordinates) => ({ ...road, coordinates }));
});
}
-39
View File
@@ -69,45 +69,6 @@ export function trafficSurfaceReady(scene) {
return currentRenderedFrame(scene)?.ready === true;
}
/**
* Ground-floor reads for one synchronous pass over road coordinates, asking
* the shared cache once per floor cell instead of once per coordinate.
*
* A road tile puts thousands of coordinates into a few hundred ~111 m cells,
* and each shared read formats and looks up two string keys, which added up
* to seconds per street-level load. Mesh floors answer per point (the nearest
* reading in the cell), so they are read through unchanged; only the DEM
* regime is one value per cell. Build a reader per pass: cells warm up
* between passes.
*/
export function floorCellReader(ground) {
const read = ground?.cachedGroundFloor;
if (!read) return () => undefined;
if (ground.meshFloorPreferred?.() !== false)
return (lat, lon) => read(lat, lon);
const cells = new Map();
return (lat, lon) => {
const y = lat * 1000,
x = lon * 1000;
const row = Math.round(y),
column = Math.round(x);
// The shared cache rounds with toFixed(3). At a cell edge that and
// Math.round can pick different cells, so an edge point asks directly.
if (!(
Math.abs(Math.abs(y - row) - 0.5) > 1e-6 &&
Math.abs(Math.abs(x - column) - 0.5) > 1e-6
))
return read(lat, lon);
const key = row * 1e6 + column;
let floor = cells.get(key);
if (floor === undefined) {
floor = read(lat, lon) ?? null;
cells.set(key, floor);
}
return floor;
};
}
/** Preserve bends and insert height samples at most 150 m apart, splitting long roads. */
export function roadSurfaceChunks(coordinates) {
const chunks = [];
+3 -12
View File
@@ -1,5 +1,5 @@
import * as Cesium from 'cesium';
import { floorCellReader, roadSurfaceChunks } from './surface.js';
import { roadSurfaceChunks } from './surface.js';
import { TRAFFIC_TIMING_ENABLED, DOT_HEIGHT_OFFSET } from './policy.js';
export function createTiming({ state: layerState, services, parts, source }) {
@@ -190,7 +190,7 @@ export function createTiming({ state: layerState, services, parts, source }) {
* waypoint-materialization time independently.
*/
function parseRoadsTimed(roadData, trace, parsedBySource) {
function parseRoadsTimed(roadData, trace) {
/* TRACE_ONLY_BEGIN */
const _trafficTimingState = trafficTimingPass(
trace,
@@ -231,19 +231,11 @@ export function createTiming({ state: layerState, services, parts, source }) {
}
const roads = [];
const floorAt = floorCellReader(services.ground);
/* TRACE_ONLY_BEGIN */
let _trafficTimingWaypointMaterializationMs = 0;
/* TRACE_ONLY_END */
for (const road of roadData.roads) {
if (!road.coordinates || road.coordinates.length < 2) continue;
// A replacing snapshot repeats records this load already parsed.
const known = parsedBySource?.get(road);
if (known) {
for (const parsed of known) roads.push(parsed);
continue;
}
const first = roads.length;
for (const coords of roadSurfaceChunks(road.coordinates)) {
const type = road.type;
@@ -253,7 +245,7 @@ export function createTiming({ state: layerState, services, parts, source }) {
const _trafficTimingMaterializeStart = performance.now();
/* TRACE_ONLY_END */
const waypoints = coords.map(([lng, lat]) => {
const floor = floorAt(lat, lng);
const floor = services.ground?.cachedGroundFloor?.(lat, lng);
const h =
(Number.isFinite(floor) && Math.abs(floor) <= 9000
? floor
@@ -287,7 +279,6 @@ export function createTiming({ state: layerState, services, parts, source }) {
flow: road.flow || null,
});
}
parsedBySource?.set(road, roads.slice(first));
}
/* TRACE_ONLY_BEGIN */
const _trafficTimingMetrics = {