Compare commits

..
Author SHA1 Message Date
Mazeyar Moeini FeizabadiandClaude Fable 5.2 829d1c1f7e docs(changelog): add the #1711 fragment
Co-Authored-By: Claude Fable 5.2 <[email protected]>
2026-11-13 16:06:03 +01:00
Mazeyar Moeini FeizabadiandClaude Fable 5.2 e79d972389 perf(firms): serve a cached body and gzip instead of serialising per request
/api/firms ran JSON.stringify over the whole fire set for every request and
sent it uncompressed: about 20 MB per 100,000 detections.

The handler now keeps the last body as bytes and reuses it for the same
cache entry and stale flag while the clock stays inside the span over which
the trailing-24 h filter keeps the same detections, so the JSON a request
gets is byte-identical to a fresh serialisation. A client that accepts gzip
gets the body gzipped, compressed once per kept body.

The span ends when the oldest kept acquisition minute turns 24 h old, so
requests minutes apart are still serialised afresh.

Co-Authored-By: Claude Fable 5.2 <[email protected]>
2026-11-13 14:43:14 +01:00
10 changed files with 304 additions and 306 deletions

No files matched your search

-4
View File
@@ -1,4 +0,0 @@
- ALPR camera badges no longer take depth samples while the overlay paints, or
again each time the layer is switched back on. On a terrain globe their
height comes from the loaded terrain and is remembered per camera; some
badges move a few metres onto the drawn surface (mazy1998, #1706).
+2
View File
@@ -0,0 +1,2 @@
- `/api/firms` keeps its response bytes and sends them gzipped instead of
serialising the whole fire set for every request (mazy1998, #1711).
+13 -8
View File
@@ -1513,14 +1513,10 @@ tile streamed in; floors resolve before new badges appear (bounded wait), and a
badge whose cell has not resolved clamps until it does, then moves in place.
Overlay anchors start on the same floor and write marker properties only on
change. Once the camera is still and tiles have settled, each overlay anchor is
replaced once by the rendered surface under it (accepted only within -30/+40 m
of the floor cell), so badges do not float or sink on slopes; wedge corners
follow their own floor cells instead of staying level with the apex. The
surface is read outside the overlay paint, one camera per frame: a terrain
globe answers from its loaded mesh (`globe.getHeight`) and only photoreal
tiles are depth-sampled (`scene.sampleHeight`). Answers are kept per camera
until the map source changes, so a refresh or turning the layer off and on
samples nothing again. Unknown bearings never acquire a wedge. The selected
replaced once by the rendered surface under it (two depth samples per paint,
accepted only within -30/+40 m of the floor cell), so badges do not float or
sink on slopes; wedge corners follow their own floor cells instead of staying
level with the apex. Unknown bearings never acquire a wedge. The selected
card's `DIRECTION` wraps after rounding to whole degrees, so a mapped bearing
of 359.5° or more reads `0°` rather than `360°`. The row reads
`N nearby · M on screen`; when every loaded camera is outside the view it adds
@@ -3301,6 +3297,15 @@ The `get_active_fires` tool reports acquisition times as UTC ISO timestamps,
including zero-padded hours and minutes; missing or invalid dates and times
remain `null`, including impossible calendar dates from a custom source.
`/api/firms` keeps its last response body as bytes and reuses it for requests
that would get the same payload: the same cache entry and `stale` flag, while
the clock stays inside the span over which the trailing-24-hour filter keeps
the same detections. That span ends when the oldest kept acquisition minute
turns 24 hours old, so requests minutes apart are still serialised afresh. A
client that sends `Accept-Encoding: gzip` gets the body gzipped (compressed
once per kept body, `Content-Encoding: gzip`); every response carries
`Vary: Accept-Encoding`. The JSON is unchanged.
The FIRMS proxy records source success after appending its rows. If aggregation
throws, that source reports failure without a contradictory success entry.
The existing per-record append continues to support large feeds; sequential
+1
View File
@@ -376,6 +376,7 @@
"modules": [
"server/providers/firms.js",
"server/providers/common/http.js",
"server/providers/common/compress.js",
"src/sources/httpBody.js",
"src/i18n/marker.js",
"src/data/firmsCsv.js"
+58 -10
View File
@@ -3,7 +3,12 @@ import https from 'node:https';
import { promises as fsp } from 'node:fs';
import { Readable } from 'node:stream';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import {
filterTrailing24h,
parseFirmsCsv,
trailing24hWindow,
} from '../../src/data/firmsCsv.js';
import { acceptsGzip, encodedBody, gzipBody } from './common/compress.js';
import { readResponseTextCapped } from './common/http.js';
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
@@ -179,6 +184,11 @@ export function firmsProxy() {
let statusCache = null;
/** @type {?Promise<?{used: number, limit: number}>} */
let statusInflight = null;
/**
* The last /api/firms payload as bytes, with its gzip. One slot.
* @type {?{entry: object, stale: boolean, from: number, until: number, body: Buffer, gzip: ?Promise<?Buffer>}}
*/
let served = null;
const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim();
@@ -259,12 +269,8 @@ export function firmsProxy() {
return { at: now, sources, fires };
}
/**
* Cache entry → response payload. Fires are RE-filtered to the trailing
* 24 h at serve time so a stale cache never serves >24h-old detections.
*/
function buildPayload(entry, stale) {
const fires = filterTrailing24h(entry.fires, Date.now());
/** Cache entry and the fires to serve from it → response payload. */
function buildPayload(entry, stale, fires) {
return {
fetchedAt: entry.at,
stale,
@@ -275,6 +281,32 @@ export function firmsProxy() {
};
}
/**
* The payload as the bytes to send. Serialising the whole fire set is tens
* of MB in a heavy season, so it is done once and kept, not once per request.
*
* Fires are RE-filtered to the trailing 24 h at serve time so a stale cache
* never serves >24h-old detections. The kept bytes are therefore reused only
* for the same entry and stale flag while the clock is inside the span over
* which that filter keeps the same records: a request never gets a body that
* serialising for it would not have produced.
*/
function payloadBody(entry, stale) {
const now = Date.now();
if (
served?.entry === entry &&
served.stale === stale &&
now >= served.from &&
now <= served.until
) {
return served;
}
const { kept, from, until } = trailing24hWindow(entry.fires, now);
const body = Buffer.from(JSON.stringify(buildPayload(entry, stale, kept)));
served = { entry, stale, from, until, body, gzip: null };
return served;
}
/** mapkey_status transactions, cached 5 min, best-effort (null on failure). */
function getTransactions(key) {
const now = Date.now();
@@ -327,6 +359,22 @@ export function firmsProxy() {
});
res.end(JSON.stringify(obj));
};
const sendPayload = async (entry, stale) => {
const kept = payloadBody(entry, stale);
// Compressed once per kept body, on the first request that accepts
// gzip, so an identity response never pays or waits for it.
const gzip = acceptsGzip(req)
? await (kept.gzip ??= gzipBody(kept.body).catch(() => null))
: null;
const encoded = encodedBody(req, kept.body, gzip);
if (res.headersSent) return;
res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...encoded.headers,
});
res.end(encoded.body);
};
try {
const subPath = String(req.url || '').split('?')[0];
const key = mapKey();
@@ -363,7 +411,7 @@ export function firmsProxy() {
const entry = mem;
if (entry && Date.now() - entry.at < TTL_MS) {
sendJson(200, buildPayload(entry, false));
await sendPayload(entry, false);
return;
}
// Stale or missing → refresh, single-flight (concurrent requests
@@ -389,9 +437,9 @@ export function firmsProxy() {
const pending = inflight;
const fresh = await pending;
if (fresh) {
sendJson(200, buildPayload(fresh, false));
await sendPayload(fresh, false);
} else if (entry) {
sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty
await sendPayload(entry, true); // upstream down — stale beats empty
} else {
sendJson(502, {
error: 'firms fetch failed and no cache available',
+27 -2
View File
@@ -182,19 +182,44 @@ export function acquisitionMsUtc(acqDate, acqTime) {
* @returns {Array<Object>} Filtered records (original objects, order preserved).
*/
export function filterTrailing24h(records, nowMs) {
if (!Array.isArray(records) || !Number.isFinite(nowMs)) return [];
return trailing24hWindow(records, nowMs).kept;
}
/**
* {@link filterTrailing24h} with the span of `nowMs` values, inclusive, over
* which it keeps exactly these records. A caller holding something built from
* `kept` can reuse it while its clock stays inside `from..until`.
* @param {Array<{acqDate: string, acqTime: string|number}>} records - Parser records.
* @param {number} nowMs - Reference epoch milliseconds.
* @returns {{kept: Array<Object>, from: number, until: number}} Span in epoch
* ms; NaN (never reusable) for unusable input.
*/
export function trailing24hWindow(records, nowMs) {
if (!Array.isArray(records) || !Number.isFinite(nowMs))
return { kept: [], from: NaN, until: NaN };
const oldest = nowMs - WINDOW_MS;
const newest = nowMs + FORWARD_SLACK_MS;
let from = -Infinity;
let until = Infinity;
const memo = new Map();
return records.filter((record) => {
const kept = records.filter((record) => {
const key = `${record.acqDate}:${record.acqTime}`;
let ms = memo.get(key);
if (ms === undefined) {
ms = acquisitionMsUtc(record.acqDate, record.acqTime);
memo.set(key, ms);
// Each distinct time narrows the span to the clock values that leave
// it on the same side of the window.
if (ms < oldest) from = Math.max(from, ms + WINDOW_MS + 1);
else if (ms > newest) until = Math.min(until, ms - FORWARD_SLACK_MS - 1);
else if (Number.isFinite(ms)) {
from = Math.max(from, ms - FORWARD_SLACK_MS);
until = Math.min(until, ms + WINDOW_MS);
}
}
return Number.isFinite(ms) && ms >= oldest && ms <= newest;
});
return { kept, from, until };
}
/** Trimmed string cell at index, '' for missing columns. */
+121 -1
View File
@@ -7,11 +7,15 @@ import dnsLookup from 'node:dns';
import https from 'node:https';
import net from 'node:net';
import { Readable } from 'node:stream';
import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js';
import os from 'node:os';
import path from 'node:path';
import { gunzipSync } from 'node:zlib';
import { filterTrailing24h, parseFirmsCsv, trailing24hWindow } from './firmsCsv.js';
import {
describeFetchError,
fetchFirms,
fetchIpv4,
firmsProxy,
isAddressSelectionFailure,
} from '../../server/providers/firms.js';
@@ -316,3 +320,119 @@ test('FIRMS requests only the satellites NASA still delivers', () => {
// a MAP_KEY transaction.
assert.ok(!SOURCES.includes('VIIRS_SNPP_NRT'));
});
// The /api/firms handler itself, over a disk cache seeded in a temp directory:
// no server, no upstream, and a placeholder MAP_KEY that is never sent anywhere.
const TTL_MS = 30 * 60_000;
const fire = (acqDate, acqTime, lat = 1) => ({ lat, lon: 2, frp: 4.5, confidence: 'n', brightness: 330.1,
brightnessTi5: 290.2, daynight: 'D', acqDate, acqTime, satellite: 'N20', instrument: 'VIIRS' });
const SERVED_SOURCES = SOURCES.map((source, index) => ({ source, count: index === 0 ? 20_001 : 0, ok: true }));
// 20,000 detections an hour old and one that turns 24 h old five minutes after NOW.
const SERVED_FIRES = [fire('2026-09-10', '1205'), ...Array.from({ length: 20_000 }, (_, i) => fire('2026-09-11', '1100', i / 1000))];
const LEAVES_AT = NOW + 5 * 60_000;
/** What the handler sent before it kept a body: one JSON.stringify of the payload per request. */
const unfixedBody = (at, stale, now) => {
const fires = filterTrailing24h(SERVED_FIRES, now);
return JSON.stringify({ fetchedAt: at, stale, ttlMs: TTL_MS, sources: SERVED_SOURCES, count: fires.length, fires });
};
function createFirmsHandler(t, at) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'firms-proxy-'));
fs.mkdirSync(path.join(dir, '.gev-cache'));
fs.writeFileSync(path.join(dir, '.gev-cache', 'firms.json'), JSON.stringify({ at, sources: SERVED_SOURCES, fires: SERVED_FIRES }));
const cwd = process.cwd();
const key = process.env.FIRMS_MAP_KEY;
const realFetch = globalThis.fetch;
process.env.FIRMS_MAP_KEY = 'unit-test-placeholder';
// A refresh must never reach NASA from a test: every upstream call fails.
globalThis.fetch = async () => { throw new Error('upstream down'); };
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['Date'], now: NOW });
t.after(() => {
globalThis.fetch = realFetch;
if (key === undefined) delete process.env.FIRMS_MAP_KEY; else process.env.FIRMS_MAP_KEY = key;
fs.rmSync(dir, { recursive: true, force: true });
});
let handler;
process.chdir(dir); // the proxy resolves its cache directory when created
try {
firmsProxy().configureServer({ middlewares: { use: (route, fn) => { handler = fn; } } });
} finally {
process.chdir(cwd);
}
return (acceptEncoding) => new Promise((resolve, reject) => {
const res = {
headersSent: false,
writeHead(status, headers) { this.headersSent = true; this.status = status; this.headers = headers; },
end(body) { resolve({ status: this.status, headers: this.headers, body }); },
};
handler({ url: '/', headers: acceptEncoding === undefined ? {} : { 'accept-encoding': acceptEncoding } }, res).catch(reject);
});
}
test('/api/firms serialises once for requests that share a payload', async (t) => {
const request = createFirmsHandler(t, NOW);
const first = await request();
t.mock.timers.setTime(LEAVES_AT); // exactly 24 h old is still inside the window
const second = await request();
assert.equal(first.status, 200);
assert.ok(Buffer.isBuffer(first.body), 'the body is kept as bytes');
assert.equal(second.body, first.body, 'the second request sends the kept bytes, not a new serialisation');
assert.equal(first.body.toString(), unfixedBody(NOW, false, NOW), 'byte-identical to the per-request serialisation');
assert.equal(JSON.parse(first.body).count, 20_001);
});
test('/api/firms sends gzip only to a client that accepts it', async (t) => {
const request = createFirmsHandler(t, NOW);
const identity = await request();
const gzipped = await request('gzip, deflate, br');
const refused = await request('gzip;q=0, br');
assert.deepEqual(identity.headers,
{ 'Content-Type': 'application/json', 'Cache-Control': 'no-store', Vary: 'Accept-Encoding' });
assert.deepEqual(gzipped.headers, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store',
'Content-Encoding': 'gzip', Vary: 'Accept-Encoding' });
assert.deepEqual(refused.headers, identity.headers);
assert.equal(refused.body, identity.body);
assert.ok(gzipped.body.byteLength < identity.body.byteLength / 4, 'the fire set compresses');
assert.equal(gunzipSync(gzipped.body).toString(), unfixedBody(NOW, false, NOW), 'decodes to the identity bytes');
assert.equal((await request('gzip')).body, gzipped.body, 'compressed once');
});
test('/api/firms never reuses a body the request would not have produced', async (t) => {
const request = createFirmsHandler(t, NOW - TTL_MS + 6 * 60_000);
const at = NOW - TTL_MS + 6 * 60_000;
const before = await request();
t.mock.timers.setTime(LEAVES_AT + 1); // one detection is now older than 24 h
const after = await request('gzip');
assert.equal(before.body.toString(), unfixedBody(at, false, NOW));
assert.equal(gunzipSync(after.body).toString(), unfixedBody(at, false, LEAVES_AT + 1));
assert.equal(JSON.parse(gunzipSync(after.body)).count, 20_000);
t.mock.timers.setTime(NOW + 6 * 60_000); // the entry reaches its TTL and the refresh fails
const stale = await request();
assert.equal(stale.body.toString(), unfixedBody(at, true, NOW + 6 * 60_000));
assert.equal((await request()).body, stale.body, 'the stale body is kept too');
});
test('trailing24hWindow spans the clock values that keep the same records', () => {
const records = [fire('2026-09-10', '1205'), fire('2026-09-10', '1300'), fire('2026-09-11', '1100'),
fire('2026-09-11', '1430'), fire('2026-09-11', '1600'), fire('bad-date', '1006')];
const edges = records.map((r) => Date.parse(`${r.acqDate}T${r.acqTime.slice(0, 2)}:${r.acqTime.slice(2)}Z`))
.filter(Number.isFinite).flatMap((ms) => [ms - 2 * 3600_000, ms + 24 * 3600_000]);
assert.equal(edges.length, 10);
const clocks = [NOW, ...edges.flatMap((ms) => [ms - 1, ms, ms + 1])];
for (const now of clocks) {
const { kept, from, until } = trailing24hWindow(records, now);
assert.deepEqual(kept, filterTrailing24h(records, now));
assert.ok(from <= now && now <= until, 'the span holds its own clock');
for (const other of clocks) {
const then = filterTrailing24h(records, other);
const same = then.length === kept.length && then.every((r, i) => r === kept[i]);
// Before the first record and after the last both keep nothing; the span covers one side.
const bothEmpty = then.length === 0 && kept.length === 0;
if (other >= from && other <= until) assert.ok(same, `${other} is inside the span of ${now}`);
else assert.ok(!same || bothEmpty, `${other} is outside the span of ${now}`);
}
}
assert.deepEqual(trailing24hWindow([], NOW), { kept: [], from: -Infinity, until: Infinity });
assert.deepEqual(trailing24hWindow(records, NaN), { kept: [], from: NaN, until: NaN });
});
+57 -139
View File
@@ -4,10 +4,10 @@ import {
MARKER_ICON_SIZE,
SELECTED_MARKER_ICON_SIZE,
MARKER_FLOOR_LIFT_M,
REMEMBERED_ANCHORS,
ANCHOR_SAMPLES_PER_PAINT,
SURFACE_BELOW_FLOOR_M,
SURFACE_ABOVE_FLOOR_M,
SURFACE_SAMPLES_PER_FRAME,
SURFACE_SAMPLES_PER_PAINT,
} from './policy.js';
import {
MARKER_IMAGE,
@@ -19,17 +19,11 @@ import {
validAlprGroundHeight,
} from './visuals.js';
const nextAnimationFrame = (step) =>
typeof requestAnimationFrame === 'function'
? requestAnimationFrame(step)
: setTimeout(step, 16);
/** Per-layer presentation on a caller-owned overlay; Cesium remains the fallback. */
export function createAlprOverlay({
state,
services,
onSurfaceChange = () => {},
requestFrame = nextAnimationFrame,
}) {
let lane,
removeMapListener,
@@ -39,12 +33,7 @@ export function createAlprOverlay({
hits = [],
images = [],
destroyed = false,
lastCamera = null,
sampling = false;
// Rendered-surface answers by camera id. Entities are rebuilt by a refresh
// and by turning the layer off and on; keeping the answers on them meant
// every rebuild sampled the same cameras again.
const surfaceAnswers = new Map();
lastCamera = null;
const requestPaint = () => {
if (!destroyed && state.enabled) lane?.requestPaint();
};
@@ -103,72 +92,38 @@ export function createAlprOverlay({
entity.gevAlprDisplayPosition = null;
nativeVisible(entity, true);
}
surfaceAnswers.clear();
state.lastAnchorSampleAt = 0;
retryUntil = Date.now() + 15000;
requestPaint();
}
function rememberSurface(record, height, checked) {
surfaceAnswers.delete(record.id);
surfaceAnswers.set(record.id, {
latitude: record.latitude,
longitude: record.longitude,
height,
checked,
});
if (surfaceAnswers.size > REMEMBERED_ANCHORS)
surfaceAnswers.delete(surfaceAnswers.keys().next().value);
}
/**
* Rendered surface height under a camera. A terrain globe answers from its
* loaded mesh; only photoreal tiles need `sampleHeight`, which renders depth
* and compiles its shaders on first use (a freeze of up to 1.9 s at load).
*/
function surfaceHeight(record, entity) {
function anchorFor(record, entity, budget) {
if (entity.gevAlprCanvasPosition) return entity.gevAlprCanvasPosition;
const scene = state.viewer.scene;
const location = Cesium.Cartographic.fromDegrees(
record.longitude,
record.latitude,
);
if (scene.globe?.show) return scene.globe.getHeight?.(location);
if (!scene.sampleHeightSupported) return undefined;
try {
return scene.sampleHeight(location, [entity]);
} catch {
/* streaming tiles */
return undefined;
}
}
/** The anchor a paint can have without sampling, or null while it waits for one. */
function anchorFor(record, entity) {
if (entity.gevAlprCanvasPosition) return entity.gevAlprCanvasPosition;
const scene = state.viewer.scene;
const answer = surfaceAnswers.get(record.id);
if (
answer?.latitude === record.latitude &&
answer.longitude === record.longitude
) {
setAnchor(record, entity, answer.height);
if (answer.checked)
entity.gevAlprSurfaceCheckedFor = entity.gevAlprCanvasPosition;
return entity.gevAlprCanvasPosition;
}
// A floor-placed marker anchors where its native badge sits. A marker
// still waiting for its floor reads the terrain globe, or on photoreal
// tiles waits for sampleAnchors().
// A floor-placed marker anchors where its native badge sits; only a
// marker still waiting for its floor samples the rendered surface, and
// each paint samples a few at most (each sample is a depth render).
let height;
const floor = services.groundFloor.cachedGroundFloor(
record.latitude,
record.longitude,
);
if (validAlprGroundHeight(floor)) height = floor + MARKER_FLOOR_LIFT_M;
else if (scene.globe.show)
height = scene.globe.getHeight?.(
Cesium.Cartographic.fromDegrees(record.longitude, record.latitude),
);
else if (scene.sampleHeightSupported) {
if (budget.samples <= 0) return null;
budget.samples -= 1;
try {
height = scene.sampleHeight(location, [entity]);
} catch {
/* streaming tiles */
}
}
if (!validAlprGroundHeight(height) && scene.globe.show)
height = scene.globe.getHeight?.(location);
if (!validAlprGroundHeight(height)) return null;
setAnchor(record, entity, height);
return entity.gevAlprCanvasPosition;
@@ -203,19 +158,14 @@ export function createAlprOverlay({
}
/**
* Sample the rendered surface for the overlay cameras, outside the paint.
*
* A camera with no floor yet gets its anchor from a sample. An anchored one
* is checked against the surface once: floors come from ~111 m cells, so on
* a slope a badge could float or sink by tens of metres. Checks run only
* while the camera is still and tiles have settled, and samples far outside
* the floor prior (a roof or a streaming tile) are rejected. Each frame
* takes SURFACE_SAMPLES_PER_FRAME samples, so no frame carries the whole
* cohort; sampling inside paint() put up to six depth renders in one frame.
* Replace floor anchors with the rendered surface under each overlay camera.
* Floors come from ~111 m cells, so on a slope a badge could float or sink
* by tens of metres. Runs only while the camera is still and tiles have
* settled, a few depth samples per paint, and each anchor at most once;
* samples far outside the floor prior (a roof or a streaming tile) are
* rejected. Returns whether anchors remain to check.
*/
function sampleAnchors() {
sampling = false;
if (destroyed || !state.enabled) return;
function refineAnchors(budget) {
const camera = state.viewer.camera;
const scene = state.viewer.scene;
const still =
@@ -238,41 +188,27 @@ export function createAlprOverlay({
position: Cesium.Cartesian3.clone(camera.positionWC),
direction: Cesium.Cartesian3.clone(camera.directionWC),
};
const canSample = scene.globe?.show || scene.sampleHeightSupported;
let samples = SURFACE_SAMPLES_PER_FRAME,
pending = false,
progressed = false,
moved = false;
// Cameras with no anchor come first: they are not on the overlay yet.
for (const record of records) {
const entity = state.dataSource.entities.getById(record.id);
if (!entity || entity.gevAlprCanvasPosition) continue;
// The floor may have resolved since the paint that found none.
if (anchorFor(record, entity)) moved = true;
else if (!canSample || samples <= 0) pending = true;
else {
samples -= 1;
const sample = surfaceHeight(record, entity);
if (validAlprGroundHeight(sample)) {
setAnchor(record, entity, sample);
rememberSurface(record, sample, false);
moved = progressed = true;
} else pending = true;
}
}
if (!scene.sampleHeightSupported) return false;
let pending = false;
for (const record of records) {
const entity = state.dataSource.entities.getById(record.id);
const anchor = entity?.gevAlprCanvasPosition;
if (!anchor || !canSample || entity.gevAlprSurfaceCheckedFor === anchor)
continue;
if (!still || !surfaceSettled(scene) || samples <= 0) {
if (!anchor || entity.gevAlprSurfaceCheckedFor === anchor) continue;
if (!still || !surfaceSettled(scene) || budget.samples <= 0) {
pending = true;
continue;
}
samples -= 1;
progressed = true;
const sample = surfaceHeight(record, entity);
let height = Cesium.Cartographic.fromCartesian(anchor).height;
budget.samples -= 1;
let sample;
try {
sample = scene.sampleHeight(
Cesium.Cartographic.fromDegrees(record.longitude, record.latitude),
[entity],
);
} catch {
/* streaming tiles */
}
const current = Cesium.Cartographic.fromCartesian(anchor).height;
const floor = services.groundFloor.cachedGroundFloor(
record.latitude,
record.longitude,
@@ -282,43 +218,21 @@ export function createAlprOverlay({
(!validAlprGroundHeight(floor) ||
(sample >= floor - SURFACE_BELOW_FLOOR_M &&
sample <= floor + SURFACE_ABOVE_FLOOR_M));
if (accepted && Math.abs(sample + MARKER_FLOOR_LIFT_M - height) > 1) {
height = sample + MARKER_FLOOR_LIFT_M;
setAnchor(record, entity, height);
moved = true;
}
if (accepted && Math.abs(sample + MARKER_FLOOR_LIFT_M - current) > 1)
setAnchor(record, entity, sample + MARKER_FLOOR_LIFT_M);
entity.gevAlprSurfaceCheckedFor = entity.gevAlprCanvasPosition;
// A surface that did not answer is asked again after a rebuild.
if (validAlprGroundHeight(sample)) rememberSurface(record, height, true);
if (entity.id === state.selectedId)
entity.gevAlprDisplayPosition = entity.gevAlprCanvasPosition;
}
if (moved) requestPaint();
if (!pending || Date.now() >= retryUntil) return;
// Take the next ration on the next frame; when nothing could be sampled
// (camera moving, tiles streaming, no floor yet) look again in 250 ms.
if (progressed) scheduleSampling();
else if (!retryTimer)
retryTimer = setTimeout(() => {
retryTimer = null;
scheduleSampling();
}, 250);
}
function scheduleSampling() {
if (sampling || destroyed) return;
sampling = true;
requestFrame(sampleAnchors);
return pending;
}
function paint({ ctx, width, height, keyhole, occluder }) {
hits = [];
if (!state.enabled || destroyed) return;
let waiting = false;
let unresolved = false;
const painted = [];
const canSample =
state.viewer.scene.globe?.show ||
state.viewer.scene.sampleHeightSupported;
const budget = { samples: ANCHOR_SAMPLES_PER_PAINT };
for (const record of records) {
const entity = state.dataSource.entities.getById(record.id);
if (!entity) continue;
@@ -332,14 +246,12 @@ export function createAlprOverlay({
nativeVisible(entity, true);
continue;
}
const anchor = anchorFor(record, entity);
const anchor = anchorFor(record, entity, budget);
if (!anchor) {
nativeVisible(entity, true);
waiting = true;
unresolved = true;
continue;
}
if (canSample && entity.gevAlprSurfaceCheckedFor !== anchor)
waiting = true;
canvasOwned(entity, true);
if (occluder && !occluder.isPointVisible(anchor)) {
setShown(entity, false);
@@ -423,9 +335,15 @@ export function createAlprOverlay({
ctx.restore();
hits.push({ id: record.id, x: origin.x, y: origin.y, radius: size / 2 });
}
// Anchors still to find or to check are sampled after this paint, a few
// per frame, never inside it.
if (waiting && !retryTimer && Date.now() < retryUntil) scheduleSampling();
const refining = refineAnchors({
samples: Math.min(budget.samples, SURFACE_SAMPLES_PER_PAINT),
});
if ((unresolved || refining) && !retryTimer && Date.now() < retryUntil) {
retryTimer = setTimeout(() => {
retryTimer = null;
requestPaint();
}, 250);
}
}
return {
+21 -137
View File
@@ -45,10 +45,8 @@ function harness(t, count = 2, { floor = null } = {}) {
active,
removed = 0,
paints = 0;
const frames = [];
const overlay = createAlprOverlay({
state,
requestFrame: (step) => frames.push(step),
services: {
groundFloor: { cachedGroundFloor: () => floor },
overlays: {
@@ -108,12 +106,6 @@ function harness(t, count = 2, { floor = null } = {}) {
entities,
draw,
paint: (occluder) => paint({ ctx, width: 800, height: 600, occluder }),
/** Run the sampling step queued for the next animation frame, if any. */
frame() {
const step = frames.shift();
step?.();
return Boolean(step);
},
mapChanged() {
mapListener({ detail: { status: 'ready' } });
},
@@ -251,149 +243,41 @@ test('overlay glyphs and native badges shrink alike with distance', () => {
assert.ok(Math.abs(markerScale(20_250) - 0.8) < 1e-9);
});
/** A photoreal scene: no globe, the rendered surface read by depth samples. */
function photoreal(h, heightAt = () => 172) {
const scene = h.state.viewer.scene;
h.state.viewer.camera.directionWC = Cesium.Cartesian3.UNIT_X;
scene.globe.show = false;
scene.sampleHeightSupported = true;
scene.primitives = { length: 0, get: () => null };
const sampled = [];
scene.sampleHeight = (carto) => {
sampled.push(carto);
return heightAt(Cesium.Math.toDegrees(carto.latitude));
};
return sampled;
}
const anchorHeight = (h, id) =>
Cesium.Cartographic.fromCartesian(
h.entities.getById(id).gevAlprCanvasPosition,
).height;
/** Run sampling frames until none is queued, waiting out the 250 ms look-again. */
function settle(t, h) {
for (let i = 0; i < 400; i++) if (!h.frame()) t.mock.timers.tick(250);
}
test('a still view replaces floor anchors with the rendered surface, within the floor window', (t) => {
// Floors come from ~111 m cells; on a slope a badge floated above the mesh.
t.mock.timers.enable({ apis: ['setTimeout', 'Date'] });
const h = harness(t, 2, { floor: 150 });
const sampled = photoreal(h, (latitude) => (latitude > 30.00005 ? 400 : 172));
const scene = h.state.viewer.scene;
const camera = h.state.viewer.camera;
camera.directionWC = Cesium.Cartesian3.UNIT_X;
scene.sampleHeightSupported = true;
scene.primitives = { length: 0, get: () => null };
let samples = 0;
scene.sampleHeight = (carto) => {
samples++;
return Cesium.Math.toDegrees(carto.latitude) > 30.00005 ? 400 : 172;
};
h.overlay.sync(h.records);
h.paint();
assert.ok(
Math.abs(anchorHeight(h, 'camera:0') - 151.5) < 0.01,
'floor anchor first',
);
assert.equal(h.frame(), true, 'the paint queued a sampling frame');
const height = (id) =>
Cesium.Cartographic.fromCartesian(
h.entities.getById(id).gevAlprCanvasPosition,
).height;
assert.equal(
sampled.length,
samples,
0,
'no depth samples while the camera may still be moving',
);
settle(t, h);
assert.ok(Math.abs(height('camera:0') - 151.5) < 0.01, 'floor anchor first');
h.paint();
assert.ok(
Math.abs(anchorHeight(h, 'camera:0') - 173.5) < 0.01,
Math.abs(height('camera:0') - 173.5) < 0.01,
'rendered surface once still',
);
assert.ok(
Math.abs(anchorHeight(h, 'camera:1') - 151.5) < 0.01,
Math.abs(height('camera:1') - 151.5) < 0.01,
'a rooftop-height sample is rejected',
);
assert.equal(sampled.length, 2);
const taken = samples;
h.paint();
settle(t, h);
assert.equal(sampled.length, 2, 'each anchor is checked once');
});
test('a paint never samples the surface; frames take one sample each', (t) => {
// Sampling inside paint() put up to six depth renders into one frame, and
// the first of them compiles shaders: a freeze of 0.8 to 1.9 s at load.
t.mock.timers.enable({ apis: ['setTimeout', 'Date'] });
const h = harness(t, 8);
const sampled = photoreal(h);
h.overlay.sync(h.records);
h.paint();
h.paint();
assert.equal(sampled.length, 0, 'no sample inside a paint');
assert.equal(h.draw.length, 0, 'unanchored cameras keep their native badge');
for (let taken = 1; taken <= 8; taken++) {
assert.equal(h.frame(), true);
assert.equal(sampled.length, taken, 'one sample per frame');
}
for (const record of h.records)
assert.ok(Math.abs(anchorHeight(h, record.id) - 172) < 0.01);
h.paint();
assert.ok(h.draw.length > 0, 'anchored cameras paint on the overlay');
});
test('a terrain globe is read from its mesh, never depth-sampled', (t) => {
t.mock.timers.enable({ apis: ['setTimeout', 'Date'] });
const h = harness(t, 3, { floor: 150 });
const sampled = photoreal(h);
const scene = h.state.viewer.scene;
scene.globe.show = true;
scene.globe.tilesLoaded = true;
let reads = 0;
scene.globe.getHeight = () => {
reads++;
return 172;
};
h.overlay.sync(h.records);
h.paint();
settle(t, h);
assert.equal(sampled.length, 0);
assert.equal(reads, 3);
// The same anchor the depth sample gave: the surface plus the marker lift.
for (const record of h.records)
assert.ok(Math.abs(anchorHeight(h, record.id) - 173.5) < 0.01);
});
test('surface answers outlive rebuilt markers and a re-enable, until the map changes', (t) => {
t.mock.timers.enable({ apis: ['setTimeout', 'Date'] });
const h = harness(t, 4, { floor: 150 });
const sampled = photoreal(h);
h.overlay.sync(h.records);
h.paint();
settle(t, h);
assert.equal(sampled.length, 4);
const before = h.records.map((record) => anchorHeight(h, record.id));
// A refresh or a toggle removes the entities and builds new ones.
const rebuild = () => {
h.overlay.clear();
h.entities.removeAll();
for (const record of h.records)
h.entities.add({
id: record.id,
billboard: { image: 'fallback', show: true },
polyline: { show: true },
polygon: { show: true },
});
h.overlay.sync(h.records);
h.paint();
settle(t, h);
};
rebuild();
h.state.enabled = false;
h.overlay.clear();
h.state.enabled = true;
rebuild();
assert.equal(sampled.length, 4, 'no camera is sampled again');
assert.deepEqual(
h.records.map((record) => anchorHeight(h, record.id)),
before,
);
// A camera that moved is a different question.
h.records[0] = { ...h.records[0], latitude: h.records[0].latitude + 0.001 };
rebuild();
assert.equal(sampled.length, 5);
h.mapChanged();
h.paint();
settle(t, h);
assert.equal(sampled.length, 9, 'a new map surface is sampled afresh');
assert.equal(samples, taken, 'each anchor is checked once');
});
+4 -5
View File
@@ -66,15 +66,14 @@ export const MAX_CANVAS_FRUSTUMS = 64;
/** Metres a floor-placed marker sits above its resolved ground cell. */
export const MARKER_FLOOR_LIFT_M = 1.5;
/** Rendered-surface samples one frame may take (on photoreal tiles each is a
* depth render of several milliseconds, so they are spread over frames). */
export const SURFACE_SAMPLES_PER_FRAME = 1;
/** Surface answers kept by camera id, so a refresh or re-enable asks nothing. */
export const REMEMBERED_ANCHORS = 2048;
/** Rendered-surface samples one overlay paint may take (each is a depth render). */
export const ANCHOR_SAMPLES_PER_PAINT = 4;
/** Accepted rendered-surface window around a floor cell (a cut or an embankment
* sits within it; a rooftop or an unstreamed tile usually does not). */
export const SURFACE_BELOW_FLOOR_M = 30;
/** Rendered-surface refinements per paint while the camera is still. */
export const SURFACE_SAMPLES_PER_PAINT = 2;
export const SURFACE_ABOVE_FLOOR_M = 40;
/** OSM attribution may collapse after five seconds; full credit stays in Data attribution. */