fix(cctv): harden the Kartverket height lookup for Vegvesen cameras

- Default backoff is 30 min, longer than the 15-minute catalog refresh,
  so a failing Kartverket costs one attempt per backoff, not per refresh.
- A point a 200 answer leaves out is cached as no height (6 h TTL)
  instead of being asked again, and waited on, every refresh.
- Only 408, 429 and 5xx stop the lookup; any other 4xx marks that
  request's points as no height and the lookup carries on.
- Each request has its own 10 s timeout inside the shared 30 s deadline;
  a lookup cut short only by the deadline resumes without a backoff.
- The lane waits only on a cold cache (>10% of cameras missing); a few
  new cameras or expired misses are looked up in the background.
- applyVegvesenGroundHeights skips cameras without finite coordinates,
  so it really never throws.
- clearKartverketHeightCache bumps a generation, so a lookup started
  before a clear cannot write to the cache, backoff or pending slot.
- Credit and DATA_SOURCES note the modification (water set to 0 m,
  rounded to 0.1 m) as CC BY 4.0 asks; docs match the stop rules.
- Tests for the deadline being shared, per-request timeouts, 503,
  oversized and non-JSON bodies, 4xx, left-out points, Retry-After end
  to end, overlapping refreshes, the warm-cache wait, bad coordinates,
  the cache-clear generation and the loader's wait budget.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/data/cctvVegvesen.test.mjs

@@ -12,11 +12,14 @@ import {
 import {
   DEFAULT_VEGVESEN_CCTV_URL,
   KARTVERKET_HEIGHT_URL,
+  KARTVERKET_CATALOG_WAIT_MS,
   KARTVERKET_CONCURRENCY,
   KARTVERKET_FAILURE_BACKOFF_MS,
   KARTVERKET_MAX_POINTS,
+  KARTVERKET_MAX_RESPONSE_BYTES,
   KARTVERKET_NULL_TTL_MS,
   KARTVERKET_USER_AGENT,
+  CCTV_SOURCE_CACHE_MS,
   VEGVESEN_IMAGE_ORIGIN,
 } from '../../server/providers/cctv/constants.js';
 import { createCctvCatalog } from '../../server/providers/cctv/catalog.js';
@@ -141,9 +144,9 @@ test('the loader dedupes, and failures degrade to an empty pack', async (t) => {
   const cameras = await loadVegvesenSourcesFromOpenData();
   // The catalog first; the rest is the Kartverket height lookup.
   assert.equal(requested[0], DEFAULT_VEGVESEN_CCTV_URL);
-  assert.ok(
-    requested.slice(1).every((url) => url.startsWith(KARTVERKET_HEIGHT_URL)),
-  );
+  // Two unique positions: one Kartverket request.
+  assert.equal(requested.length, 2);
+  assert.ok(requested[1].startsWith(KARTVERKET_HEIGHT_URL));
   // Nearest-to-anchor first: Oslo leads.
   assert.deepEqual(
     cameras.map((c) => c.id),
@@ -258,6 +261,8 @@ test('Kartverket heights come in batches of 20, matched by echoed coordinates',
     requested.push(String(url));
     assert.ok(String(url).startsWith(KARTVERKET_HEIGHT_URL));
     assert.equal(init.headers['User-Agent'], KARTVERKET_USER_AGENT);
+    // Redirects are refused, so a request can never leave the fixed host.
+    assert.equal(init.redirect, 'error');
     // Reply in reverse order: matching must not depend on list position.
     const response = kartverketResponse(url, (x) =>
       x > 7.04 ? null : Math.round((x - 7) * 10000),
@@ -402,26 +407,138 @@ test('a transport failure stops the lookup: no retries, no new requests', async
   // after it starts, and no point is resent.
   assert.equal(calls, KARTVERKET_CONCURRENCY);
   assert.equal(heights.size, 0);
-  assert.deepEqual(failure, { status: 429, retryAfter: '120' });
+  assert.deepEqual(failure, {
+    status: 429,
+    retryAfter: '120',
+    deadline: false,
+  });
 });
 
+/** A fetch that never answers, and rejects only when its signal aborts. */
+const hangingFetch = (url, init) => {
+  assert.ok(init.signal instanceof AbortSignal);
+  return new Promise((_, reject) => {
+    init.signal.addEventListener('abort', () => reject(init.signal.reason));
+  });
+};
+
 test('the lookup deadline bounds a hanging service', async (t) => {
   withHeightsEnv(t);
-  t.mock.method(
-    globalThis,
-    'fetch',
-    (url, init) =>
-      new Promise((_, reject) => {
-        init.signal.addEventListener('abort', () => reject(init.signal.reason));
-      }),
-  );
+  t.mock.method(globalThis, 'fetch', hangingFetch);
   const started = Date.now();
   const { heights, failure } = await fetchKartverketHeights([cameraAt(0)], {
     timeoutMs: 30,
   });
-  assert.ok(Date.now() - started < 2000);
+  const elapsed = Date.now() - started;
+  assert.ok(elapsed >= 25 && elapsed < 2000, `elapsed ${elapsed} ms`);
   assert.equal(heights.size, 0);
-  assert.equal(failure.status, null);
+  assert.deepEqual(failure, { status: null, retryAfter: null, deadline: true });
+});
+
+test('one hanging request times out on its own, as a failing service', async (t) => {
+  withHeightsEnv(t);
+  t.mock.method(globalThis, 'fetch', hangingFetch);
+  const { failure } = await fetchKartverketHeights([cameraAt(0)], {
+    timeoutMs: 60_000,
+    requestTimeoutMs: 20,
+  });
+  assert.deepEqual(failure, {
+    status: null,
+    retryAfter: null,
+    deadline: false,
+  });
+});
+
+test('the deadline is shared: later requests do not get a fresh one', async (t) => {
+  withHeightsEnv(t);
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', (url, init) => {
+    calls += 1;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(
+        () => resolve(kartverketResponse(url, () => 1)),
+        40,
+      );
+      init.signal.addEventListener('abort', () => {
+        clearTimeout(timer);
+        reject(init.signal.reason);
+      });
+    });
+  });
+  // Five waves of 40 ms against a 100 ms deadline for the whole lookup.
+  const { heights, failure } = await fetchKartverketHeights(
+    Array.from({ length: 5 * KARTVERKET_CONCURRENCY * 20 }, (_, i) =>
+      cameraAt(i),
+    ),
+    { timeoutMs: 100 },
+  );
+  assert.equal(failure?.deadline, true);
+  assert.ok(heights.size < 5 * KARTVERKET_CONCURRENCY * 20);
+  assert.ok(calls < 5 * KARTVERKET_CONCURRENCY, `calls ${calls}`);
+});
+
+test('a 503, an unreadable or an oversized body stops the lookup', async (t) => {
+  withHeightsEnv(t);
+  const answers = {
+    503: () => new Response('down', { status: 503 }),
+    'not json': () =>
+      new Response('<html>maintenance</html>', {
+        headers: { 'Content-Type': 'text/html' },
+      }),
+    oversized: () =>
+      Response.json({
+        punkter: [],
+        pad: 'x'.repeat(KARTVERKET_MAX_RESPONSE_BYTES + 1),
+      }),
+  };
+  for (const [label, answer] of Object.entries(answers)) {
+    clearKartverketHeightCache();
+    let calls = 0;
+    t.mock.method(globalThis, 'fetch', async () => {
+      calls += 1;
+      return answer();
+    });
+    const { heights, failure } = await fetchKartverketHeights(
+      Array.from({ length: 400 }, (_, i) => cameraAt(i)),
+    );
+    assert.equal(calls, KARTVERKET_CONCURRENCY, label);
+    assert.equal(heights.size, 0, label);
+    assert.equal(failure?.deadline, false, label);
+    assert.equal(failure?.status, label === '503' ? 503 : null, label);
+  }
+});
+
+test('a 4xx rejects that request only: its points get no height, the rest go on', async (t) => {
+  withHeightsEnv(t);
+  t.mock.method(globalThis, 'fetch', async (url) =>
+    sentPoints(url)[0][0] === 7
+      ? new Response('bad point', { status: 400 })
+      : kartverketResponse(url, () => 42),
+  );
+  const { heights, failure } = await fetchKartverketHeights(
+    Array.from({ length: 40 }, (_, i) => cameraAt(i)),
+  );
+  assert.equal(failure, null);
+  assert.equal(heights.size, 40);
+  assert.equal(heights.get('7.000000,60.000000'), null);
+  assert.equal(heights.get('7.020000,60.020000'), 42);
+});
+
+test('a point a 200 answer leaves out is cached as no height', async (t) => {
+  withHeightsEnv(t);
+  const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => {
+    const points = sentPoints(url).slice(1); // drop the first point
+    return Response.json({
+      koordsys: 4258,
+      punkter: points.map(([x, y]) => ({ x, y, z: 7 })),
+    });
+  });
+  const cameras = [cameraAt(0), cameraAt(1)];
+  assert.equal(await applyVegvesenGroundHeights(cameras), 1);
+  assert.equal(cameras[0].groundElevationM, 150);
+  // The next refresh asks nothing: the left-out point is a cached miss.
+  assert.equal(await applyVegvesenGroundHeights([cameraAt(0), cameraAt(1)]), 1);
+  assert.equal(fetchMock.mock.callCount(), 1);
 });
 
 test('Retry-After sets the backoff, bounded to [1 min, 1 h]', () => {
@@ -478,8 +595,12 @@ test('after a failure no lookup starts until the backoff ends', async (t) => {
   assert.equal(cameras[0].groundElevationM, 150);
   assert.equal(fetchMock.mock.callCount(), 1);
 
-  // Refreshes inside the backoff cost nothing.
-  clock += KARTVERKET_FAILURE_BACKOFF_MS - 1;
+  // The default backoff outlasts the catalog refresh, so the next refresh
+  // and every one inside the backoff cost nothing.
+  assert.ok(KARTVERKET_FAILURE_BACKOFF_MS > CCTV_SOURCE_CACHE_MS);
+  clock += CCTV_SOURCE_CACHE_MS;
+  assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0);
+  clock += KARTVERKET_FAILURE_BACKOFF_MS - CCTV_SOURCE_CACHE_MS - 1;
   assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0);
   assert.equal(fetchMock.mock.callCount(), 1);
 
@@ -494,6 +615,143 @@ test('after a failure no lookup starts until the backoff ends', async (t) => {
   assert.equal(fetchMock.mock.callCount(), 2);
 });
 
+test('a 429 backs off for its Retry-After, not the default', async (t) => {
+  withHeightsEnv(t);
+  let clock = 1_000_000;
+  const now = () => clock;
+  const retryAfterS = 50 * 60;
+  const fetchMock = t.mock.method(
+    globalThis,
+    'fetch',
+    async () =>
+      new Response('busy', {
+        status: 429,
+        headers: { 'Retry-After': String(retryAfterS) },
+      }),
+  );
+  await applyVegvesenGroundHeights([cameraAt(0)], { now });
+  assert.equal(fetchMock.mock.callCount(), 1);
+  clock += KARTVERKET_FAILURE_BACKOFF_MS + 1;
+  await applyVegvesenGroundHeights([cameraAt(0)], { now });
+  assert.equal(fetchMock.mock.callCount(), 1);
+  clock += retryAfterS * 1000 - KARTVERKET_FAILURE_BACKOFF_MS;
+  await applyVegvesenGroundHeights([cameraAt(0)], { now });
+  assert.equal(fetchMock.mock.callCount(), 2);
+});
+
+test('a lookup cut short by its deadline resumes on the next refresh', async (t) => {
+  withHeightsEnv(t);
+  let clock = 1_000_000;
+  const now = () => clock;
+  t.mock.method(globalThis, 'fetch', hangingFetch);
+  // The lookup runs out of time; that is not a failing service.
+  assert.equal(
+    await applyVegvesenGroundHeights([cameraAt(0)], {
+      now,
+      timeoutMs: 10,
+      waitMs: 1000,
+    }),
+    0,
+  );
+  const fetchMock = t.mock.method(globalThis, 'fetch', async (url) =>
+    kartverketResponse(url, () => 64),
+  );
+  clock += CCTV_SOURCE_CACHE_MS;
+  const cameras = [cameraAt(0)];
+  assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 1);
+  assert.equal(cameras[0].groundElevationM, 64);
+  assert.equal(fetchMock.mock.callCount(), 1);
+});
+
+test('overlapping refreshes share one lookup', async (t) => {
+  withHeightsEnv(t);
+  let release;
+  const gate = new Promise((resolve) => {
+    release = resolve;
+  });
+  const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => {
+    await gate;
+    return kartverketResponse(url, () => 3);
+  });
+  await Promise.all([
+    applyVegvesenGroundHeights([cameraAt(0)], { waitMs: 5 }),
+    applyVegvesenGroundHeights([cameraAt(0)], { waitMs: 5 }),
+  ]);
+  assert.equal(fetchMock.mock.callCount(), 1);
+  release();
+});
+
+test('a warm cache does not wait for a few new cameras', async (t) => {
+  withHeightsEnv(t);
+  const cameras = Array.from({ length: 20 }, (_, i) => cameraAt(i));
+  t.mock.method(globalThis, 'fetch', async (url) =>
+    kartverketResponse(url, () => 500),
+  );
+  assert.equal(await applyVegvesenGroundHeights(cameras), 20);
+  let release;
+  const gate = new Promise((resolve) => {
+    release = resolve;
+  });
+  const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => {
+    await gate;
+    return kartverketResponse(url, () => 600);
+  });
+  const refresh = [...cameras, cameraAt(20)];
+  const started = Date.now();
+  assert.equal(
+    await applyVegvesenGroundHeights(refresh, { waitMs: 5_000 }),
+    20,
+  );
+  assert.ok(Date.now() - started < 1000);
+  assert.equal(fetchMock.mock.callCount(), 1);
+  // The background lookup fills the cache for the next refresh.
+  release();
+  await new Promise((resolve) => setTimeout(resolve, 20));
+  const next = [...cameras, cameraAt(20)];
+  assert.equal(await applyVegvesenGroundHeights(next), 21);
+  assert.equal(next[20].groundElevationM, 600);
+});
+
+test('cameras without numeric coordinates are skipped, never thrown on', async (t) => {
+  withHeightsEnv(t);
+  t.mock.method(globalThis, 'fetch', async (url) =>
+    kartverketResponse(url, () => 9),
+  );
+  const cameras = [
+    { id: 'no-coordinates' },
+    null,
+    { id: 'strings', lat: '60', lon: '7' },
+    cameraAt(0),
+  ];
+  assert.equal(await applyVegvesenGroundHeights(cameras), 1);
+  assert.equal(cameras[3].groundElevationM, 9);
+});
+
+test('a lookup started before a cache clear cannot write after it', async (t) => {
+  withHeightsEnv(t);
+  let release;
+  const gate = new Promise((resolve) => {
+    release = resolve;
+  });
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    await gate;
+    return kartverketResponse(url, () => 111);
+  });
+  await applyVegvesenGroundHeights([cameraAt(0)], { waitMs: 5 });
+  clearKartverketHeightCache();
+  t.mock.method(globalThis, 'fetch', async (url) =>
+    kartverketResponse(url, () => 222),
+  );
+  const fresh = [cameraAt(0)];
+  assert.equal(await applyVegvesenGroundHeights(fresh), 1);
+  assert.equal(fresh[0].groundElevationM, 222);
+  release();
+  await new Promise((resolve) => setTimeout(resolve, 20));
+  const later = [cameraAt(0)];
+  await applyVegvesenGroundHeights(later, { waitMs: 0 });
+  assert.equal(later[0].groundElevationM, 222);
+});
+
 test('CCTV_VEGVESEN_HEIGHTS=0 skips the lookup', async (t) => {
   withHeightsEnv(t);
   const fetchMock = t.mock.method(globalThis, 'fetch', async (url) =>
@@ -541,3 +799,24 @@ test('the loader applies Kartverket heights to the cameras it returns', async (t
     [['no-vegvesen-3000063_1', 987.6]],
   );
 });
+
+test('the loader budgets the height wait from its own start', async (t) => {
+  withHeightsEnv(t);
+  const realNow = Date.now.bind(Date);
+  let offset = 0;
+  t.mock.method(Date, 'now', () => realNow() + offset);
+  t.mock.method(globalThis, 'fetch', (url, init) => {
+    if (String(url).startsWith(KARTVERKET_HEIGHT_URL)) {
+      return hangingFetch(url, init);
+    }
+    // The catalog fetch uses up all but 10 ms of the lane's budget.
+    offset += KARTVERKET_CATALOG_WAIT_MS - 10;
+    return Promise.resolve(
+      Response.json({ type: 'FeatureCollection', features: [feature()] }),
+    );
+  });
+  const started = performance.now();
+  const cameras = await loadVegvesenSourcesFromOpenData();
+  assert.ok(performance.now() - started < 1000);
+  assert.equal(cameras[0].groundElevationM, 150);
+});

src/data/dataCredits.js

@@ -295,7 +295,8 @@ export const DATA_CREDITS = [
       '<a href="https://data.norge.no/nlod/en/2.0" target="_blank" rel="noopener">Norwegian licence for Open Government data (NLOD)</a> ' +
       'distributed by Statens vegvesen · camera ground heights © ' +
       '<a href="https://kartverket.no/en/api-and-data/terrengdata" target="_blank" rel="noopener">Kartverket</a> ' +
-      '(<a href="https://creativecommons.org/licenses/by/4.0/" target="_blank" rel="noopener">CC BY 4.0</a>)',
+      '(<a href="https://creativecommons.org/licenses/by/4.0/" target="_blank" rel="noopener">CC BY 4.0</a>); ' +
+      'modified: heights over water set to 0 m and rounded to 0.1 m',
   },
   {
     key: 'nyc-cctv',