fix: address review on Kartverket ground heights

- Send 20 points per request, not 50: above ~20 Kartverket answers the
  extra points from contour lines (hoydekurver) or not at all. At 20,
  nearly every camera resolves from dtm1, so the null-retry pass is gone.
  Eight requests in flight keep a cold lookup ~6 s.
- Cache misses: positions with no height are not asked again for 6 h.
- Transport failures: the first 429, 5xx or network error stops the
  lookup with no retries, and no lookup starts until the backoff ends
  (Retry-After when sent, else 10 min; bounded to 1 min-1 h).
- Budget the 12 s wait from the start of the loader, catalog fetch
  included.
- Record answers only for the points each request asked about.
- Send the CCTV User-Agent; link CC BY 4.0 in the credit.
- Docs: bridge cameras sit at the water surface, below the deck.
- Tests: batching, 8-in-flight cap, request count on failure, the shared
  deadline, Retry-After bounds, null TTL, backoff, the -20 m bound and
  each water marker on its own.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/data/cctvVegvesen.test.mjs

@@ -5,14 +5,18 @@ import {
   applyVegvesenGroundHeights,
   clearKartverketHeightCache,
   fetchKartverketHeights,
+  kartverketBackoffMs,
   loadVegvesenSourcesFromOpenData,
   vegvesenCameraToSource,
 } from '../../server/providers/cctv/sources.js';
 import {
   DEFAULT_VEGVESEN_CCTV_URL,
   KARTVERKET_HEIGHT_URL,
+  KARTVERKET_CONCURRENCY,
+  KARTVERKET_FAILURE_BACKOFF_MS,
   KARTVERKET_MAX_POINTS,
-  KARTVERKET_RETRY_POINTS,
+  KARTVERKET_NULL_TTL_MS,
+  KARTVERKET_USER_AGENT,
   VEGVESEN_IMAGE_ORIGIN,
 } from '../../server/providers/cctv/constants.js';
 import { createCctvCatalog } from '../../server/providers/cctv/catalog.js';
@@ -244,145 +248,261 @@ const withHeightsEnv = (t) => {
   t.mock.method(console, 'warn', () => {});
 };
 
-test('Kartverket heights come in batches of 50, matched by echoed coordinates', async (t) => {
+const sentPoints = (url) =>
+  JSON.parse(new URL(url).searchParams.get('punkter'));
+
+test('Kartverket heights come in batches of 20, matched by echoed coordinates', async (t) => {
   withHeightsEnv(t);
   const requested = [];
-  t.mock.method(globalThis, 'fetch', async (url) => {
+  t.mock.method(globalThis, 'fetch', async (url, init) => {
     requested.push(String(url));
     assert.ok(String(url).startsWith(KARTVERKET_HEIGHT_URL));
+    assert.equal(init.headers['User-Agent'], KARTVERKET_USER_AGENT);
     // Reply in reverse order: matching must not depend on list position.
     const response = kartverketResponse(url, (x) =>
-      x > 7.05 ? null : Math.round((x - 7) * 10000),
+      x > 7.04 ? null : Math.round((x - 7) * 10000),
     );
     const body = await response.json();
     body.punkter.reverse();
     return Response.json(body);
   });
-  const points = Array.from({ length: 120 }, (_, i) => cameraAt(i));
-  const heights = await fetchKartverketHeights(points);
-  const sizes = requested.map(
-    (url) => JSON.parse(new URL(url).searchParams.get('punkter')).length,
+  const { heights, failure } = await fetchKartverketHeights(
+    Array.from({ length: 50 }, (_, i) => cameraAt(i)),
   );
-  // Three full batches, then one retry pass over the 69 points with no height.
-  assert.deepEqual(sizes.slice(0, 3), [KARTVERKET_MAX_POINTS, 50, 20]);
-  assert.equal(
-    sizes.slice(3).reduce((a, b) => a + b, 0),
-    69,
+  assert.equal(failure, null);
+  // One pass, no retries: 20 + 20 + 10.
+  assert.deepEqual(
+    requested.map((url) => sentPoints(url).length),
+    [KARTVERKET_MAX_POINTS, 20, 10],
   );
-  assert.ok(sizes.slice(3).every((n) => n <= KARTVERKET_RETRY_POINTS));
   for (const url of requested) {
     assert.equal(new URL(url).searchParams.get('koordsys'), '4258');
   }
-  // Points past x = 7.05 have no height (outside coverage) and are left out.
-  assert.equal(heights.size, 51);
+  assert.equal(heights.size, 50);
   assert.equal(heights.get('7.010000,60.010000'), 100);
+  // Past x = 7.04 the API has no height: recorded as null, never as 0.
+  assert.equal(heights.get('7.049000,60.049000'), null);
 });
 
-test('points a full batch answers null for are retried once in small batches', async (t) => {
-  withHeightsEnv(t);
-  const sizes = [];
-  t.mock.method(globalThis, 'fetch', async (url) => {
-    const points = JSON.parse(new URL(url).searchParams.get('punkter'));
-    sizes.push(points.length);
-    // A full batch drops the first two points; a small batch answers them,
-    // except the one point that has no data at all.
-    const full = points.length > KARTVERKET_RETRY_POINTS;
-    return kartverketResponse(url, (x) => {
-      if (Math.abs(x - 7.002) < 1e-9) return null;
-      if (full && x < 7.002) return null;
-      return 500;
-    });
-  });
-  const heights = await fetchKartverketHeights(
-    Array.from({ length: 60 }, (_, i) => cameraAt(i)),
-  );
-  assert.deepEqual(sizes, [50, 10, 3]);
-  assert.equal(heights.size, 59);
-  assert.equal(heights.get('7.000000,60.000000'), 500);
-  assert.equal(heights.has('7.002000,60.002000'), false);
-});
-
-test('a sea-floor depth counts as the water surface', async (t) => {
+test('only the points a request asked about are recorded', async (t) => {
   withHeightsEnv(t);
   t.mock.method(globalThis, 'fetch', async (url) => {
-    const points = JSON.parse(new URL(url).searchParams.get('punkter'));
+    const points = sentPoints(url);
     return Response.json({
       koordsys: 4258,
-      punkter: points.map(([x, y], i) =>
-        i === 0
-          ? { x, y, z: -547.1, datakilde: 'dybdekurver', terreng: 'Havflate' }
-          : { x, y, z: -0.2, datakilde: 'dtm1', terreng: 'ÅpentOmråde' },
-      ),
+      punkter: [
+        ...points.map(([x, y]) => ({ x, y, z: 10 })),
+        { x: 7.5, y: 60.5, z: 999 }, // never asked
+      ],
     });
   });
-  const heights = await fetchKartverketHeights([cameraAt(0), cameraAt(1)]);
-  // Under a bridge: the surface, not the sea floor. A quay just below the
-  // datum on land data stays as measured.
-  assert.equal(heights.get('7.000000,60.000000'), 0);
-  assert.equal(heights.get('7.001000,60.001000'), -0.2);
+  const { heights } = await fetchKartverketHeights([cameraAt(0)]);
+  assert.deepEqual([...heights.entries()], [['7.000000,60.000000', 10]]);
 });
 
-test('implausible heights and points outside Norway are ignored', async (t) => {
+test('either water marker counts as the surface', async (t) => {
+  withHeightsEnv(t);
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    const [a, b, c] = sentPoints(url);
+    return Response.json({
+      koordsys: 4258,
+      punkter: [
+        {
+          x: a[0],
+          y: a[1],
+          z: -547.1,
+          datakilde: 'dybdekurver',
+          terreng: null,
+        },
+        { x: b[0], y: b[1], z: -9.2, datakilde: 'dtm1', terreng: 'Havflate' },
+        {
+          x: c[0],
+          y: c[1],
+          z: -0.2,
+          datakilde: 'dtm1',
+          terreng: 'ÅpentOmråde',
+        },
+      ],
+    });
+  });
+  const { heights } = await fetchKartverketHeights([
+    cameraAt(0),
+    cameraAt(1),
+    cameraAt(2),
+  ]);
+  assert.equal(heights.get('7.000000,60.000000'), 0);
+  assert.equal(heights.get('7.001000,60.001000'), 0);
+  // A quay just below the datum on land data stays as measured.
+  assert.equal(heights.get('7.002000,60.002000'), -0.2);
+});
+
+test('heights outside -20..2500 m and points outside Norway are not used', async (t) => {
   withHeightsEnv(t);
   const requested = [];
   t.mock.method(globalThis, 'fetch', async (url) => {
     requested.push(String(url));
-    return kartverketResponse(url, () => 99999);
+    const zs = [-20, -20.5, 2500, 2500.5];
+    const points = sentPoints(url);
+    return Response.json({
+      koordsys: 4258,
+      punkter: points.map(([x, y], i) => ({ x, y, z: zs[i] })),
+    });
   });
-  const heights = await fetchKartverketHeights([
+  const { heights } = await fetchKartverketHeights([
+    cameraAt(0),
     cameraAt(1),
+    cameraAt(2),
+    cameraAt(3),
     { lat: 55.68, lon: 12.57 }, // Copenhagen: never sent
   ]);
-  assert.equal(heights.size, 0);
-  // The first pass and its one retry, each carrying only the Norway point.
-  assert.equal(requested.length, 2);
-  for (const url of requested) {
-    assert.equal(
-      JSON.parse(new URL(url).searchParams.get('punkter')).length,
-      1,
-    );
-  }
+  assert.equal(requested.length, 1);
+  assert.equal(sentPoints(requested[0]).length, 4);
+  assert.deepEqual(
+    [0, 1, 2, 3].map((i) =>
+      heights.get(
+        `${(7 + i * 0.001).toFixed(6)},${(60 + i * 0.001).toFixed(6)}`,
+      ),
+    ),
+    [-20, null, 2500, null],
+  );
 });
 
-test('cameras get their terrain height, and a refresh reuses the cache', async (t) => {
+test('no more than eight requests are in flight', async (t) => {
+  withHeightsEnv(t);
+  let inFlight = 0;
+  let peak = 0;
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    inFlight += 1;
+    peak = Math.max(peak, inFlight);
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    inFlight -= 1;
+    return kartverketResponse(url, () => 1);
+  });
+  const { heights } = await fetchKartverketHeights(
+    Array.from({ length: 400 }, (_, i) => cameraAt(i)),
+  );
+  assert.equal(heights.size, 400);
+  assert.equal(peak, KARTVERKET_CONCURRENCY);
+});
+
+test('a transport failure stops the lookup: no retries, no new requests', async (t) => {
   withHeightsEnv(t);
   let calls = 0;
-  t.mock.method(globalThis, 'fetch', async (url) => {
+  t.mock.method(globalThis, 'fetch', async () => {
     calls += 1;
+    return new Response('busy', {
+      status: 429,
+      headers: { 'Retry-After': '120' },
+    });
+  });
+  const { heights, failure } = await fetchKartverketHeights(
+    Array.from({ length: 400 }, (_, i) => cameraAt(i)),
+  );
+  // The first wave is already in flight when the first failure lands; nothing
+  // after it starts, and no point is resent.
+  assert.equal(calls, KARTVERKET_CONCURRENCY);
+  assert.equal(heights.size, 0);
+  assert.deepEqual(failure, { status: 429, retryAfter: '120' });
+});
+
+test('the lookup deadline bounds a hanging service', async (t) => {
+  withHeightsEnv(t);
+  t.mock.method(
+    globalThis,
+    'fetch',
+    (url, init) =>
+      new Promise((_, reject) => {
+        init.signal.addEventListener('abort', () => reject(init.signal.reason));
+      }),
+  );
+  const started = Date.now();
+  const { heights, failure } = await fetchKartverketHeights([cameraAt(0)], {
+    timeoutMs: 30,
+  });
+  assert.ok(Date.now() - started < 2000);
+  assert.equal(heights.size, 0);
+  assert.equal(failure.status, null);
+});
+
+test('Retry-After sets the backoff, bounded to [1 min, 1 h]', () => {
+  const now = Date.parse('2026-10-06T12:00:00Z');
+  assert.equal(kartverketBackoffMs('120', now), 120_000);
+  assert.equal(kartverketBackoffMs('5', now), 60_000);
+  assert.equal(kartverketBackoffMs('86400', now), 3_600_000);
+  assert.equal(
+    kartverketBackoffMs('Mon, 06 Oct 2026 12:05:00 GMT', now),
+    300_000,
+  );
+  assert.equal(kartverketBackoffMs(null, now), KARTVERKET_FAILURE_BACKOFF_MS);
+  assert.equal(kartverketBackoffMs('soon', now), KARTVERKET_FAILURE_BACKOFF_MS);
+});
+
+test('heights are cached, and misses are not asked again until they expire', async (t) => {
+  withHeightsEnv(t);
+  let clock = 1_000_000;
+  const now = () => clock;
+  const asked = [];
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    asked.push(sentPoints(url).length);
     return kartverketResponse(url, (x) => (x < 7.002 ? 1043.26 : null));
   });
   const cameras = [cameraAt(0), cameraAt(1), cameraAt(5)];
-  assert.equal(await applyVegvesenGroundHeights(cameras), 2);
+  assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 2);
   assert.deepEqual(
     cameras.map((c) => c.groundElevationM),
     [1043.3, 1043.3, 150],
   );
-  // One full pass plus one retry for the point with no height.
-  assert.equal(calls, 2);
-  // Same positions on the next catalog refresh: answered from the cache.
-  const again = [cameraAt(0), cameraAt(1)];
-  assert.equal(await applyVegvesenGroundHeights(again), 2);
-  assert.equal(calls, 2);
+  // The next refresh asks nothing: heights and the miss are both cached.
+  assert.equal(
+    await applyVegvesenGroundHeights([cameraAt(0), cameraAt(1), cameraAt(5)], {
+      now,
+    }),
+    2,
+  );
+  assert.deepEqual(asked, [3]);
+  // Once the miss expires, only that point is asked again.
+  clock += KARTVERKET_NULL_TTL_MS + 1;
+  await applyVegvesenGroundHeights([cameraAt(0), cameraAt(5)], { now });
+  assert.deepEqual(asked, [3, 1]);
 });
 
-test('a failed or disabled lookup keeps the flat prior', async (t) => {
+test('after a failure no lookup starts until the backoff ends', async (t) => {
   withHeightsEnv(t);
+  let clock = 1_000_000;
+  const now = () => clock;
   const fetchMock = t.mock.method(globalThis, 'fetch', async () => {
     throw new Error('offline');
   });
   const cameras = [cameraAt(0)];
-  assert.equal(await applyVegvesenGroundHeights(cameras), 0);
+  assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 0);
   assert.equal(cameras[0].groundElevationM, 150);
+  assert.equal(fetchMock.mock.callCount(), 1);
 
-  fetchMock.mock.mockImplementation(
-    async () => new Response('nope', { status: 503 }),
+  // Refreshes inside the backoff cost nothing.
+  clock += KARTVERKET_FAILURE_BACKOFF_MS - 1;
+  assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0);
+  assert.equal(fetchMock.mock.callCount(), 1);
+
+  // After it, the failed point is asked again.
+  clock += 2;
+  fetchMock.mock.mockImplementation(async (url) =>
+    kartverketResponse(url, () => 321),
   );
+  const later = [cameraAt(0)];
+  assert.equal(await applyVegvesenGroundHeights(later, { now }), 1);
+  assert.equal(later[0].groundElevationM, 321);
+  assert.equal(fetchMock.mock.callCount(), 2);
+});
+
+test('CCTV_VEGVESEN_HEIGHTS=0 skips the lookup', async (t) => {
+  withHeightsEnv(t);
+  const fetchMock = t.mock.method(globalThis, 'fetch', async (url) =>
+    kartverketResponse(url, () => 5),
+  );
+  process.env.CCTV_VEGVESEN_HEIGHTS = '0';
+  const cameras = [cameraAt(0)];
   assert.equal(await applyVegvesenGroundHeights(cameras), 0);
   assert.equal(cameras[0].groundElevationM, 150);
-
-  process.env.CCTV_VEGVESEN_HEIGHTS = '0';
-  fetchMock.mock.resetCalls();
-  assert.equal(await applyVegvesenGroundHeights(cameras), 0);
   assert.equal(fetchMock.mock.callCount(), 0);
 });