Merge remote-tracking branch 'origin/pr945' into codex/cctv-shared-batch-20261019

server/providers/cctv/sources.js

@@ -91,6 +91,17 @@ import {
   DEFAULT_VEGVESEN_MAX_SOURCES,
   VEGVESEN_MAX_CATALOG_BYTES,
   NORWAY_ANCHORS,
+  KARTVERKET_HEIGHT_URL,
+  KARTVERKET_MAX_POINTS,
+  KARTVERKET_CONCURRENCY,
+  KARTVERKET_LOOKUP_TIMEOUT_MS,
+  KARTVERKET_CATALOG_WAIT_MS,
+  KARTVERKET_NULL_TTL_MS,
+  KARTVERKET_FAILURE_BACKOFF_MS,
+  KARTVERKET_MAX_BACKOFF_MS,
+  KARTVERKET_USER_AGENT,
+  KARTVERKET_MAX_RESPONSE_BYTES,
+  KARTVERKET_HEIGHT_RANGE_M,
   NYC_CAMERAS_URL,
   NYC_IMAGE_URL,
   NYC_CAMERA_ID_RE,
@@ -2457,14 +2468,11 @@ export function vegvesenCameraToSource(feature) {
     fovDeg: 44,
     rangeM: 145,
     mountHeightM: 8,
-    // KNOWN LIMITATION: the feed's coordinates are 2D, so this is one flat
-    // prior for the whole country, while cameras run from sea level to
-    // mountain passes near 1,000 m (Haukelifjell, Sjonfjellet, Rugeldalen).
-    // The point-height prior and the client's one-shot ground snap correct it
-    // where they resolve; on a stack where neither does (no 3D tiles), a pass
-    // camera stays hundreds of metres below the terrain, the same risk the
-    // Caltrans pack documents. A per-camera height (e.g. Kartverket's keyless
-    // point-height API) would remove it.
+    // The feed's coordinates are 2D. The loader replaces this flat prior with
+    // Kartverket's terrain height for each camera (applyVegvesenGroundHeights);
+    // it only survives when that lookup fails, and then a mountain-pass camera
+    // can sit below the terrain on a stack without 3D tiles, the same risk the
+    // Caltrans pack documents.
     groundElevationM: 150,
     feedType: videoUrl ? 'hls' : 'image',
     url: videoUrl || imageUrl,
@@ -2484,6 +2492,7 @@ export function vegvesenCameraToSource(feature) {
  * @returns {Promise<Array<object>>} Normalized camera source objects.
  */
 export async function loadVegvesenSourcesFromOpenData() {
+  const startedAt = Date.now();
   try {
     const endpoint = process.env.CCTV_VEGVESEN_URL || DEFAULT_VEGVESEN_CCTV_URL;
     const resp = await fetch(endpoint, {
@@ -2529,6 +2538,12 @@ export async function loadVegvesenSourcesFromOpenData() {
       ? Math.max(8, Math.min(1000, Math.floor(maxRaw)))
       : DEFAULT_VEGVESEN_MAX_SOURCES;
     const prioritized = prioritizeSources(cameras, maxCount, NORWAY_ANCHORS);
+    await applyVegvesenGroundHeights(prioritized, {
+      waitMs: Math.max(
+        0,
+        KARTVERKET_CATALOG_WAIT_MS - (Date.now() - startedAt),
+      ),
+    });
     console.log(
       `[CCTV] Loaded Vegvesen camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
     );
@@ -2542,6 +2557,271 @@ export async function loadVegvesenSourcesFromOpenData() {
   }
 }
 
+/**
+ * Kartverket answers by rounded position: `{z, expires}`, where z is metres or
+ * null for "no height there". Heights never expire (camera positions barely
+ * change); nulls expire after KARTVERKET_NULL_TTL_MS. The 15-minute catalog
+ * refresh therefore only asks about new or moved cameras. Bounded well above
+ * the ~900 cameras the pack can load.
+ */
+const kartverketHeightCache = new Map();
+const KARTVERKET_CACHE_LIMIT = 5000;
+/** The lookup in flight, so overlapping catalog loads never start a second. */
+let kartverketPending = null;
+/** No lookup starts before this time (epoch ms) after a transport failure. */
+let kartverketBackoffUntil = 0;
+
+/** Cache key for one position, at the ~0.1 m precision the request sends. */
+const kartverketKey = (lon, lat) => `${lon.toFixed(6)},${lat.toFixed(6)}`;
+
+/** Forget cached heights, the backoff and any lookup in flight. Tests only. */
+export function clearKartverketHeightCache() {
+  kartverketHeightCache.clear();
+  kartverketPending = null;
+  kartverketBackoffUntil = 0;
+}
+
+/**
+ * How long to back off after a failed request: the Retry-After the service
+ * sent (seconds or an HTTP date), else the default, bounded to [1 min, 1 h].
+ *
+ * @param {?string} retryAfter - The Retry-After header, if any.
+ * @param {number} now - Epoch ms.
+ * @returns {number} Milliseconds.
+ */
+export function kartverketBackoffMs(retryAfter, now = Date.now()) {
+  let ms = KARTVERKET_FAILURE_BACKOFF_MS;
+  const text = String(retryAfter ?? '').trim();
+  if (/^\d+$/.test(text)) ms = Number(text) * 1000;
+  else if (text && Number.isFinite(Date.parse(text))) {
+    ms = Date.parse(text) - now;
+  }
+  return Math.max(60 * 1000, Math.min(KARTVERKET_MAX_BACKOFF_MS, ms));
+}
+
+/**
+ * Terrain heights above sea level (NN2000) from Kartverket's keyless
+ * point-height API, KARTVERKET_MAX_POINTS points per request. Each answer is
+ * matched back by the coordinates the API echoes, and only for points that
+ * request asked about, then reported through `into.set(key, z)` as its
+ * request lands, so a caller that stops waiting keeps what has arrived. `z` is
+ * metres, or null when the API has no usable height there.
+ *
+ * Over water `/punkt` returns the depth of the sea floor (`datakilde:
+ * dybdekurver`, `terreng: Havflate`; −547 m under Nordhordlandsbrua). Either
+ * marker counts as 0 m, the water surface. For the ~14 bridge cameras that is
+ * below the deck, but far closer than the sea floor or a flat prior.
+ *
+ * The first transport failure (a non-200 answer, a network error or the
+ * shared deadline) stops the lookup: no further requests start, and the
+ * failure is returned so the caller can back off. Points in failed requests
+ * are not reported, so they are asked again after the backoff.
+ *
+ * @param {Array<{lon:number, lat:number}>} points
+ * @param {{into?: {set: Function}, timeoutMs?: number}} [options]
+ * @returns {Promise<{heights: Map<string, ?number>, failure: ?{status: ?number, retryAfter: ?string}}>}
+ *   `heights` is `into` when it is a Map, else a Map of what was reported.
+ */
+export async function fetchKartverketHeights(
+  points,
+  { into = new Map(), timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS } = {},
+) {
+  const heights = into instanceof Map ? into : new Map();
+  const report = (key, z) => {
+    if (heights !== into) heights.set(key, z);
+    into.set(key, z);
+  };
+  const unique = Array.from(
+    new Map(
+      (points || [])
+        .filter((p) => isLikelyNorwayCoordinate(p?.lat, p?.lon))
+        .map((p) => [kartverketKey(p.lon, p.lat), p]),
+    ).values(),
+  );
+  let failure = null;
+  if (!unique.length) return { heights, failure };
+  const batches = [];
+  for (let i = 0; i < unique.length; i += KARTVERKET_MAX_POINTS) {
+    batches.push(unique.slice(i, i + KARTVERKET_MAX_POINTS));
+  }
+  const signal = AbortSignal.timeout(timeoutMs);
+  const fetchBatch = async (batch) => {
+    const asked = new Set(batch.map((p) => kartverketKey(p.lon, p.lat)));
+    const url = new URL(KARTVERKET_HEIGHT_URL);
+    url.searchParams.set(
+      'punkter',
+      JSON.stringify(
+        batch.map((p) => [Number(p.lon.toFixed(6)), Number(p.lat.toFixed(6))]),
+      ),
+    );
+    url.searchParams.set('koordsys', '4258');
+    const resp = await fetch(url, {
+      headers: {
+        Accept: 'application/json',
+        'User-Agent': KARTVERKET_USER_AGENT,
+      },
+      redirect: 'error',
+      signal,
+    });
+    if (!resp.ok) {
+      try {
+        await resp.body?.cancel();
+      } catch {
+        /* no-op */
+      }
+      const error = new Error(`HTTP ${resp.status}`);
+      error.status = resp.status;
+      error.retryAfter = resp.headers.get('retry-after');
+      throw error;
+    }
+    const payload = await readResponseJsonCapped(
+      resp,
+      KARTVERKET_MAX_RESPONSE_BYTES,
+    );
+    for (const point of Array.isArray(payload?.punkter)
+      ? payload.punkter
+      : []) {
+      const x = toFiniteNumber(point?.x);
+      const y = toFiniteNumber(point?.y);
+      if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
+      const key = kartverketKey(x, y);
+      // Only the points this request asked about: an odd answer can neither
+      // grow the cache nor overwrite another camera.
+      if (!asked.has(key)) continue;
+      asked.delete(key);
+      // `z: null` means no height there; Number(null) would read 0.
+      const raw = point?.z;
+      let z =
+        raw === null || raw === undefined || raw === ''
+          ? NaN
+          : toFiniteNumber(raw);
+      if (
+        Number.isFinite(z) &&
+        (point?.datakilde === 'dybdekurver' || point?.terreng === 'Havflate')
+      ) {
+        z = 0;
+      }
+      const usable =
+        Number.isFinite(z) &&
+        z >= KARTVERKET_HEIGHT_RANGE_M.min &&
+        z <= KARTVERKET_HEIGHT_RANGE_M.max;
+      report(key, usable ? z : null);
+    }
+  };
+  let next = 0;
+  const worker = async () => {
+    while (next < batches.length && !failure) {
+      const batch = batches[next++];
+      try {
+        await fetchBatch(batch);
+      } catch (error) {
+        if (!failure) {
+          failure = {
+            status: Number.isFinite(error?.status) ? error.status : null,
+            retryAfter: error?.retryAfter ?? null,
+          };
+          console.warn(
+            '[CCTV] Kartverket height lookup failed:',
+            error?.message || error,
+          );
+        }
+      }
+    }
+  };
+  await Promise.all(
+    Array.from(
+      { length: Math.min(KARTVERKET_CONCURRENCY, batches.length) },
+      worker,
+    ),
+  );
+  return { heights, failure };
+}
+
+/**
+ * Give each Vegvesen camera its own terrain height as the ground prior, in
+ * place. Waits at most `waitMs` for positions it has no answer for; a lookup
+ * still running then keeps filling the cache for the next catalog refresh.
+ * Cameras without a height keep the flat prior. After a transport failure no
+ * lookup starts until the backoff ends, so a failing Kartverket costs one
+ * attempt, not one per refresh. Never throws. CCTV_VEGVESEN_HEIGHTS=0 turns it
+ * off.
+ *
+ * @param {Array<object>} cameras - Sources from vegvesenCameraToSource().
+ * @param {{waitMs?: number, now?: () => number}} [options]
+ * @returns {Promise<number>} How many cameras got a terrain height.
+ */
+export async function applyVegvesenGroundHeights(
+  cameras,
+  { waitMs = KARTVERKET_CATALOG_WAIT_MS, now = Date.now } = {},
+) {
+  if (String(process.env.CCTV_VEGVESEN_HEIGHTS || '1').trim() === '0') return 0;
+  const list = Array.isArray(cameras) ? cameras : [];
+  const startedAt = now();
+  const missing = list.filter((c) => {
+    const entry = kartverketHeightCache.get(kartverketKey(c.lon, c.lat));
+    return !entry || entry.expires <= startedAt;
+  });
+  if (
+    missing.length &&
+    !kartverketPending &&
+    startedAt >= kartverketBackoffUntil
+  ) {
+    if (kartverketHeightCache.size + missing.length > KARTVERKET_CACHE_LIMIT) {
+      kartverketHeightCache.clear();
+    }
+    const sink = {
+      set: (key, z) =>
+        kartverketHeightCache.set(key, {
+          z,
+          expires: z === null ? now() + KARTVERKET_NULL_TTL_MS : Infinity,
+        }),
+    };
+    kartverketPending = fetchKartverketHeights(missing, { into: sink })
+      .then(({ failure }) => {
+        if (failure) {
+          kartverketBackoffUntil =
+            now() + kartverketBackoffMs(failure.retryAfter, now());
+        }
+      })
+      .catch((error) => {
+        kartverketBackoffUntil = now() + KARTVERKET_FAILURE_BACKOFF_MS;
+        console.warn(
+          '[CCTV] Kartverket height lookup error:',
+          error?.message || error,
+        );
+      })
+      .finally(() => {
+        kartverketPending = null;
+      });
+  }
+  if (missing.length && kartverketPending && waitMs > 0) {
+    let timer;
+    await Promise.race([
+      kartverketPending,
+      new Promise((resolve) => {
+        timer = setTimeout(resolve, waitMs);
+        timer.unref?.();
+      }),
+    ]);
+    clearTimeout(timer);
+  }
+  let applied = 0;
+  for (const camera of list) {
+    const z = kartverketHeightCache.get(
+      kartverketKey(camera.lon, camera.lat),
+    )?.z;
+    if (!Number.isFinite(z)) continue;
+    camera.groundElevationM = Math.round(z * 10) / 10;
+    applied += 1;
+  }
+  if (list.length) {
+    console.log(
+      `[CCTV] Kartverket ground heights: ${applied}/${list.length} Vegvesen cameras`,
+    );
+  }
+  return applied;
+}
+
 const NYC_BOROUGHS = new Set([
   'Manhattan',
   'Brooklyn',