Merge upstream main into fix/tarktee-datex-catalog-hardening

Resolve conflicts with #1234: keep redirect: 'manual' on both DATEX
fetches and keep main's 30 s TARKTEE_FETCH_TIMEOUT_MS. Drop this
branch's 15 s constant and its import, which auto-merged into a
duplicate export. Keep both sets of Estonia tests and drop the 15 s
claim from the changelog fragment.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/providers/cctv/catalog.js

@@ -25,6 +25,7 @@ import {
   loadVegvesenSourcesFromOpenData,
   loadNycSourcesFromOpenData,
   loadTorontoSourcesFromOpenData,
+  loadFloridaSourcesFromOpenData,
 } from './sources.js';
 
 /** Env kill switch: unset or anything but "0" means enabled. */
@@ -144,6 +145,14 @@ const LIVE_PACKS = [
     enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'),
     load: loadLithuaniaSources,
   },
+  {
+    // FL511's terms limit its content to individual, personal use (other use
+    // needs FDOT's written consent), so the pack is off unless the operator
+    // opts in (see DATA_SOURCES.md).
+    name: 'florida',
+    enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
+    load: loadFloridaSourcesFromOpenData,
+  },
 ];
 /**
  * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
@@ -195,16 +204,19 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
   /** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
    * callers so a post-TTL burst launches ONE refetch, not one per request. */
   let _cctvSourceInflight = null;
-  /** @type {Array<{pack: string, available: number, served: number}>} Packs
-   * that serve fewer cameras than they offer. */
-  let _cctvTrimmedPacks = [];
+  /** @type {WeakMap<Array<object>, Array<{pack: string, available: number, served: number}>>}
+   * Packs that serve fewer cameras than they offer, keyed by the source list
+   * they describe, so a request served the old list during a background
+   * refresh never pairs it with the new list's trim notes. */
+  const _cctvTrimmedPacks = new WeakMap();
 
   /**
    * Assemble and cache the merged CCTV source list.
    *
    * Merges every source pack (live open-data packs, local file, env
    * variable), deduplicates by ID, shares the catalog cap fairly across
-   * packs, and caches for CCTV_SOURCE_CACHE_MS.
+   * packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is
+   * returned while a background refresh replaces it; only a first load waits.
    *
    * @returns {Promise<Array<object>>} Deduplicated, capped source list.
    */
@@ -219,11 +231,31 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
     // Single-flight: a burst of requests arriving past the TTL shares ONE refresh
     // instead of each launching the full multi-provider refetch. The `.finally`
     // clears the ref so the next post-TTL cycle starts fresh.
-    if (_cctvSourceInflight) return _cctvSourceInflight;
-    _cctvSourceInflight = refreshCctvSources().finally(() => {
-      _cctvSourceInflight = null;
-    });
-    return _cctvSourceInflight;
+    if (!_cctvSourceInflight) {
+      _cctvSourceInflight = refreshCctvSources().finally(() => {
+        _cctvSourceInflight = null;
+      });
+      // A background refresh has no caller waiting on it: report its failure
+      // here, once per refresh rather than once per request it serves. Keep
+      // the old catalog for another TTL, as an empty refresh does, so a refresh
+      // that keeps failing is retried once per TTL, not back to back.
+      if (_cctvSourceCache.length) {
+        _cctvSourceInflight.catch((error) => {
+          _cctvSourceCacheAt = Date.now();
+          console.warn(
+            '[CCTV] background catalog refresh failed; serving',
+            _cctvSourceCache.length,
+            'cameras from the last catalog:',
+            error?.message || error,
+          );
+        });
+      }
+    }
+    // With a catalog in hand, answer with it while the refresh runs: the
+    // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up
+    // to its 30 s timeout), and every /api/cctv request would otherwise wait
+    // for it. Only the first load, with nothing to serve yet, waits.
+    return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight;
   }
 
   /**
@@ -313,7 +345,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
     }
     if (capped.length > 0 || _cctvSourceCache.length === 0) {
       _cctvSourceCache = capped;
-      _cctvTrimmedPacks = trimmedPacks;
+      _cctvTrimmedPacks.set(capped, trimmedPacks);
     } else {
       // Every source came back empty (all live packs timed out / upstream outage)
       // but a good catalog is already cached — serve it stale rather than blanking
@@ -327,7 +359,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
     return _cctvSourceCache;
   }
 
-  /** Packs serving fewer cameras than they offer, as of the last refresh. */
-  getCctvSources.trimmedPacks = () => _cctvTrimmedPacks;
+  /** Packs serving fewer cameras than they offer in `sources` (by default the
+   * current catalog). Pass the list a request was served to keep the two in
+   * step while a background refresh swaps the catalog. */
+  getCctvSources.trimmedPacks = (sources = _cctvSourceCache) =>
+    _cctvTrimmedPacks.get(sources) ?? [];
   return getCctvSources;
 }

server/providers/cctv/constants.js

@@ -113,6 +113,27 @@ export const CALTRANS_ANCHORS = [
   { lat: 32.7157, lon: -117.1611 }, // San Diego
   { lat: 38.5816, lon: -121.4944 }, // Sacramento
 ];
+export const FLORIDA_CCTV_URL =
+  'https://services.arcgis.com/3wFbqsFPLeKqOlIK/arcgis/rest/services/FL511_Traffic_Cameras/FeatureServer/0/query';
+export const FLORIDA_IMAGE_HOSTS = [
+  'images-dis.divas.cloud',
+  'snapshots.divas.cloud',
+];
+export const DEFAULT_FLORIDA_MAX_SOURCES = 500;
+/** Byte cap per catalog page (2,000 rows, about 650 KiB measured). */
+export const FLORIDA_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
+/** Catalog page ceiling: 40 pages covers 4,000+ cameras even at 100 rows a page. */
+export const FLORIDA_MAX_CATALOG_PAGES = 40;
+export const FLORIDA_ANCHORS = [
+  { lat: 25.7617, lon: -80.1918 }, // Miami
+  { lat: 28.5384, lon: -81.3789 }, // Orlando
+  { lat: 27.9506, lon: -82.4572 }, // Tampa
+  { lat: 30.3322, lon: -81.6557 }, // Jacksonville
+  { lat: 30.4383, lon: -84.2807 }, // Tallahassee
+  { lat: 26.1224, lon: -80.1373 }, // Fort Lauderdale
+  { lat: 26.7153, lon: -80.0534 }, // West Palm Beach
+  { lat: 30.4213, lon: -87.2169 }, // Pensacola (panhandle)
+];
 /** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */
 export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam';
 export const TFL_IMAGE_ORIGIN =
@@ -190,9 +211,6 @@ export const TARKTEE_IMAGE_ORIGIN = 'https://tarktee.transpordiamet.ee/images/';
 export const DEFAULT_TARKTEE_MAX_SOURCES = 179;
 /** Hard ceiling per Tarktee DATEX XML document (~120–165 KB live). */
 export const TARKTEE_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
-/** Two DATEX XML documents download concurrently from the same origin (~5.3 s
- * baseline) alongside the other packs, so the 8 s default can abort mid-stream. */
-export const TARKTEE_FETCH_TIMEOUT_MS = 15_000;
 export const TARKTEE_ANCHORS = [
   { lat: 59.437, lon: 24.753 }, // Tallinn
   { lat: 58.378, lon: 26.729 }, // Tartu
@@ -444,6 +462,10 @@ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
  * pending forever — a hung fetch aborts, the loader returns [], and
  * serve-stale/other packs take over. */
 export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000;
+/** Tarktee's two DATEX feeds (120 and 160 KB) answered in 2 to 16.6 s across
+ * ten requests on 2026-10-18, so the shared 15 s timeout dropped the Estonian
+ * pack on about a third of loads. Both must arrive for any camera to load. */
+export const TARKTEE_FETCH_TIMEOUT_MS = 30 * 1000;
 /** Individual CCTV image fetches must settle before the active 10-second
  * client refresh cadence. A bounded miss can fall through to Street View or
  * the synthetic frame instead of leaving the browser preview pending. */

server/providers/cctv/normalize.js

@@ -377,6 +377,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
   );
 }
 
+/** Florida statewide, Key West to Georgia state line, with slack at each edge */
+export function isLikelyFloridaCoordinate(lat, lon) {
+  return (
+    isPlausibleLatLon(lat, lon) &&
+    lat >= 24.3 &&
+    lat <= 31.1 &&
+    lon >= -87.7 &&
+    lon <= -79.8
+  );
+}
+
 /** New York City's five boroughs, with slack for the harbor and city line. */
 export function isLikelyNycCoordinate(lat, lon) {
   return (

server/providers/cctv/sources.js

@@ -43,7 +43,6 @@ import {
   TARKTEE_IMAGE_ORIGIN,
   DEFAULT_TARKTEE_MAX_SOURCES,
   TARKTEE_MAX_CATALOG_BYTES,
-  TARKTEE_FETCH_TIMEOUT_MS,
   TARKTEE_ANCHORS,
   DEFAULT_WARENDORF_SOURCE_FILE,
   WARENDORF_IMAGE_ORIGINS,
@@ -57,6 +56,12 @@ import {
   DEFAULT_CALGARY_MAX_SOURCES,
   CALGARY_DOWNTOWN,
   CALGARY_MAX_CATALOG_BYTES,
+  FLORIDA_CCTV_URL,
+  FLORIDA_IMAGE_HOSTS,
+  DEFAULT_FLORIDA_MAX_SOURCES,
+  FLORIDA_ANCHORS,
+  FLORIDA_MAX_CATALOG_BYTES,
+  FLORIDA_MAX_CATALOG_PAGES,
   CATALONIA_CAMERAS_URL,
   CATALONIA_MAX_CATALOG_BYTES,
   CATALONIA_IMAGE_HOSTS,
@@ -72,6 +77,7 @@ import {
   TORONTO_GROUND_ELEVATION_M,
   TORONTO_MAX_CATALOG_BYTES,
   CCTV_SOURCE_FETCH_TIMEOUT_MS,
+  TARKTEE_FETCH_TIMEOUT_MS,
   DELDOT_CCTV_URL,
   DEFAULT_DELDOT_MAX_SOURCES,
   DELDOT_ANCHORS,
@@ -115,6 +121,7 @@ import {
   isLikelyTexasCoordinate,
   isLikelyNswCoordinate,
   isLikelyCalgaryCoordinate,
+  isLikelyFloridaCoordinate,
   isLikelyCataloniaCoordinate,
   decodeNumericEntities,
   isLikelyNycCoordinate,
@@ -2826,3 +2833,203 @@ export async function loadTorontoSourcesFromOpenData() {
     return [];
   }
 }
+
+/** FL511 DIRECTION codes as headings; "NOT DIRECTIONAL" has none. */
+const FLORIDA_DIRECTION_HEADINGS = Object.freeze({
+  N: 0,
+  E: 90,
+  S: 180,
+  W: 270,
+});
+
+/**
+ * Stable camera id from a frame URL. FL511's ID column repeats across regions
+ * (ID 1220 is both an I-75 and an I-95 camera), so the image channel is the
+ * key, as Calgary keys on its frame filename. Falls back to a path slug.
+ *
+ * @param {string} imageUrl - A host-pinned FL511 frame URL.
+ * @returns {?string} Provider-stable id, or null when underivable.
+ */
+export function floridaCameraId(imageUrl) {
+  let path;
+  try {
+    path = new URL(String(imageUrl ?? '')).pathname;
+  } catch {
+    return null;
+  }
+  const channel = /\/chan-(\d+)_[a-z]+\.jpg$/i.exec(path);
+  if (channel) return `fl-${channel[1]}`;
+  const slug = path
+    .replace(/^\/+|\.[a-z0-9]+$/gi, '')
+    .replace(/[^a-zA-Z0-9]+/g, '-')
+    .toLowerCase();
+  return slug ? `fl-${slug}` : null;
+}
+
+/**
+ * One FL511 ArcGIS feature -> one catalog source, or null. DIRECTION is a
+ * one-letter code the free-text parser won't read, so a table maps it.
+ *
+ * @param {object} feature - ArcGIS feature from the FL511 camera layer.
+ * @returns {?object}
+ */
+export function floridaCameraToSource(feature) {
+  const a = feature?.attributes;
+  if (!a || typeof a !== 'object') return null;
+  const lat = toFiniteNumber(a.LATITUDE);
+  const lon = toFiniteNumber(a.LONGITUDE);
+  if (!isLikelyFloridaCoordinate(lat, lon)) return null;
+
+  const imageUrl = String(a.IMAGE || '');
+  let image;
+  try {
+    image = new URL(imageUrl);
+  } catch {
+    return null;
+  }
+  // Official image hosts only, HTTPS, no embedded credentials.
+  if (
+    image.protocol !== 'https:' ||
+    !FLORIDA_IMAGE_HOSTS.includes(image.host) ||
+    image.username ||
+    image.password
+  )
+    return null;
+
+  const cameraId = floridaCameraId(imageUrl);
+  if (!cameraId) return null;
+
+  const heading =
+    FLORIDA_DIRECTION_HEADINGS[
+      String(a.DIRECTION || '')
+        .trim()
+        .toUpperCase()
+    ];
+  const hasHeading = Number.isFinite(heading);
+
+  return {
+    id: cameraId,
+    name: String(a.DESCRIPT || '').trim() || `FL511 ${cameraId.slice(3)}`,
+    city: String(a.COUNTY ? `${a.COUNTY} County` : 'Florida'),
+    cityId: 'florida',
+    provider: 'FL511',
+    lat,
+    lon,
+    headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
+    headingConfidence: hasHeading ? 'high' : 'low',
+    pitchDeg: hasHeading ? -24 : -18,
+    fovDeg: hasHeading ? 56 : 44,
+    rangeM: hasHeading ? 210 : 145,
+    mountHeightM: hasHeading ? 10 : 8,
+    groundElevationM: 5, // Coastal prior; the client's ground snap corrects.
+    feedType: 'image',
+    url: imageUrl,
+    snapshotUrl: imageUrl,
+    sourceKind: 'fl511-open-data',
+    license:
+      'FL511 (FDOT), individual personal use only; other use needs FDOT written consent',
+  };
+}
+
+/**
+ * Fetch FL511 (Florida DOT) cameras, keyless: an ArcGIS FeatureServer paged
+ * up to 2,000 rows at a time, ordered by the layer's object id so offsets are
+ * stable. Paging follows `exceededTransferLimit` and advances by the rows each
+ * page returned, so a lower server-side max record count still reads the whole
+ * layer. Frames are stills on the Divas Cloud hosts. A redirect on any page
+ * drops the pack; any other failed page keeps the rows already read.
+ *
+ * @returns {Promise<Array<object>>} Normalized camera source objects.
+ */
+export async function loadFloridaSourcesFromOpenData() {
+  try {
+    const cameras = [];
+    const seen = new Set();
+    const pageSize = 2000;
+    let offset = 0;
+    // Page ceiling, in case the service ignores resultOffset.
+    for (let page = 0; ; page += 1) {
+      if (page >= FLORIDA_MAX_CATALOG_PAGES) {
+        console.warn(
+          `[CCTV] Florida catalog stopped at the ${FLORIDA_MAX_CATALOG_PAGES}-page ceiling; the catalog may be incomplete`,
+        );
+        break;
+      }
+      // OBJECTID_1 is this layer's object id field (a plain OBJECTID is a 400).
+      const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&orderByFields=OBJECTID_1&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
+      let resp;
+      try {
+        resp = await fetch(url, {
+          headers: { Accept: 'application/json' },
+          redirect: 'manual',
+          signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
+        });
+      } catch (error) {
+        console.warn(
+          '[CCTV] Florida catalog page failed:',
+          error?.message || error,
+        );
+        break;
+      }
+      // Release the body on every path that will not read it.
+      const discard = async () => {
+        try {
+          await resp.body?.cancel();
+        } catch {
+          /* no-op */
+        }
+      };
+      if (resp.status >= 300 && resp.status < 400) {
+        console.warn(
+          '[CCTV] Florida catalog redirected; redirects are not followed',
+        );
+        await discard();
+        return [];
+      }
+      if (!resp.ok) {
+        console.warn('[CCTV] Florida camera download failed:', resp.status);
+        await discard();
+        break;
+      }
+      let body;
+      try {
+        body = await readResponseJsonCapped(resp, FLORIDA_MAX_CATALOG_BYTES);
+      } catch (error) {
+        console.warn(
+          '[CCTV] Florida catalog page unreadable:',
+          error?.message || error,
+        );
+        break;
+      }
+      const feats = Array.isArray(body?.features) ? body.features : [];
+      for (const feature of feats) {
+        const camera = floridaCameraToSource(feature);
+        if (!camera || seen.has(camera.id)) continue;
+        seen.add(camera.id);
+        cameras.push(camera);
+      }
+      // ArcGIS sets exceededTransferLimit while more rows remain. A page may
+      // be shorter than pageSize when the layer's max record count is lower.
+      if (body?.exceededTransferLimit !== true || feats.length === 0) break;
+      offset += feats.length;
+    }
+
+    const maxRaw = Number(
+      process.env.CCTV_FLORIDA_MAX_SOURCES || DEFAULT_FLORIDA_MAX_SOURCES,
+    );
+    const maxCount = Number.isFinite(maxRaw)
+      ? Math.max(8, Math.min(4200, Math.floor(maxRaw)))
+      : DEFAULT_FLORIDA_MAX_SOURCES;
+    const prioritized = prioritizeSources(cameras, maxCount, FLORIDA_ANCHORS);
+    console.log(
+      `[CCTV] Loaded Florida camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
+    );
+    return prioritized;
+  } catch (error) {
+    console.warn(
+      '[CCTV] Florida camera download error:',
+      error?.message || error,
+    );
+    return [];
+  }
+}