gods-eye-view/server/providers/cctv/normalize.js

import {
  normalizeFeedType,
  isVideoFeedType,
} from '../../../src/sources/cctvTypes.js';
export { normalizeFeedType, isVideoFeedType };
import { directionToHeading } from '../../../src/data/directionText.js';
import { haversineKm } from '../common/geo.js';
/**
 * FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random
 * values (e.g. hue for synthetic SVG billboards, fallback heading angles).
 *
 * @param {string} text
 * @returns {number} Unsigned 32-bit hash.
 */
export function hashSeed(text) {
  let h = 2166136261 >>> 0; // FNV offset basis
  for (let i = 0; i < text.length; i++) {
    h ^= text.charCodeAt(i);
    h = Math.imul(h, 16777619); // FNV prime
  }
  return h >>> 0;
}

/**
 * Escape special XML/HTML characters for safe embedding in SVG text nodes.
 *
 * @param {string} text
 * @returns {string}
 */
export function escapeXml(text) {
  return String(text || '')
    .replace(/&/g, '&amp;')
    .replace(/</g, '&lt;')
    .replace(/>/g, '&gt;')
    .replace(/"/g, '&quot;')
    .replace(/'/g, '&#39;');
}

/**
 * Coerce a value to a finite number, returning fallback if NaN/Infinity.
 *
 * Absent, blank and boolean values take the fallback too: `Number` reads
 * null, '' and false as 0, which would turn a missing coordinate, heading or
 * elevation into a real zero.
 *
 * @param {*} value
 * @param {number} [fallback=NaN]
 * @returns {number}
 */
export function toFiniteNumber(value, fallback = NaN) {
  if (
    value == null ||
    typeof value === 'boolean' ||
    (typeof value === 'string' && !value.trim())
  )
    return fallback;
  const num = Number(value);
  return Number.isFinite(num) ? num : fallback;
}

/**
 * Normalize a column/field name to a lowercase snake_case key.
 *
 * @param {string} text
 * @returns {string}
 */
export function normalizeKey(text) {
  return String(text || '')
    .trim()
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '_')
    .replace(/^_+|_+$/g, '');
}

/**
 * Parse a WKT POINT string (e.g. "POINT(-97.74 30.27)") into lat/lon.
 *
 * WKT uses (lon lat) order; returned object uses {lat, lon}.
 *
 * @param {string} value
 * @returns {{lat:number, lon:number}}
 */
export function parsePointString(value) {
  const match = String(value || '').match(
    /POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
  );
  if (!match) return { lat: NaN, lon: NaN };
  return {
    lon: toFiniteNumber(match[1]),
    lat: toFiniteNumber(match[2]),
  };
}

/**
 * Extract lat/lon from a variety of coordinate representations.
 *
 * Handles WKT POINT strings, and objects with latitude/lat/y or
 * longitude/lon/lng/x properties (various casing).
 *
 * @param {string|object|null} value
 * @returns {{lat:number, lon:number}}
 */
export function coerceLatLon(value) {
  if (!value) return { lat: NaN, lon: NaN };

  if (typeof value === 'string') {
    return parsePointString(value);
  }

  if (typeof value !== 'object') {
    return { lat: NaN, lon: NaN };
  }

  const lat = toFiniteNumber(
    value.latitude ?? value.lat ?? value.y ?? value.Latitude ?? value.Lat,
    NaN,
  );
  const lon = toFiniteNumber(
    value.longitude ??
      value.lon ??
      value.lng ??
      value.x ??
      value.Longitude ??
      value.Lon,
    NaN,
  );
  return { lat, lon };
}

/**
 * Extract geographic coordinates from an Austin Open Data camera record.
 *
 * Tries several candidate fields (location, coordinates, the_geom,
 * point, geocoded_column) via coerceLatLon, then falls back to
 * explicit latitude/longitude scalar fields.
 *
 * @param {object} record - Flattened camera record.
 * @returns {{lat:number, lon:number}}
 */
export function extractAustinCoords(record) {
  const candidates = [
    record.location,
    record.coordinates,
    record.the_geom,
    record.point,
    record.geocoded_column,
  ];
  for (const candidate of candidates) {
    const parsed = coerceLatLon(candidate);
    if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon))
      return parsed;
  }

  const lat = toFiniteNumber(
    record.latitude ??
      record.lat ??
      record.camera_latitude ??
      record.location_latitude,
    NaN,
  );
  const lon = toFiniteNumber(
    record.longitude ??
      record.lon ??
      record.lng ??
      record.camera_longitude ??
      record.location_longitude,
    NaN,
  );
  return { lat, lon };
}

/**
 * Extract a numeric camera ID from an Austin Open Data record.
 *
 * Tries well-known field names first, then scans any field whose key
 * contains "camera"/"cam"/"device" + "id".
 *
 * @param {object} record - Flattened camera record.
 * @returns {string} Numeric ID string, or '' if none found.
 */
export function extractAustinCameraId(record) {
  const preferredKeys = [
    'camera_id',
    'cameraid',
    'cam_id',
    'device_id',
    'intersection_id',
    'id',
  ];
  for (const key of preferredKeys) {
    const value = record[key];
    if (value == null) continue;
    const asText = String(value).trim();
    if (!asText) continue;
    if (/^\d+$/.test(asText)) return asText;
  }

  for (const [key, value] of Object.entries(record)) {
    if (!/camera|cam|device/.test(key)) continue;
    if (!/id/.test(key)) continue;
    const asText = String(value || '').trim();
    if (!asText) continue;
    if (/^\d+$/.test(asText)) return asText;
  }

  return '';
}

/**
 * Extract a human-readable camera name from an Austin record.
 *
 * @param {object} record - Flattened camera record.
 * @param {string} cameraId - Fallback identifier if no name field found.
 * @returns {string}
 */
export function extractAustinName(record, cameraId) {
  const preferredKeys = [
    'camera_name',
    'location_name',
    'intersection_name',
    'location',
    'cross_street',
    'description',
    'name',
  ];
  for (const key of preferredKeys) {
    const value = record[key];
    if (typeof value !== 'string') continue;
    const text = value.trim();
    if (text) return text;
  }
  return `Austin Camera ${cameraId}`;
}

/**
 * Extract camera heading (compass bearing) from an Austin record.
 *
 * Tries explicit numeric heading fields first, then direction-keyword
 * fields, then infers from the camera name/description text.
 *
 * @param {object} record - Flattened camera record.
 * @returns {number} Heading in degrees [0..360), or NaN if unknown.
 */
export function extractAustinHeading(record) {
  const direct = toFiniteNumber(
    record.heading_deg ?? record.heading ?? record.bearing,
    NaN,
  );
  if (Number.isFinite(direct)) return ((direct % 360) + 360) % 360;

  // Dedicated direction fields: bare cardinal words ("West") are real facings.
  const directionKeys = [
    'direction',
    'travel_direction',
    'facing',
    'facing_direction',
  ];
  for (const key of directionKeys) {
    const heading = directionToHeading(record[key], true);
    if (Number.isFinite(heading)) return heading;
  }

  // Free-form name/intersection text: only explicit travel forms ("WESTBOUND"/
  // "WB") count — a bare "West" here is a street name ("5TH ST / WEST AVE"), not
  // a facing, and must not promote the camera to a false high-confidence heading.
  const nameProbe = [
    record.camera_name,
    record.location_name,
    record.intersection_name,
    record.location,
    record.cross_street,
    record.description,
    record.name,
  ]
    .filter(Boolean)
    .join(' ');
  const inferred = directionToHeading(nameProbe);
  if (Number.isFinite(inferred)) return inferred;

  return NaN;
}

/**
 * Bounding-box sanity check: is this coordinate plausibly in the Austin metro area?
 *
 * @param {number} lat
 * @param {number} lon
 * @returns {boolean}
 */
export function isLikelyAustinCoordinate(lat, lon) {
  if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
  return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.4;
}

/**
 * Bounding-box sanity check: is this coordinate plausibly on the Finnish road
 * network? Generous around the observed catalog extent (59.86..70.09 N,
 * 19.62..31.28 E) so a real new station is never dropped, tight enough that a
 * swapped lat/lon or a null island record is.
 *
 * @param {number} lat
 * @param {number} lon
 * @returns {boolean}
 */
/** Longest unselected camera label the HUD shows before it gets noisy. */
export const CAMERA_CODE_MAX_CHARS = 28;

/**
 * Short display code for the unselected camera label ("CAM-<code>"): the
 * feed's own name for the camera ("5TH ST / CONGRESS AVE", "TRAFALGAR
 * SQUARE"), trimmed to CAMERA_CODE_MAX_CHARS. A pack may pass an explicit
 * `code` (TxDOT's device key, NSW's title); the id is the last resort.
 *
 * @param {string} text
 * @returns {string}
 */
export function cameraDisplayCode(text) {
  const clean = String(text || '')
    .replace(/\s+/g, ' ')
    .trim();
  if (clean.length <= CAMERA_CODE_MAX_CHARS) return clean;
  return `${clean.slice(0, CAMERA_CODE_MAX_CHARS - 1).trimEnd()}…`;
}

/** Finite, in range, and not the null island that Number(null) produces. */
export function isPlausibleLatLon(lat, lon) {
  return (
    Number.isFinite(lat) &&
    Number.isFinite(lon) &&
    Math.abs(lat) <= 90 &&
    Math.abs(lon) <= 180 &&
    !(lat === 0 && lon === 0)
  );
}

/** British Columbia bounding box (with the neighbouring border crossings). */
export function isLikelyBcCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 48 &&
    lat <= 60.5 &&
    lon >= -139.5 &&
    lon <= -114
  );
}

/** Texas bounding box. */
export function isLikelyTexasCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 25.5 &&
    lat <= 36.7 &&
    lon >= -107 &&
    lon <= -93.4
  );
}

/** New South Wales bounding box (incl. the ACT and Lord Howe Island). */
export function isLikelyNswCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= -38 &&
    lat <= -28 &&
    lon >= 140.9 &&
    lon <= 159.2
  );
}

/** Calgary's municipal extent, with slack for the ring road. */
export function isLikelyCalgaryCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 50.8 &&
    lat <= 51.25 &&
    lon >= -114.4 &&
    lon <= -113.8
  );
}

/** New York City's five boroughs, with slack for the harbor and city line. */
export function isLikelyNycCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 40.45 &&
    lat <= 40.95 &&
    lon >= -74.3 &&
    lon <= -73.65
  );
}

/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 57.9 &&
    lat <= 71.3 &&
    lon >= 4.4 &&
    lon <= 31.3
  );
}

export function isLikelyFinlandCoordinate(lat, lon) {
  if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
  return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
}

/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */
export function isLikelyCataloniaCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 40.3 &&
    lat <= 42.9 &&
    lon >= 0.0 &&
    lon <= 3.4
  );
}

/**
 * Decode numeric HTML character references, decimal ("&#237;") and hex
 * ("&#xED;") alike, the entity form the Catalonia feed uses for accented
 * place names, plus the handful of named entities XML/HTML text commonly
 * carries. A reference that names no valid character (NUL, a surrogate, or
 * anything past U+10FFFF) is left as written rather than thrown on, so one
 * bad field cannot fail a whole catalog.
 *
 * @param {string} text
 * @returns {string}
 */
export function decodeNumericEntities(text) {
  return String(text || '')
    .replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => {
      const code = hex ? Number.parseInt(hex, 16) : Number(dec);
      const valid =
        code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);
      return valid ? String.fromCodePoint(code) : match;
    })
    .replace(/&amp;/g, '&')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .trim();
}

/**
 * Human label for one Fintraffic preset (camera view).
 *
 * Station names are machine-shaped road codes ("vt3_Hyvinkää_Noppo"); the
 * underscores become spaces. A preset id is always its station id plus a
 * two-digit view number, so the remainder distinguishes the several views that
 * share one station position. The station list endpoint carries no
 * presentationName ("Helsinkiin"); that lives only on the per-station detail
 * endpoint, which would cost one request per station.
 *
 * @param {string} stationName - Raw `properties.name`.
 * @param {string} stationId - Raw `properties.id` (e.g. "C01503").
 * @param {string} presetId - Raw preset id (e.g. "C0150301").
 * @returns {string}
 */
export function fintrafficCameraName(stationName, stationId, presetId) {
  const base =
    String(stationName || '')
      .replace(/_/g, ' ')
      .trim() || `Fintraffic ${stationId}`;
  const view = String(presetId || '').slice(String(stationId || '').length);
  return view ? `${base} (view ${view})` : base;
}

/**
 * Derive a deterministic fallback heading from a camera ID hash.
 *
 * Produces one of 16 evenly-spaced compass directions (0, 22.5, 45, ...).
 *
 * @param {string} cameraId
 * @returns {number} Heading in degrees [0..360).
 */
export function fallbackHeadingFromId(cameraId) {
  return (hashSeed(String(cameraId)) % 16) * 22.5;
}

/**
 * Convert a Socrata rows.json array row into a keyed object using column metadata.
 *
 * @param {Array} row - Array of cell values from the Socrata payload.
 * @param {Array<{fieldName?:string, name?:string}>} columns - Column descriptors.
 * @returns {object} Keyed record with normalized snake_case keys.
 */
export function rowArrayToObject(row, columns) {
  const record = {};
  for (let idx = 0; idx < columns.length; idx++) {
    const col = columns[idx];
    const key = normalizeKey(col.fieldName || col.name || `col_${idx}`);
    if (!key) continue;
    record[key] = row[idx];
  }
  return record;
}

/**
 * Distance-prioritizes cameras to a cap: keeps the maxCount cameras closest
 * to ANY of the given anchor points (min distance over anchors), tie-broken
 * by original array order. Used by every live source pack (Austin: one
 * downtown anchor; Caltrans: one anchor per major CA metro; TfL: central
 * London) so a cap always keeps the densest, most interesting cores.
 *
 * @param {Array<object>} cameras - Normalized camera source objects.
 * @param {number} maxCount - Cap (<=0 or >= length disables).
 * @param {Array<{lat:number,lon:number}>} anchors - At least one anchor.
 * @returns {Array<object>} Capped, priority-ordered camera list.
 */
export function prioritizeSources(cameras, maxCount, anchors) {
  const list = Array.isArray(cameras) ? cameras : [];
  const anchorList = (Array.isArray(anchors) ? anchors : []).filter(
    (a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon),
  );
  if (!anchorList.length) return list;
  // Always sort when anchors exist, even when the pack fits its own cap: the
  // catalog-wide cap (cap.js) thins a pack from the END of this order, so
  // "nearest to an anchor first" has to hold whether or not the pack was
  // trimmed here.
  const cap =
    Number.isFinite(maxCount) && maxCount > 0
      ? Math.min(maxCount, list.length)
      : list.length;

  const scored = list.map((camera, idx) => {
    const lat = Number(camera?.lat);
    const lon = Number(camera?.lon);
    const distKm =
      Number.isFinite(lat) && Number.isFinite(lon)
        ? Math.min(
            ...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon)),
          )
        : Number.POSITIVE_INFINITY;
    return { camera, idx, distKm };
  });

  scored.sort((a, b) => {
    if (a.distKm !== b.distKm) return a.distKm - b.distKm;
    return a.idx - b.idx;
  });

  const kept = scored.slice(0, cap).map((entry) => entry.camera);
  // How many cameras the pack offered before trimming, and where, for
  // coverage reports.
  Object.defineProperty(kept, 'available', { value: list.length });
  Object.defineProperty(kept, 'region', { value: cameraRegion(list) });
  return kept;
}

/** The bounding box of cameras with coordinates, or null. */
export function cameraRegion(cameras) {
  let region = null;
  for (const camera of cameras || []) {
    const lat = Number(camera?.lat);
    const lon = Number(camera?.lon);
    if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
    region = region
      ? {
          west: Math.min(region.west, lon),
          south: Math.min(region.south, lat),
          east: Math.max(region.east, lon),
          north: Math.max(region.north, lat),
        }
      : { west: lon, south: lat, east: lon, north: lat };
  }
  return region;
}

/**
 * Normalize a raw CCTV source item into a canonical shape with safe defaults.
 *
 * @param {object} item - Raw source from file, env, or Austin Open Data.
 * @returns {object} Normalized source with all expected fields populated.
 */
export function normalizeSourceItem(item) {
  return {
    id: String(item.id || '').trim(),
    name: String(item.name || item.id || '').trim(),
    city: String(item.city || ''),
    cityId: String(item.cityId || ''),
    provider: String(item.provider || 'Configured CCTV Source'),
    lat: toFiniteNumber(item.lat),
    lon: toFiniteNumber(item.lon),
    headingDeg: toFiniteNumber(item.headingDeg),
    headingConfidence: String(
      item.headingConfidence || item.headingSource || '',
    ).toLowerCase(),
    pitchDeg: toFiniteNumber(item.pitchDeg),
    fovDeg: toFiniteNumber(item.fovDeg),
    rangeM: toFiniteNumber(item.rangeM),
    mountHeightM: toFiniteNumber(item.mountHeightM),
    groundElevationM: toFiniteNumber(item.groundElevationM),
    feedType: normalizeFeedType(item.feedType || item.type || ''),
    url: typeof item.url === 'string' ? item.url : '',
    snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '',
    license: String(item.license || item.licenseNote || ''),
    // Per-camera attribution for feeds a partner supplies inside a pack
    // (DriveBC: TransLink, city cameras). Shown beside the provider.
    credit: String(item.credit || '').trim(),
    // Unselected-label code: the pack's explicit short name, else the feed's
    // name, else the id.
    code: cameraDisplayCode(
      item.code || String(item.name || '').toUpperCase() || item.id || '',
    ),
    sourceKind: String(item.sourceKind || item.kind || 'configured'),
    // Optional CAL badge input (cctv-v2 design §3b/§9.2, additive-only per the
    // global constraints — nothing else in this file changes): hand-authored
    // file/env catalog entries may declare poseSource:'curated' so the panel
    // badge can distinguish them from raw automated priors (e.g. Austin Open
    // Data, which never sets this field). Passed through as-is to the client.
    poseSource: item.poseSource === 'curated' ? 'curated' : undefined,
    // Optional hardware-model passthrough (additive, like poseSource): packs
    // whose catalog publishes the camera's hardware model — King County's
    // layer carries Manufacturer/Model per camera — pass it through so
    // identified-hardware consumers can use it. Absent everywhere else.
    model:
      typeof item.model === 'string' && item.model.trim()
        ? item.model.trim()
        : undefined,
    // Bare model strings collide across vendors (e.g. "3950"), so the
    // publishing pack's manufacturer rides along for disambiguation.
    manufacturer:
      typeof item.manufacturer === 'string' && item.manufacturer.trim()
        ? item.manufacturer.trim()
        : undefined,
  };
}

/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */
export function isLikelyTorontoCoordinate(lat, lon) {
  return (
    isPlausibleLatLon(lat, lon) &&
    lat >= 43.55 &&
    lat <= 43.9 &&
    lon >= -79.7 &&
    lon <= -79.1
  );
}