From 73336b297e2994d4741be5fc46bb36ec096ffbb6 Mon Sep 17 00:00:00 2001 From: Jerry Lai <419vive@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:24:20 +0800 Subject: [PATCH 1/3] feat(cctv): add Taiwan Freeway Bureau camera pack Read ~1,800 national freeway cameras at runtime from the Freeway Bureau's keyless MOTC 2.0 CCTV.xml table. Each published MJPEG stream maps to the same camera server's single-JPEG path; only those exact HTTPS pairs on the four freeway.gov.tw camera servers are registered. On by default, CCTV_TAIWAN_ENABLED=0 skips it. Co-Authored-By: Claude Opus 5.5 --- .env.example | 2 + DATA_SOURCES.md | 1 + README.md | 2 +- changelog.d/taiwan-freeway-cctv.md | 4 + docs/CURRENT-STATE.md | 16 ++ scripts/format-scope.json | 2 + server/providers/cctv/catalog.js | 6 + server/providers/cctv/taiwan.js | 249 +++++++++++++++++++++++++++++ src/data/cctvTaiwan.test.mjs | 200 +++++++++++++++++++++++ src/data/dataCredits.js | 7 + 10 files changed, 488 insertions(+), 1 deletion(-) create mode 100644 changelog.d/taiwan-freeway-cctv.md create mode 100644 server/providers/cctv/taiwan.js create mode 100644 src/data/cctvTaiwan.test.mjs diff --git a/.env.example b/.env.example index a2f8afa24..ae89e4df9 100644 --- a/.env.example +++ b/.env.example @@ -237,6 +237,8 @@ AISSTREAM_API_KEY= # CCTV_TORONTO_LIST_URL= # override the Toronto RESCU camera list URL # CCTV_LITHUANIA_ENABLED= # off by default; set 1 to load Via Lietuva / Eismoinfo road cameras (photo licence unconfirmed; see DATA_SOURCES.md) # CCTV_LITHUANIA_MAX_SOURCES=400 # cap Lithuanian cameras; no API key required +# CCTV_TAIWAN_ENABLED=1 # set 0 to skip Taiwan Freeway Bureau (MOTC) road cameras +# CCTV_TAIWAN_MAX_SOURCES=600 # cap Taiwan freeway cameras (1-2000); no API key required # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index d930bcb8f..a05cd5511 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -110,6 +110,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei **Quebec is thinly represented**, and that is upstream rather than a filter here: the province runs its own Info-Smog program, so only a handful of Quebec stations appear in ECCC's national feed. The reporting station count also varies hour to hour as stations report or fall silent. - **Via Lietuva / Eismoinfo (Lithuania).** Public road-camera catalogs and periodically updated still images from [Eismoinfo](https://eismoinfo.lt/), credited to Via Lietuva / eismoinfo.lt in the app. **Off by default.** The camera endpoints are the eismoinfo.lt website backend and are not among the services listed in Via Lietuva's [open-data statement](https://eismoinfo.lt/eismoinfo-backend/content?id=44), whose conditions are attribution and reuse on the same terms. Until the statement is confirmed to cover the camera photos, the pack loads only when an operator sets `CCTV_LITHUANIA_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to Eismoinfo. `eismoinfo-backend/camera-info-table` is joined by ID to `layer-static-features/VKR`. Without `lks=true`, the latter returns WGS84 `[latitude, longitude]`; the table's `x/y` instead use LKS-94 metres. Only the official HTTPS `/eismoinfo-backend/image-provider/camera/last?id=…` image endpoint is registered. Catalogs use the shared 15-minute cache; frames are fetched on demand and never bundled or archived. These are periodic snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. `CCTV_LITHUANIA_MAX_SOURCES` defaults to 400. +- **Freeway Bureau, MOTC (Taiwan).** National freeway cameras from the Freeway Bureau's keyless [CCTV static table](https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml) in the MOTC 2.0 standard, refreshed daily and fetched at runtime (never bundled). Freeway Bureau traffic data is published under the [Open Government Data License, Taiwan (Version 1.0)](https://data.gov.tw/license), which permits reuse with attribution; the app credits the Freeway Bureau, MOTC. The table publishes each camera's MJPEG stream on one of four official camera servers (`cctvn`, `cctvn5`, `cctvc`, `cctvs` under `freeway.gov.tw`). The frame proxy relays single images, so each published stream URL is mapped to the same server's single-JPEG path for the same camera number (`/abs2mjpg/bmjpg` → `/abs2mjpg/jpg`, `/live-view/mjpg/video.cgi` → `/live-view/jpg/image.cgi`); only those exact HTTPS pairs are registered, and the still path is not itself listed in the table. These are on-demand snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. On by default; `CCTV_TAIWAN_ENABLED=0` skips the pack, and `CCTV_TAIWAN_MAX_SOURCES` defaults to 600 (bounded to 1–2000). - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. diff --git a/README.md b/README.md index c49c53ece..c91edecce 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~4,000 public cameras (shared cap) projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Toronto · King County (WA) · Washington (WSDOT) · Catalonia (Servei Català de Trànsit), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement) and opt-in Lithuania (Via Lietuva / Eismoinfo, photo licence unconfirmed). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | ~4,000 public cameras (shared cap) projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Toronto · King County (WA) · Washington (WSDOT) · Catalonia (Servei Català de Trànsit) · Taiwan (Freeway Bureau, MOTC), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement) and opt-in Lithuania (Via Lietuva / Eismoinfo, photo licence unconfirmed). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/changelog.d/taiwan-freeway-cctv.md b/changelog.d/taiwan-freeway-cctv.md new file mode 100644 index 000000000..7fc0339e3 --- /dev/null +++ b/changelog.d/taiwan-freeway-cctv.md @@ -0,0 +1,4 @@ +- Add Taiwan's national freeway cameras from the Freeway Bureau, MOTC to CCTV: + ~1,800 cameras read at runtime from the official MOTC 2.0 open-data table, + with on-demand snapshots and attribution. No API key is required; set + `CCTV_TAIWAN_ENABLED=0` to skip it. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9562dc9f5..61aede327 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1075,6 +1075,22 @@ The images are periodically updated road-condition photographs, not continuous video; camera poses use the existing low-confidence heading prior. `CCTV_LITHUANIA_MAX_SOURCES` sets its cap (default 400, bounded to 1–1000). +## Taiwan CCTV source pack + +The CCTV catalog includes Taiwan national freeway cameras from the Freeway +Bureau, MOTC. One keyless request reads the daily MOTC 2.0 `CCTV.xml` table +(~1,800 cameras); the request refuses redirects, has a 15-second deadline and a +6 MiB response cap, and uses the shared 15-minute cache and fair catalog +allocation. An unavailable source contributes an empty pack. The table +publishes MJPEG stream URLs; each is mapped to the same camera server's +single-JPEG path for the same camera number, and only those exact HTTPS pairs +on the four `freeway.gov.tw` camera servers are registered with the frame +proxy. Camera keys come from the server and camera number, because published +CCTV IDs contain Chinese place names and a few repeat across cameras. Poses use +the low-confidence heading prior. `CCTV_TAIWAN_ENABLED=0` disables the pack; +`CCTV_TAIWAN_MAX_SOURCES` sets its cap (default 600, bounded to 1–2000). +Gate: `src/data/cctvTaiwan.test.mjs`. + ## DriveBC CCTV source pack The CCTV catalog adds DriveBC highway cameras for British Columbia alongside the diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 5ec1adb0c..7c2c3fdbd 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -114,6 +114,8 @@ "server/providers/cctv/catalog.js", "server/providers/cctv/lithuania.js", "src/data/cctvLithuania.test.mjs", + "server/providers/cctv/taiwan.js", + "src/data/cctvTaiwan.test.mjs", "server/providers/cctv/constants.js", "server/providers/cctv/media.js", "server/providers/cctv/normalize.js", diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 75e58d2a6..35ee4485e 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -5,6 +5,7 @@ import { allocateSourceCap, resolveCatalogCap } from './cap.js'; import { loadGroundHeights, joinGroundHeights } from './groundHeights.js'; import { cameraRegion, normalizeSourceItem } from './normalize.js'; import { loadLithuaniaSources } from './lithuania.js'; +import { loadTaiwanSources } from './taiwan.js'; import { loadAustinSourcesFromOpenData, loadCaltransSourcesFromOpenData, @@ -144,6 +145,11 @@ const LIVE_PACKS = [ enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'), load: loadLithuaniaSources, }, + { + name: 'taiwan', + enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'), + load: loadTaiwanSources, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/taiwan.js b/server/providers/cctv/taiwan.js new file mode 100644 index 000000000..0a876f7cf --- /dev/null +++ b/server/providers/cctv/taiwan.js @@ -0,0 +1,249 @@ +import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js'; +import { + cameraDisplayCode, + fallbackHeadingFromId, + prioritizeSources, +} from './normalize.js'; +import { readResponseTextCapped } from '../common/http.js'; + +/** Freeway Bureau (MOTC) CCTV static table in the national MOTC 2.0 XML + * standard, published keyless and refreshed daily (~1,800 cameras). */ +export const TAIWAN_FREEWAY_CCTV_URL = + 'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml'; +/** The whole network is ~1.2 MB. */ +export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024; +export const DEFAULT_TAIWAN_MAX_SOURCES = 600; + +// The table publishes each camera's MJPEG stream. The same camera servers +// answer a single JPEG for the same camera ID on a sibling path, which is what +// the frame proxy can relay; only these exact pairs are ever registered. +const STREAM_TO_FRAME = new Map([ + ['cctvn.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']], + ['cctvn5.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']], + ['cctvc.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']], + [ + 'cctvs.freeway.gov.tw', + ['/live-view/mjpg/video.cgi', '/live-view/jpg/image.cgi'], + ], +]); + +const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' }; + +/** Prioritization anchors, north to south, so a cap keeps the whole island + * rather than only greater Taipei. */ +const ANCHORS = [ + { lat: 25.033, lon: 121.5654 }, // Taipei + { lat: 24.9936, lon: 121.301 }, // Taoyuan + { lat: 24.8138, lon: 120.9675 }, // Hsinchu + { lat: 24.7021, lon: 121.7378 }, // Yilan + { lat: 24.1477, lon: 120.6736 }, // Taichung + { lat: 22.9999, lon: 120.227 }, // Tainan + { lat: 22.6273, lon: 120.3014 }, // Kaohsiung +]; + +/** Taiwan main island with slack for the coast; the freeway network has no + * offshore-island cameras. */ +export function isLikelyTaiwanCoordinate(lat, lon) { + return ( + Number.isFinite(lat) && + Number.isFinite(lon) && + lat >= 21.8 && + lat <= 25.4 && + lon >= 119.9 && + lon <= 122.1 + ); +} + +const XML_ENTITIES = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", +}; + +function decodeXml(text) { + return String(text ?? '') + .replace(/&(amp|lt|gt|quot|apos);/g, (_, name) => XML_ENTITIES[name]) + .trim(); +} + +function tag(block, name) { + const match = new RegExp(`<${name}>([^<]*)`).exec(block); + return match ? decodeXml(match[1]) : ''; +} + +/** + * Parse the MOTC 2.0 `CCTVList` document into flat rows. Regex-based like the + * Catalonia and Tarktee parsers: the schema is flat, and only the fields used + * below are read. + * + * @param {string} xml + * @returns {Array} + */ +export function parseTaiwanCctvXml(xml) { + const rows = []; + const blockRe = /([\s\S]*?)<\/CCTV>/g; + let match; + while ((match = blockRe.exec(String(xml || ''))) !== null) { + const block = match[1]; + rows.push({ + cctvId: tag(block, 'CCTVID'), + streamUrl: tag(block, 'VideoStreamURL'), + lat: Number(tag(block, 'PositionLat')), + lon: Number(tag(block, 'PositionLon')), + roadName: tag(block, 'RoadName'), + direction: tag(block, 'RoadDirection'), + sectionStart: tag(block, 'Start'), + sectionEnd: tag(block, 'End'), + mile: tag(block, 'LocationMile'), + }); + } + return rows; +} + +/** + * Published MJPEG stream URL -> the same camera's still, plus the host and + * camera number that identify it, or null. + * + * @param {string} streamUrl + * @returns {?{frameUrl: string, host: string, camera: string}} + */ +export function taiwanFrame(streamUrl) { + let url; + try { + url = new URL(String(streamUrl ?? '')); + } catch { + return null; + } + if (url.protocol !== 'https:' || url.username || url.password || url.port) + return null; + const paths = STREAM_TO_FRAME.get(url.hostname); + if (!paths || url.pathname !== paths[0]) return null; + const camera = url.searchParams.get('camera') || ''; + if (!/^[A-Za-z0-9-]{1,40}$/.test(camera)) return null; + return { + // Reconstruct so extra query parameters cannot alter the request. + frameUrl: `https://${url.hostname}${paths[1]}?camera=${camera}`, + host: url.hostname.split('.')[0], + camera, + }; +} + +/** + * One parsed Freeway Bureau row -> one catalog source, or null. + * + * @param {object} row - Output of parseTaiwanCctvXml. + * @returns {?object} + */ +export function taiwanCameraToSource(row) { + if (!row || typeof row !== 'object') return null; + const { lat, lon } = row; + if (!isLikelyTaiwanCoordinate(lat, lon)) return null; + const frame = taiwanFrame(row.streamUrl); + if (!frame) return null; + const imageUrl = frame.frameUrl; + // CCTVID carries Chinese place names and a few IDs repeat across distinct + // cameras, so the stable key is the camera server plus its camera number. + const rawId = String(row.cctvId ?? '') + .trim() + .slice(0, 80); + const cameraId = `tw-freeway-${frame.host}-${frame.camera.toLowerCase()}`; + const road = String(row.roadName ?? '').trim(); + const direction = DIRECTIONS[String(row.direction ?? '').trim()] || ''; + const mile = String(row.mile ?? '').trim(); + const section = [row.sectionStart, row.sectionEnd] + .map((part) => String(part ?? '').trim()) + .filter(Boolean) + .join('→'); + const name = + [road, direction, mile].filter(Boolean).join(' ') + + (section ? ` ${section}` : '') || + rawId || + `Freeway camera ${frame.camera}`; + + return { + id: cameraId, + name: name.slice(0, 160), + // One island-wide category in the camera picker, like Finland and Norway. + city: 'Taiwan / 臺灣', + cityId: 'taiwan', + provider: 'Freeway Bureau, MOTC', + lat, + lon, + // The table gives the carriageway direction, not the camera's bearing, + // so the pose uses the shared low-confidence prior. + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'tw-freeway-motc', + license: + 'Freeway Bureau, MOTC open data, Open Government Data License, Taiwan (Version 1.0)', + code: cameraDisplayCode( + `${road} ${mile}`.trim().toUpperCase() || frame.camera, + ), + }; +} + +/** + * Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table. + * One bounded request per shared catalog refresh. + * + * @returns {Promise>} + */ +export async function loadTaiwanSources() { + try { + const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS); + const response = await fetch( + process.env.CCTV_TAIWAN_URL || TAIWAN_FREEWAY_CCTV_URL, + { + headers: { Accept: 'application/xml,text/xml' }, + redirect: 'manual', + signal, + }, + ); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + console.warn( + '[CCTV] Taiwan camera download failed:', + response.status >= 300 && response.status < 400 + ? 'catalog redirected; redirects are not followed' + : response.status, + ); + return []; + } + const xml = await readResponseTextCapped( + response, + TAIWAN_MAX_CATALOG_BYTES, + signal, + ); + const cameras = new Map(); + for (const row of parseTaiwanCctvXml(xml)) { + const camera = taiwanCameraToSource(row); + if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera); + } + const rawCap = Number( + process.env.CCTV_TAIWAN_MAX_SOURCES || DEFAULT_TAIWAN_MAX_SOURCES, + ); + const cap = Number.isFinite(rawCap) + ? Math.max(1, Math.min(2000, Math.floor(rawCap))) + : DEFAULT_TAIWAN_MAX_SOURCES; + const sources = prioritizeSources([...cameras.values()], cap, ANCHORS); + console.log( + `[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`, + ); + return sources; + } catch (error) { + console.warn( + '[CCTV] Taiwan camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvTaiwan.test.mjs b/src/data/cctvTaiwan.test.mjs new file mode 100644 index 000000000..73604e08c --- /dev/null +++ b/src/data/cctvTaiwan.test.mjs @@ -0,0 +1,200 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + loadTaiwanSources, + parseTaiwanCctvXml, + taiwanCameraToSource, + taiwanFrame, + TAIWAN_FREEWAY_CCTV_URL, +} from '../../server/providers/cctv/taiwan.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +function cctvXml(...blocks) { + return ` + + NFB + ${blocks.join('')} +`; +} + +function block({ + id = 'CCTV-N1-S-0.000-M', + stream = 'https://cctvn.freeway.gov.tw/abs2mjpg/bmjpg?camera=10000', + lat = '25.1229931', + lon = '121.735695', +} = {}) { + return ` + ${id} + ${stream} + ${lon} + ${lat} + 國道1號 + S + 基隆端基隆交流道 + 0K+000 + `; +} + +test('parses the MOTC 2.0 table and registers the same camera still', () => { + const [row] = parseTaiwanCctvXml(cctvXml(block())); + const camera = taiwanCameraToSource(row); + assert.equal(camera.id, 'tw-freeway-cctvn-10000'); + assert.equal(camera.lat, 25.1229931); + assert.equal(camera.lon, 121.735695); + assert.equal(camera.name, '國道1號 S 0K+000 基隆端→基隆交流道'); + assert.equal(camera.cityId, 'taiwan'); + assert.equal(camera.feedType, 'image'); + assert.equal(camera.headingConfidence, 'low'); + assert.equal( + camera.url, + 'https://cctvn.freeway.gov.tw/abs2mjpg/jpg?camera=10000', + ); + assert.equal(camera.snapshotUrl, camera.url); + assert.match(camera.license, /Open Government Data License, Taiwan/); +}); + +test('each camera server maps its stream path to its own still path only', () => { + assert.equal( + taiwanFrame( + 'https://cctvs.freeway.gov.tw/live-view/mjpg/video.cgi?camera=373', + ).frameUrl, + 'https://cctvs.freeway.gov.tw/live-view/jpg/image.cgi?camera=373', + ); + assert.equal( + taiwanFrame( + 'https://cctvn5.freeway.gov.tw/abs2mjpg/bmjpg?camera=c3a580c9-0232-46a1-bfc6-927590758ab4', + ).frameUrl, + 'https://cctvn5.freeway.gov.tw/abs2mjpg/jpg?camera=c3a580c9-0232-46a1-bfc6-927590758ab4', + ); + assert.equal( + taiwanFrame( + 'https://cctvc.freeway.gov.tw/abs2mjpg/bmjpg?camera=495&extra=1', + ).frameUrl, + 'https://cctvc.freeway.gov.tw/abs2mjpg/jpg?camera=495', + ); +}); + +test('off-host, downgraded or malformed stream URLs never enter the proxy registry', () => { + for (const stream of [ + 'http://cctvn.freeway.gov.tw/abs2mjpg/bmjpg?camera=10000', + 'https://cctvn.freeway.gov.tw.evil.test/abs2mjpg/bmjpg?camera=10000', + 'https://user:pass@cctvn.freeway.gov.tw/abs2mjpg/bmjpg?camera=10000', + 'https://cctvn.freeway.gov.tw:8443/abs2mjpg/bmjpg?camera=10000', + 'https://cctvn.freeway.gov.tw/live-view/mjpg/video.cgi?camera=10000', + 'https://cctvs.freeway.gov.tw/abs2mjpg/bmjpg?camera=373', + 'https://cctvn.freeway.gov.tw/abs2mjpg/bmjpg?camera=../../admin', + 'https://cctvs.freeway.gov.tw/live-view/mjpg/video.cgi?cacamera=373', + 'http://127.0.0.1/private', + '', + ]) { + assert.equal(taiwanFrame(stream), null, stream); + const [row] = parseTaiwanCctvXml(cctvXml(block({ stream }))); + assert.equal(taiwanCameraToSource(row), null, stream); + } +}); + +test('coordinates outside Taiwan, swapped or missing are rejected', () => { + for (const [lat, lon] of [ + ['121.735695', '25.1229931'], + ['35.68', '139.76'], + ['', '121.7'], + ['NaN', '121.7'], + ]) { + const [row] = parseTaiwanCctvXml(cctvXml(block({ lat, lon }))); + assert.equal(taiwanCameraToSource(row), null, `${lat},${lon}`); + } + assert.equal(taiwanCameraToSource(null), null); +}); + +test('Chinese place names in CCTVID stay in the record without shaping the key', () => { + const [row] = parseTaiwanCctvXml( + cctvXml(block({ id: 'CCTV-N1-N-55-R-中壢服務區&(北)' })), + ); + const camera = taiwanCameraToSource(row); + assert.equal(camera.id, 'tw-freeway-cctvn-10000'); + assert.equal(row.cctvId, 'CCTV-N1-N-55-R-中壢服務區&(北)'); +}); + +function mockCatalog(t, xml) { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url, options) => { + requested.push(String(url)); + if (String(url) === TAIWAN_FREEWAY_CCTV_URL) { + assert.equal(options.redirect, 'manual'); + return new Response(xml, { + headers: { 'content-type': 'application/xml' }, + }); + } + return Response.json([]); + }); + return requested; +} + +test('loader deduplicates cameras that share a camera number', async (t) => { + mockCatalog( + t, + cctvXml( + block(), + block({ id: 'CCTV-N1-S-0.000-M-dup' }), + block({ + stream: 'https://cctvc.freeway.gov.tw/abs2mjpg/bmjpg?camera=495', + lat: '24.1', + lon: '120.6', + }), + ), + ); + assert.deepEqual((await loadTaiwanSources()).map((c) => c.id).sort(), [ + 'tw-freeway-cctvc-495', + 'tw-freeway-cctvn-10000', + ]); +}); + +test('a redirected or failed catalog contributes an empty pack', async (t) => { + t.mock.method(console, 'warn', () => {}); + for (const status of [302, 503]) { + t.mock.method( + globalThis, + 'fetch', + async () => + new Response('', { + status, + headers: status === 302 ? { location: 'https://evil.test/' } : {}, + }), + ); + assert.deepEqual(await loadTaiwanSources(), []); + } +}); + +async function runCatalogWithTaiwanFlag(t, flag) { + const requested = mockCatalog(t, cctvXml(block())); + const saved = process.env.CCTV_TAIWAN_ENABLED; + try { + if (flag === undefined) delete process.env.CCTV_TAIWAN_ENABLED; + else process.env.CCTV_TAIWAN_ENABLED = flag; + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; + } finally { + if (saved === undefined) delete process.env.CCTV_TAIWAN_ENABLED; + else process.env.CCTV_TAIWAN_ENABLED = saved; + } +} + +test('the Taiwan pack is in the catalog by default', async (t) => { + const { sources } = await runCatalogWithTaiwanFlag(t, undefined); + assert.equal( + sources.find((c) => c.id === 'tw-freeway-cctvn-10000')?.cityId, + 'taiwan', + ); +}); + +test('CCTV_TAIWAN_ENABLED=0 makes no Freeway Bureau request', async (t) => { + const { requested, sources } = await runCatalogWithTaiwanFlag(t, '0'); + assert.ok(requested.length > 0, 'the other packs still refreshed'); + assert.equal( + requested.some((href) => href.includes('freeway.gov.tw')), + false, + ); + assert.ok(!sources.some((c) => c.cityId === 'taiwan')); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 8d3949351..8ba07e1b0 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -257,6 +257,13 @@ export const DATA_CREDITS = [ 'Road-camera stills (Lithuania, when enabled): ' + 'Via Lietuva / Eismoinfo; periodically updated images', }, + { + key: 'taiwan-cctv', + html: + 'Freeway camera stills (Taiwan): ' + + 'Freeway Bureau, MOTC, ' + + 'Open Government Data License, Taiwan (Version 1.0)', + }, { key: 'calgary-cctv', html: From e324e2231c86e1356b758c091e5fb9b07061cf3a Mon Sep 17 00:00:00 2001 From: Jerry Lai <419vive@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:53:27 +0800 Subject: [PATCH 2/3] docs(changelog): name the Taiwan CCTV fragment for #1237 Co-Authored-By: Claude Opus 5.5 --- .../{taiwan-freeway-cctv.md => 1237-taiwan-freeway-cctv.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename changelog.d/{taiwan-freeway-cctv.md => 1237-taiwan-freeway-cctv.md} (80%) diff --git a/changelog.d/taiwan-freeway-cctv.md b/changelog.d/1237-taiwan-freeway-cctv.md similarity index 80% rename from changelog.d/taiwan-freeway-cctv.md rename to changelog.d/1237-taiwan-freeway-cctv.md index 7fc0339e3..1578971d4 100644 --- a/changelog.d/taiwan-freeway-cctv.md +++ b/changelog.d/1237-taiwan-freeway-cctv.md @@ -1,4 +1,4 @@ - Add Taiwan's national freeway cameras from the Freeway Bureau, MOTC to CCTV: ~1,800 cameras read at runtime from the official MOTC 2.0 open-data table, with on-demand snapshots and attribution. No API key is required; set - `CCTV_TAIWAN_ENABLED=0` to skip it. + `CCTV_TAIWAN_ENABLED=0` to skip it (419vive, #1237). From ce5f69b586b4f5d23fa02a080d6ba10508d5b7ec Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Tue, 20 Oct 2026 14:51:48 +0100 Subject: [PATCH 3/3] fix(cctv): back off the Taiwan catalog when unreachable; review fixes - taiwan.js: a failed download serves the last good table (up to 24 h) and is not retried for an hour. The table host does not answer from some networks outside Taiwan, and each attempt held the shared catalog refresh for the ~10 s connect timeout. - Own server/providers/cctv/taiwan.js in package-boundaries.json (check:boundaries failed). - Decode numeric character references; name falls back to the CCTVID when a row has no road details. - Credit names the dataset and links the OGDL Taiwan licence. - Document CCTV_TAIWAN_URL; list Taiwan in the CURRENT-STATE source table. - Tests: isolate catalog env, reset loader state, cover the byte cap, failed fetch, backoff, last-good fallback, cap clamping, redirect with a valid body, name fallback, numeric entities and UUID camera ids. Co-Authored-By: Claude Opus 5.5 --- .env.example | 1 + DATA_SOURCES.md | 2 +- docs/CURRENT-STATE.md | 10 +- scripts/package-boundaries.json | 1 + server/providers/cctv/taiwan.js | 88 +++++++++++++---- src/data/cctvTaiwan.test.mjs | 166 +++++++++++++++++++++++++++++--- src/data/dataCredits.js | 3 +- 7 files changed, 237 insertions(+), 34 deletions(-) diff --git a/.env.example b/.env.example index c318bfd49..78264489b 100644 --- a/.env.example +++ b/.env.example @@ -249,6 +249,7 @@ AISSTREAM_API_KEY= # CCTV_FLORIDA_MAX_SOURCES=500 # cap cameras loaded from the FL511 catalog (nearest regional anchors first) # CCTV_TAIWAN_ENABLED=1 # set 0 to skip Taiwan Freeway Bureau (MOTC) road cameras # CCTV_TAIWAN_MAX_SOURCES=600 # cap Taiwan freeway cameras (1-2000); no API key required +# CCTV_TAIWAN_URL= # override the Freeway Bureau MOTC 2.0 CCTV.xml URL # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index d9dddf995..728381632 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -112,7 +112,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei **Quebec is thinly represented**, and that is upstream rather than a filter here: the province runs its own Info-Smog program, so only a handful of Quebec stations appear in ECCC's national feed. The reporting station count also varies hour to hour as stations report or fall silent. - **Via Lietuva / Eismoinfo (Lithuania).** Public road-camera catalogs and periodically updated still images from [Eismoinfo](https://eismoinfo.lt/), credited to Via Lietuva / eismoinfo.lt in the app. **Off by default.** The camera endpoints are the eismoinfo.lt website backend and are not among the services listed in Via Lietuva's [open-data statement](https://eismoinfo.lt/eismoinfo-backend/content?id=44), whose conditions are attribution and reuse on the same terms. Until the statement is confirmed to cover the camera photos, the pack loads only when an operator sets `CCTV_LITHUANIA_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to Eismoinfo. `eismoinfo-backend/camera-info-table` is joined by ID to `layer-static-features/VKR`. Without `lks=true`, the latter returns WGS84 `[latitude, longitude]`; the table's `x/y` instead use LKS-94 metres. Only the official HTTPS `/eismoinfo-backend/image-provider/camera/last?id=…` image endpoint is registered. Catalogs use the shared 15-minute cache; frames are fetched on demand and never bundled or archived. These are periodic snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. `CCTV_LITHUANIA_MAX_SOURCES` defaults to 400. -- **Freeway Bureau, MOTC (Taiwan).** National freeway cameras from the Freeway Bureau's keyless [CCTV static table](https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml) in the MOTC 2.0 standard, refreshed daily and fetched at runtime (never bundled). Freeway Bureau traffic data is published under the [Open Government Data License, Taiwan (Version 1.0)](https://data.gov.tw/license), which permits reuse with attribution; the app credits the Freeway Bureau, MOTC. The table publishes each camera's MJPEG stream on one of four official camera servers (`cctvn`, `cctvn5`, `cctvc`, `cctvs` under `freeway.gov.tw`). The frame proxy relays single images, so each published stream URL is mapped to the same server's single-JPEG path for the same camera number (`/abs2mjpg/bmjpg` → `/abs2mjpg/jpg`, `/live-view/mjpg/video.cgi` → `/live-view/jpg/image.cgi`); only those exact HTTPS pairs are registered, and the still path is not itself listed in the table. These are on-demand snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. On by default; `CCTV_TAIWAN_ENABLED=0` skips the pack, and `CCTV_TAIWAN_MAX_SOURCES` defaults to 600 (bounded to 1–2000). +- **Freeway Bureau, MOTC (Taiwan).** National freeway cameras from the Freeway Bureau's keyless [CCTV static table](https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml) in the MOTC 2.0 standard, refreshed daily and fetched at runtime (never bundled). Freeway Bureau traffic data is published under the [Open Government Data License, Taiwan (Version 1.0)](https://data.gov.tw/license), which permits reuse, including commercial reuse, with attribution; the app credits the Freeway Bureau, MOTC, names the dataset and links the licence, as its attribution exhibit asks. The table publishes each camera's MJPEG stream on one of four official camera servers (`cctvn`, `cctvn5`, `cctvc`, `cctvs` under `freeway.gov.tw`). The frame proxy relays single images, so each published stream URL is mapped to the same server's single-JPEG path for the same camera number (`/abs2mjpg/bmjpg` → `/abs2mjpg/jpg`, `/live-view/mjpg/video.cgi` → `/live-view/jpg/image.cgi`); only those exact HTTPS pairs are registered, and the still path is not itself listed in the table. These are on-demand snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. The table host does not answer from some networks outside Taiwan (the camera servers do); there the pack is empty, logs one warning and retries hourly. On by default; `CCTV_TAIWAN_ENABLED=0` skips the pack, and `CCTV_TAIWAN_MAX_SOURCES` defaults to 600 (bounded to 1–2000). - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index a0a190153..1c64f60fe 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1102,14 +1102,18 @@ The CCTV catalog includes Taiwan national freeway cameras from the Freeway Bureau, MOTC. One keyless request reads the daily MOTC 2.0 `CCTV.xml` table (~1,800 cameras); the request refuses redirects, has a 15-second deadline and a 6 MiB response cap, and uses the shared 15-minute cache and fair catalog -allocation. An unavailable source contributes an empty pack. The table +allocation. A failed download serves the last good table (up to a day old), +or an empty pack, and is not retried for an hour: the catalog host does not +answer from some networks outside Taiwan, and without the backoff every refresh +there would wait out the connect timeout. The table publishes MJPEG stream URLs; each is mapped to the same camera server's single-JPEG path for the same camera number, and only those exact HTTPS pairs on the four `freeway.gov.tw` camera servers are registered with the frame proxy. Camera keys come from the server and camera number, because published CCTV IDs contain Chinese place names and a few repeat across cameras. Poses use the low-confidence heading prior. `CCTV_TAIWAN_ENABLED=0` disables the pack; -`CCTV_TAIWAN_MAX_SOURCES` sets its cap (default 600, bounded to 1–2000). +`CCTV_TAIWAN_MAX_SOURCES` sets its cap (default 600, bounded to 1–2000), and +`CCTV_TAIWAN_URL` overrides the table URL. Gate: `src/data/cctvTaiwan.test.mjs`. ## DriveBC CCTV source pack @@ -3474,7 +3478,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + FL511 (FL, opt-in) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + FL511 (FL, opt-in) + Freeway Bureau, MOTC (Taiwan) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | | Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil), Bicing (Barcelona) and seven other Spanish cities + Urban Sharing, Entur, JCDecaux (Lillestrøm, Valencia, Seville) + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 5e3f3eb69..bf33bebc7 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -191,6 +191,7 @@ "server/providers/cctv/groundHeights.js", "server/providers/cctv/catalog.js", "server/providers/cctv/lithuania.js", + "server/providers/cctv/taiwan.js", "server/providers/cctv/constants.js", "server/providers/cctv/media.js", "server/providers/cctv/normalize.js", diff --git a/server/providers/cctv/taiwan.js b/server/providers/cctv/taiwan.js index 0a876f7cf..830ed47fd 100644 --- a/server/providers/cctv/taiwan.js +++ b/server/providers/cctv/taiwan.js @@ -13,6 +13,13 @@ export const TAIWAN_FREEWAY_CCTV_URL = /** The whole network is ~1.2 MB. */ export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024; export const DEFAULT_TAIWAN_MAX_SOURCES = 600; +/** After a failed download, wait this long before asking again. The catalog + * host does not answer from some networks outside Taiwan, and every attempt + * there holds the shared catalog refresh until the connect timeout. */ +export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000; +/** The table is republished daily; a good copy stands in for a failed + * download for up to this long. */ +export const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000; // The table publishes each camera's MJPEG stream. The same camera servers // answer a single JPEG for the same camera ID on a sibling path, which is what @@ -62,9 +69,20 @@ const XML_ENTITIES = { apos: "'", }; +function codePoint(value, radix) { + const code = parseInt(value, radix); + return Number.isInteger(code) && code > 0 && code <= 0x10ffff + ? String.fromCodePoint(code) + : ''; +} + function decodeXml(text) { return String(text ?? '') - .replace(/&(amp|lt|gt|quot|apos);/g, (_, name) => XML_ENTITIES[name]) + .replace( + /&(?:(amp|lt|gt|quot|apos)|#(\d{1,7})|#x([0-9a-fA-F]{1,6}));/g, + (_, name, dec, hex) => + name ? XML_ENTITIES[name] : codePoint(dec ?? hex, dec ? 10 : 16), + ) .trim(); } @@ -156,9 +174,11 @@ export function taiwanCameraToSource(row) { .map((part) => String(part ?? '').trim()) .filter(Boolean) .join('→'); + // A bare carriageway letter is not a name; fall back to the published ID. const name = - [road, direction, mile].filter(Boolean).join(' ') + - (section ? ` ${section}` : '') || + (road || mile || section + ? [road, direction, mile, section].filter(Boolean).join(' ') + : '') || rawId || `Freeway camera ${frame.camera}`; @@ -191,13 +211,42 @@ export function taiwanCameraToSource(row) { }; } +let lastGood = null; +let lastGoodAt = 0; +let retryAfter = 0; + +/** Forget the last good table and any failure backoff (tests only). */ +export function resetTaiwanSourcesCache() { + lastGood = null; + lastGoodAt = 0; + retryAfter = 0; +} + +function fallbackSources() { + return lastGood && Date.now() - lastGoodAt <= TAIWAN_LAST_GOOD_MAX_AGE_MS + ? lastGood + : []; +} + +function taiwanSourceCap() { + const rawCap = Number( + process.env.CCTV_TAIWAN_MAX_SOURCES || DEFAULT_TAIWAN_MAX_SOURCES, + ); + return Number.isFinite(rawCap) + ? Math.max(1, Math.min(2000, Math.floor(rawCap))) + : DEFAULT_TAIWAN_MAX_SOURCES; +} + /** * Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table. - * One bounded request per shared catalog refresh. + * One bounded request per shared catalog refresh. A failed download serves the + * last good table (up to a day old) and is not retried for an hour, so a host + * that cannot reach the Freeway Bureau does not wait on it every refresh. * * @returns {Promise>} */ export async function loadTaiwanSources() { + if (Date.now() < retryAfter) return fallbackSources(); try { const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS); const response = await fetch( @@ -210,13 +259,11 @@ export async function loadTaiwanSources() { ); if (!response.ok) { await response.body?.cancel().catch(() => {}); - console.warn( - '[CCTV] Taiwan camera download failed:', + throw new Error( response.status >= 300 && response.status < 400 ? 'catalog redirected; redirects are not followed' - : response.status, + : `HTTP ${response.status}`, ); - return []; } const xml = await readResponseTextCapped( response, @@ -228,22 +275,29 @@ export async function loadTaiwanSources() { const camera = taiwanCameraToSource(row); if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera); } - const rawCap = Number( - process.env.CCTV_TAIWAN_MAX_SOURCES || DEFAULT_TAIWAN_MAX_SOURCES, + if (!cameras.size) throw new Error('no usable cameras in the table'); + const sources = prioritizeSources( + [...cameras.values()], + taiwanSourceCap(), + ANCHORS, ); - const cap = Number.isFinite(rawCap) - ? Math.max(1, Math.min(2000, Math.floor(rawCap))) - : DEFAULT_TAIWAN_MAX_SOURCES; - const sources = prioritizeSources([...cameras.values()], cap, ANCHORS); console.log( `[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`, ); + lastGood = sources; + lastGoodAt = Date.now(); + retryAfter = 0; return sources; } catch (error) { + retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS; + const fallback = fallbackSources(); console.warn( - '[CCTV] Taiwan camera download error:', - error?.message || error, + `[CCTV] Taiwan camera download failed (${error?.message || error}); ` + + (fallback.length + ? `serving the last good ${fallback.length} cameras` + : 'no Taiwan cameras') + + ', next attempt in 1 h', ); - return []; + return fallback; } } diff --git a/src/data/cctvTaiwan.test.mjs b/src/data/cctvTaiwan.test.mjs index 73604e08c..4794f11a8 100644 --- a/src/data/cctvTaiwan.test.mjs +++ b/src/data/cctvTaiwan.test.mjs @@ -1,14 +1,45 @@ -import { test } from 'node:test'; +import { afterEach, beforeEach, test } from 'node:test'; import assert from 'node:assert/strict'; import { loadTaiwanSources, parseTaiwanCctvXml, + resetTaiwanSourcesCache, taiwanCameraToSource, taiwanFrame, TAIWAN_FREEWAY_CCTV_URL, + TAIWAN_MAX_CATALOG_BYTES, + TAIWAN_RETRY_AFTER_FAILURE_MS, } from '../../server/providers/cctv/taiwan.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; +// Settings that would steer the loader or the catalog away from the mocks. +const ISOLATED_ENV = [ + 'CCTV_SOURCES_FILE', + 'CCTV_SOURCES_JSON', + 'CCTV_PREFER_AUSTIN', + 'CCTV_FORCE_AUSTIN', + 'CCTV_TAIWAN_ENABLED', + 'CCTV_TAIWAN_URL', + 'CCTV_TAIWAN_MAX_SOURCES', +]; + +let savedEnv = {}; + +beforeEach(() => { + resetTaiwanSourcesCache(); + savedEnv = Object.fromEntries( + ISOLATED_ENV.map((key) => [key, process.env[key]]), + ); + for (const key of ISOLATED_ENV) delete process.env[key]; +}); + +afterEach(() => { + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + function cctvXml(...blocks) { return ` @@ -158,27 +189,21 @@ test('a redirected or failed catalog contributes an empty pack', async (t) => { globalThis, 'fetch', async () => - new Response('', { + new Response(cctvXml(block()), { status, headers: status === 302 ? { location: 'https://evil.test/' } : {}, }), ); + resetTaiwanSourcesCache(); assert.deepEqual(await loadTaiwanSources(), []); } }); async function runCatalogWithTaiwanFlag(t, flag) { const requested = mockCatalog(t, cctvXml(block())); - const saved = process.env.CCTV_TAIWAN_ENABLED; - try { - if (flag === undefined) delete process.env.CCTV_TAIWAN_ENABLED; - else process.env.CCTV_TAIWAN_ENABLED = flag; - const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); - return { requested, sources }; - } finally { - if (saved === undefined) delete process.env.CCTV_TAIWAN_ENABLED; - else process.env.CCTV_TAIWAN_ENABLED = saved; - } + if (flag !== undefined) process.env.CCTV_TAIWAN_ENABLED = flag; + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; } test('the Taiwan pack is in the catalog by default', async (t) => { @@ -198,3 +223,120 @@ test('CCTV_TAIWAN_ENABLED=0 makes no Freeway Bureau request', async (t) => { ); assert.ok(!sources.some((c) => c.cityId === 'taiwan')); }); + +test('a catalog over the byte cap or a failed request contributes an empty pack', async (t) => { + t.mock.method(console, 'warn', () => {}); + const padding = ``; + t.mock.method( + globalThis, + 'fetch', + async () => new Response(cctvXml(block()) + padding), + ); + assert.deepEqual(await loadTaiwanSources(), []); + resetTaiwanSourcesCache(); + t.mock.method(globalThis, 'fetch', async () => { + throw new TypeError('fetch failed'); + }); + assert.deepEqual(await loadTaiwanSources(), []); +}); + +test('a failed download is not retried until the backoff ends', async (t) => { + t.mock.method(console, 'warn', () => {}); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + throw new TypeError('fetch failed'); + }); + t.mock.method(Date, 'now', () => 1_000_000); + assert.deepEqual(await loadTaiwanSources(), []); + assert.deepEqual(await loadTaiwanSources(), []); + assert.equal(calls, 1, 'the second refresh made no request'); + Date.now.mock.mockImplementation( + () => 1_000_000 + TAIWAN_RETRY_AFTER_FAILURE_MS + 1, + ); + await loadTaiwanSources(); + assert.equal(calls, 2, 'the backoff ends after an hour'); +}); + +test('a failed download serves the last good table', async (t) => { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + const fetchMock = t.mock.method( + globalThis, + 'fetch', + async () => new Response(cctvXml(block())), + ); + const first = await loadTaiwanSources(); + assert.deepEqual( + first.map((c) => c.id), + ['tw-freeway-cctvn-10000'], + ); + fetchMock.mock.mockImplementation( + async () => new Response('', { status: 503 }), + ); + assert.deepEqual( + (await loadTaiwanSources()).map((c) => c.id), + ['tw-freeway-cctvn-10000'], + ); +}); + +test('CCTV_TAIWAN_MAX_SOURCES is clamped to 1-2000', async (t) => { + t.mock.method(console, 'log', () => {}); + const rows = Array.from({ length: 3 }, (_, i) => + block({ + stream: `https://cctvn.freeway.gov.tw/abs2mjpg/bmjpg?camera=${i + 1}`, + }), + ); + t.mock.method( + globalThis, + 'fetch', + async () => new Response(cctvXml(...rows)), + ); + for (const [value, expected] of [ + ['0', 1], + ['2', 2], + ['5000', 3], + ['abc', 3], + ]) { + resetTaiwanSourcesCache(); + process.env.CCTV_TAIWAN_MAX_SOURCES = value; + const sources = await loadTaiwanSources(); + assert.equal(sources.length, expected, value); + assert.equal(sources.available, 3, value); + } +}); + +test('names fall back to the published ID when the row has no road details', () => { + const xml = cctvXml(block({ id: 'CCTV-N3-N-100-M' })) + .replace('國道1號', '') + .replace('0K+000', '') + .replace('基隆端基隆交流道', ''); + const camera = taiwanCameraToSource(parseTaiwanCctvXml(xml)[0]); + assert.equal(camera.name, 'CCTV-N3-N-100-M'); +}); + +test('numeric character references in the table are decoded', () => { + const xml = cctvXml(block()).replace( + '國道1號', + '國道1號', + ); + assert.equal(parseTaiwanCctvXml(xml)[0].roadName, '國道1號'); +}); + +test('camera numbers that are UUIDs keep their case in the frame URL only', () => { + const camera = taiwanCameraToSource( + parseTaiwanCctvXml( + cctvXml( + block({ + stream: + 'https://cctvn5.freeway.gov.tw/abs2mjpg/bmjpg?camera=C3A580C9-0232-46A1-BFC6-927590758AB4', + }), + ), + )[0], + ); + assert.equal( + camera.id, + 'tw-freeway-cctvn5-c3a580c9-0232-46a1-bfc6-927590758ab4', + ); + assert.match(camera.url, /camera=C3A580C9-0232-46A1-BFC6-927590758AB4$/); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 06c616dd7..7b6cba9b7 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -263,7 +263,8 @@ export const DATA_CREDITS = [ html: 'Freeway camera stills (Taiwan): ' + 'Freeway Bureau, MOTC, ' + - 'Open Government Data License, Taiwan (Version 1.0)', + 'CCTV static data (MOTC 2.0), released under the ' + + 'Open Government Data License, Taiwan (Version 1.0)', }, { key: 'calgary-cctv',