From 3792d7377e2aedb3138b86d313b0512442cfcd47 Mon Sep 17 00:00:00 2001 From: yaka0007 Date: Mon, 19 Oct 2026 01:12:18 -0400 Subject: [PATCH] fix(cctv): harden Tarktee DATEX catalog fetches and dedupe before cap --- server/providers/cctv/constants.js | 5 ++ server/providers/cctv/sources.js | 58 ++++++++++-- src/data/cctvEstonia.test.mjs | 139 +++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 8 deletions(-) diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 20b963770..9bdb5e511 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -188,6 +188,11 @@ export const TARKTEE_IMAGES_URL = 'https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages'; export const TARKTEE_IMAGE_ORIGIN = 'https://tarktee.transpordiamet.ee/images/'; export const DEFAULT_TARKTEE_MAX_SOURCES = 179; +/** Hard ceiling per Tarktee DATEX XML document (~120–165 KB live). */ +export const TARKTEE_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** Two DATEX XML documents download concurrently from the same origin (~5.3 s + * baseline) alongside the other packs, so the 8 s default can abort mid-stream. */ +export const TARKTEE_FETCH_TIMEOUT_MS = 15_000; export const TARKTEE_ANCHORS = [ { lat: 59.437, lon: 24.753 }, // Tallinn { lat: 58.378, lon: 26.729 }, // Tartu diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index c3d11e752..cba857565 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -42,6 +42,8 @@ import { TARKTEE_IMAGES_URL, TARKTEE_IMAGE_ORIGIN, DEFAULT_TARKTEE_MAX_SOURCES, + TARKTEE_MAX_CATALOG_BYTES, + TARKTEE_FETCH_TIMEOUT_MS, TARKTEE_ANCHORS, DEFAULT_WARENDORF_SOURCE_FILE, WARENDORF_IMAGE_ORIGINS, @@ -1194,25 +1196,62 @@ export async function loadTarkteeSourcesFromDatex() { const [locResp, imgResp] = await Promise.all([ fetch(TARKTEE_LOCATIONS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + redirect: 'manual', + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), fetch(TARKTEE_IMAGES_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + redirect: 'manual', + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), ]); + // Both responses are fetched in parallel; if either is rejected or overflows + // its byte cap, both bodies must be released so neither transport leaks. + const discardBoth = async () => { + await Promise.allSettled([ + locResp?.body?.cancel(), + imgResp?.body?.cancel(), + ]); + return []; + }; + if ( + (locResp.status >= 300 && locResp.status < 400) || + (imgResp.status >= 300 && imgResp.status < 400) + ) { + console.warn( + '[CCTV] Tarktee DATEX catalog redirected; redirects are not followed', + ); + return discardBoth(); + } if (!locResp.ok) { console.warn('[CCTV] Tarktee locations download failed:', locResp.status); - return []; + return discardBoth(); } if (!imgResp.ok) { console.warn('[CCTV] Tarktee images download failed:', imgResp.status); - return []; + return discardBoth(); + } + let locXml; + let imgXml; + const readAbort = new AbortController(); + try { + [locXml, imgXml] = await Promise.all([ + readResponseTextCapped( + locResp, + TARKTEE_MAX_CATALOG_BYTES, + readAbort.signal, + ), + readResponseTextCapped( + imgResp, + TARKTEE_MAX_CATALOG_BYTES, + readAbort.signal, + ), + ]); + } catch (error) { + readAbort.abort(); + await discardBoth(); + throw error; } - const [locXml, imgXml] = await Promise.all([ - locResp.text(), - imgResp.text(), - ]); const locations = parseTarkteeDatexLocations(locXml); const images = parseTarkteeDatexImages(imgXml); if (!locations.size || !images.size) { @@ -1224,6 +1263,7 @@ export async function loadTarkteeSourcesFromDatex() { } const cameras = []; + const seen = new Set(); for (const [locationId, loc] of locations.entries()) { const imageUrl = images.get(locationId); if (!imageUrl) continue; @@ -1235,6 +1275,8 @@ export async function loadTarkteeSourcesFromDatex() { const cameraId = numMatch ? `ee-tarktee-${numMatch[1]}` : `ee-tarktee-${locationId}`; + if (seen.has(cameraId)) continue; + seen.add(cameraId); cameras.push({ id: cameraId, name: loc.name, diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 7c0ebaba1..a6385854a 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -1,11 +1,23 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { + TARKTEE_IMAGES_URL, + TARKTEE_LOCATIONS_URL, + TARKTEE_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; import { loadTallinnSourcesFromCatalog, + loadTarkteeSourcesFromDatex, parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + const SAMPLE_LOCATIONS_XML = ` @@ -84,3 +96,130 @@ test('Tarktee DATEX image parser rejects non-official hosts', () => { const images = parseTarkteeDatexImages(xml); assert.equal(images.size, 0); }); + +test('Tarktee loader deduplicates cameras sharing an image number before the cap', async (t) => { + quiet(t); + const dupLocationsXml = ` + + + + + Tallinn Ringtee + 59.437024.7530 + + + Tallinn Ringtee duplicate + 59.437124.7531 + + + +`; + const dupImagesXml = ` + + + https://tarktee.transpordiamet.ee/images/42/42_1.jpg + https://tarktee.transpordiamet.ee/images/42/42_2.jpg + +`; + + const seen = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init?.redirect]); + const body = + String(url) === TARKTEE_LOCATIONS_URL ? dupLocationsXml : dupImagesXml; + return new Response(body, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + + const cameras = await loadTarkteeSourcesFromDatex(); + assert.deepEqual(seen, [ + [TARKTEE_LOCATIONS_URL, 'manual'], + [TARKTEE_IMAGES_URL, 'manual'], + ]); + assert.equal(cameras.length, 1); + assert.equal(cameras[0].id, 'ee-tarktee-42'); + assert.equal(cameras[0].name, 'Tallinn Ringtee'); +}); + +test('Tarktee loader refuses a redirect and cancels both response bodies', async (t) => { + quiet(t); + const cancelled = { locations: false, images: false }; + t.mock.method(globalThis, 'fetch', async (url) => { + const isLocations = String(url) === TARKTEE_LOCATIONS_URL; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('')); + }, + cancel() { + if (isLocations) cancelled.locations = true; + else cancelled.images = true; + }, + }); + return new Response(body, { + status: isLocations ? 302 : 200, + headers: isLocations ? { location: 'https://evil.example/loc.xml' } : {}, + }); + }); + + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal(cancelled.locations, true, 'redirected body is cancelled'); + assert.equal(cancelled.images, true, 'sibling 200 body is also cancelled'); +}); + +test('Tarktee loader cancels the sibling body when one feed fails or overflows the byte cap', async (t) => { + quiet(t); + let siblingCancelled = false; + t.mock.method(globalThis, 'fetch', async (url) => { + const isLocations = String(url) === TARKTEE_LOCATIONS_URL; + if (isLocations) { + return new Response('', { status: 503 }); + } + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(SAMPLE_IMAGES_XML)); + }, + cancel() { + siblingCancelled = true; + }, + }); + return new Response(body, { status: 200 }); + }); + + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal( + siblingCancelled, + true, + 'sibling 200 stream is cancelled on 503', + ); + + let overflowSiblingCancelled = false; + t.mock.method(globalThis, 'fetch', async (url) => { + const isLocations = String(url) === TARKTEE_LOCATIONS_URL; + if (isLocations) { + const padded = + SAMPLE_LOCATIONS_XML + ' '.repeat(TARKTEE_MAX_CATALOG_BYTES); + return new Response(padded, { + status: 200, + headers: { 'Content-Length': String(padded.length) }, + }); + } + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(SAMPLE_IMAGES_XML)); + }, + cancel() { + overflowSiblingCancelled = true; + }, + }); + return new Response(body, { status: 200 }); + }); + + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal( + overflowSiblingCancelled, + true, + 'sibling stream is cancelled when the other feed exceeds the byte cap', + ); +});