diff --git a/changelog.d/1266-cctv-shared-fix-batch.md b/changelog.d/1266-cctv-shared-fix-batch.md index 898606fafd..f35f3ac82c 100644 --- a/changelog.d/1266-cctv-shared-fix-batch.md +++ b/changelog.d/1266-cctv-shared-fix-batch.md @@ -1,10 +1,12 @@ - Refuse Ontario 511 catalog redirects and keep the key-bearing request URL and upstream error text out of failure logs, which now name only the error - class and system code. Credit Ontario 511 with the Open Government - Licence – Ontario attribution statement (asturksever, #1266). + class and system code; release the body of an error response. Credit + Ontario 511 with the Open Government Licence – Ontario attribution statement + (asturksever, #1266). - Abort the other Tarktee (Estonia) DATEX request as soon as one fails, before its headers or while its body is read, instead of waiting out the shared - 30 s deadline; a response that arrives after the failure is released too + 30 s deadline; a response that arrives after the failure is released too. + The Lithuania pack reads its two catalogs through the same helper (asturksever, #1266). - Frame the calibration gizmo before the CCTV QA drag regression drags it; default autofocus could leave the arrow shorter than the 10 px grab diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 661e962e15..d0c06f38c8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -7546,8 +7546,11 @@ silently demoting every later lookup for the session. have a 4 MiB per-document limit, enforced against declared length and streamed bytes. A failed request, unsuccessful response or body read aborts the other feed at once and releases every response body, including one that - arrives after the failure. Cameras sharing an image number are deduplicated - before the Tarktee pack cap is applied. + arrives after the failure. The Lithuania pack reads its two catalogs + through the same helper (`readCatalogsTogether`, + `server/providers/cctv/catalogPair.js`) with its 15 s deadline. An Ontario + error response has its body released. Cameras sharing an image number are + deduplicated before the Tarktee pack cap is applied. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a tri-state cycle `OFF → ON → VIEWSHED`; viewshed mode renders each visible camera's frustum diff --git a/scripts/format-scope.json b/scripts/format-scope.json index e86ae6c4ea..3c846d449b 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -148,6 +148,7 @@ "server/providers/cctv/groundHeights.js", "server/providers/cctv/catalog.js", "server/providers/cctv/lithuania.js", + "server/providers/cctv/catalogPair.js", "src/data/cctvLithuania.test.mjs", "server/providers/cctv/taiwan.js", "src/data/cctvTaiwan.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 31245c7cf2..d600ec7446 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -563,6 +563,7 @@ "server/providers/cctv/groundHeights.js", "server/providers/cctv/catalog.js", "server/providers/cctv/lithuania.js", + "server/providers/cctv/catalogPair.js", "server/providers/cctv/taiwan.js", "server/providers/cctv/quebec.js", "server/providers/cctv/ottawa.js", diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 734fb98fd6..652de38b7d 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -1792,31 +1792,40 @@ async function main() { // Frame the actual gizmo for the drag fixture. Default autofocus can leave // its shaft below the 10px interaction threshold; that distant view is not - // the drag-usability scenario this regression exercises. + // the drag-usability scenario this regression exercises. The gizmo is drawn + // through the renderer ports, so its arrow is read from the layer's drawn + // state (plain {x,y,z} points), not from viewer.entities. const dragFraming = await page.evaluate(async () => { const viewer = window.__godsEyeView.viewer; - const e = viewer.entities.getById('cctv-gizmo-move-east'); - if (!e) return null; - const pts = e.polyline.positions.getValue(viewer.clock.currentTime); - const target = pts?.[0]; - if (!target) return null; + const mod = window.__godsEyeView.dataManager.layers.get('cctv').module; + const arrow = () => + mod._cctvDrawnForTest().gizmo['cctv-gizmo-move-east']?.positions; + if (!arrow()?.length) return null; + const Cartesian3 = viewer.camera.positionWC.constructor; const measure = () => { + const pts = arrow(); + if (!pts?.length) return { shaftPx: null, rangeM: null }; const a = viewer.scene.cartesianToCanvasCoordinates(pts[0]); const b = viewer.scene.cartesianToCanvasCoordinates(pts[pts.length - 1]); return { shaftPx: a && b ? Math.hypot(b.x - a.x, b.y - a.y) : null, - rangeM: target.constructor.distance(viewer.camera.positionWC, target), + rangeM: Cartesian3.distance(viewer.camera.positionWC, pts[0]), }; }; const before = measure(); + const target = Cartesian3.clone(arrow()[0]); viewer.camera.cancelFlight(); - viewer.camera.lookAt(target, new target.constructor(0, -200, 180)); + viewer.camera.lookAt(target, new Cartesian3(0, -200, 180)); viewer.camera.lookAtTransform(viewer.camera.transform.constructor.IDENTITY); viewer.scene.requestRender(); const frames = await window.__qaRenderFrames(3, 10000); return { before, after: measure(), frames }; }); - console.log('Drag fixture framing:', JSON.stringify(dragFraming)); + record( + 'drag fixture frames the east arrow above the 10px grab threshold', + dragFraming?.after?.shaftPx >= 10, + JSON.stringify(dragFraming), + ); // Synthetic CDP drag on the EAST arrow — the owner's field-test // regression. Sample effective elevation after every mouse move: it must diff --git a/server/providers/cctv/catalogPair.js b/server/providers/cctv/catalogPair.js new file mode 100644 index 0000000000..b6f0ffa901 --- /dev/null +++ b/server/providers/cctv/catalogPair.js @@ -0,0 +1,51 @@ +import { readResponseTextCapped } from '../common/http.js'; + +/** + * Read catalogs that are only useful together, because the pack joins them. + * + * The requests share one deadline and refuse redirects. Whichever fails first + * (before its headers, on its status, or while its capped body is read) aborts + * the others at once, so a stalled feed is not waited on until the deadline. + * Every response body is released, including one that arrives after another + * feed has already failed. + * + * @param {Array<{url: string, label: string}>} feeds `label` names the feed + * in the error ("locations download failed: 503"). + * @param {{accept: string, maxBytes: number, timeoutMs: number}} options + * @returns {Promise} The body texts, in feed order. + */ +export async function readCatalogsTogether( + feeds, + { accept, maxBytes, timeoutMs }, +) { + const sibling = new AbortController(); + const signal = AbortSignal.any([ + sibling.signal, + AbortSignal.timeout(timeoutMs), + ]); + return Promise.all( + feeds.map(async ({ url, label }) => { + let response; + try { + response = await fetch(url, { + headers: { Accept: accept }, + redirect: 'manual', + signal, + }); + signal.throwIfAborted(); + if (!response.ok) { + throw new Error( + response.status >= 300 && response.status < 400 + ? `${label} catalog redirected; redirects are not followed` + : `${label} download failed: ${response.status}`, + ); + } + return await readResponseTextCapped(response, maxBytes, signal); + } catch (error) { + sibling.abort(); + await response?.body?.cancel().catch(() => {}); + throw error; + } + }), + ); +} diff --git a/server/providers/cctv/lithuania.js b/server/providers/cctv/lithuania.js index 840e68c6cf..5ce67d411b 100644 --- a/server/providers/cctv/lithuania.js +++ b/server/providers/cctv/lithuania.js @@ -4,7 +4,7 @@ import { fallbackHeadingFromId, prioritizeSources, } from './normalize.js'; -import { readResponseJsonCapped } from '../common/http.js'; +import { readCatalogsTogether } from './catalogPair.js'; const BASE = 'https://eismoinfo.lt/eismoinfo-backend'; export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`; @@ -81,46 +81,22 @@ export function lithuaniaCameraToSource(row, feature) { }; } -async function readCatalog(url, sibling) { - const signal = AbortSignal.any([ - AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), - sibling.signal, - ]); - try { - const response = await fetch(url, { - headers: { Accept: 'application/json' }, - redirect: 'manual', - signal, - }); - if (!response.ok) { - await response.body?.cancel().catch(() => {}); - throw new Error( - response.status >= 300 && response.status < 400 - ? 'catalog redirected; redirects are not followed' - : `Catalog HTTP ${response.status}`, - ); - } - return await readResponseJsonCapped( - response, - LITHUANIA_MAX_CATALOG_BYTES, - signal, - ); - } catch (error) { - // The join needs both catalogs; release the other request at once - // rather than leaving its body open until the timeout. - sibling.abort(); - throw error; - } -} - /** Two bounded requests per shared catalog refresh, no API key required. */ export async function loadLithuaniaSources() { try { - const sibling = new AbortController(); - const [rows, layers] = await Promise.all([ - readCatalog(LITHUANIA_CAMERAS_URL, sibling), - readCatalog(LITHUANIA_LOCATIONS_URL, sibling), - ]); + const [rows, layers] = ( + await readCatalogsTogether( + [ + { url: LITHUANIA_CAMERAS_URL, label: 'cameras' }, + { url: LITHUANIA_LOCATIONS_URL, label: 'locations' }, + ], + { + accept: 'application/json', + maxBytes: LITHUANIA_MAX_CATALOG_BYTES, + timeoutMs: CCTV_SOURCE_FETCH_TIMEOUT_MS, + }, + ) + ).map((text) => JSON.parse(text)); if (!Array.isArray(rows) || !Array.isArray(layers)) return []; const features = layers.find((layer) => layer?.layer === 'VKR')?.features; if (!Array.isArray(features)) return []; diff --git a/server/providers/cctv/ontario511.test.mjs b/server/providers/cctv/ontario511.test.mjs index 9a8ba0e643..2f24388df9 100644 --- a/server/providers/cctv/ontario511.test.mjs +++ b/server/providers/cctv/ontario511.test.mjs @@ -167,3 +167,55 @@ test('Ontario encodes reserved key characters and refuses credential-bearing red assert.equal(JSON.stringify(cameras).includes('synthetic'), false); assert.equal(lines.join(' ').includes('synthetic'), false); }); + +test('Ontario names a refused redirect and a system code without the request text', async (t) => { + const lines = captureConsole(t); + const key = 'synthetic-secret'; + const failures = [ + // The shape Node's fetch gives a refused redirect and a DNS failure. + new TypeError('fetch failed', { + cause: new Error('unexpected redirect'), + }), + new TypeError('fetch failed', { + cause: Object.assign(new Error(`getaddrinfo ENOTFOUND ${key}`), { + code: 'ENOTFOUND', + }), + }), + ]; + t.mock.method(globalThis, 'fetch', async () => { + throw failures.shift(); + }); + for (let i = 0; i < 2; i++) { + assert.deepEqual( + await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: key }), + [], + ); + } + assert.deepEqual(lines, [ + '[CCTV] Ontario 511 camera download error (TypeError, redirect refused)', + '[CCTV] Ontario 511 camera download error (TypeError ENOTFOUND)', + ]); +}); + +test('Ontario releases the body of an error response', async (t) => { + captureConsole(t); + let cancelled = false; + t.mock.method( + globalThis, + 'fetch', + async () => + new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { status: 503 }, + ), + ); + assert.deepEqual( + await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: 'k' }), + [], + ); + assert.equal(cancelled, true); +}); diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 43cd0f1f3f..d7df49530a 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -172,6 +172,7 @@ import { pinnedCameraUrl, } from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; +import { readCatalogsTogether } from './catalogPair.js'; import { readResponseBytesCapped, readResponseJsonCapped, @@ -621,6 +622,7 @@ export async function loadOntarioSourcesFromOpenData( console.warn( `[CCTV] Ontario 511 camera download failed: ${resp.status}${keyHint}`, ); + await resp.body?.cancel().catch(() => {}); return []; } const rows = await resp.json(); @@ -1296,47 +1298,18 @@ export function parseTarkteeDatexImages(xml) { * @returns {Promise>} Normalized camera source objects. */ export async function loadTarkteeSourcesFromDatex() { - // One deadline for both feeds. Whichever read fails first aborts the other, - // so a stalled sibling is not waited on until the deadline. - const sibling = new AbortController(); - const signal = AbortSignal.any([ - sibling.signal, - AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), - ]); - const readXml = async (url, feed) => { - let response; - try { - response = await fetch(url, { - headers: { Accept: 'application/xml,text/xml,*/*' }, - redirect: 'manual', - signal, - }); - signal.throwIfAborted(); - if (!response.ok) { - throw new Error( - response.status >= 300 && response.status < 400 - ? `${feed} catalog redirected; redirects are not followed` - : `${feed} download failed: ${response.status}`, - ); - } - return await readResponseTextCapped( - response, - TARKTEE_MAX_CATALOG_BYTES, - signal, - ); - } catch (error) { - // The join needs both feeds: stop the other one now, and release this - // body, including a response that arrived after the other failed. - sibling.abort(); - await response?.body?.cancel().catch(() => {}); - throw error; - } - }; try { - const [locXml, imgXml] = await Promise.all([ - readXml(TARKTEE_LOCATIONS_URL, 'locations'), - readXml(TARKTEE_IMAGES_URL, 'images'), - ]); + const [locXml, imgXml] = await readCatalogsTogether( + [ + { url: TARKTEE_LOCATIONS_URL, label: 'locations' }, + { url: TARKTEE_IMAGES_URL, label: 'images' }, + ], + { + accept: 'application/xml,text/xml,*/*', + maxBytes: TARKTEE_MAX_CATALOG_BYTES, + timeoutMs: TARKTEE_FETCH_TIMEOUT_MS, + }, + ); const locations = parseTarkteeDatexLocations(locXml); const images = parseTarkteeDatexImages(imgXml); if (!locations.size || !images.size) {