Merge main into codex/fix-cctv-session-admission

Main's Street View budget tests in the media feed-type file now mount the
provider through the shared CCTV test harness, which also resets the Google
rate-limit setting.
This commit is contained in:
German Khokhlov committed 2026-11-20 17:39:46 +04:00
commit 23efd3b343
196 files changed
+21064 -562

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
import { readResponseTextCapped } from '../common/http.js';
/**
* Read catalogs that are only useful together, because the pack joins them.
*
* The requests share one deadline and refuse redirects. Whichever fails first
* (before its headers, on its status, or while its capped body is read or
* parsed) aborts the others at once, so a stalled feed is not waited on until
* the deadline. Every response body is released, including one that arrives
* after another feed has already failed.
*
* @param {Array<{url: string, label: string}>} feeds `label` names the feed
* in the error ("locations download failed: 503").
* @param {{accept: string, maxBytes: number, timeoutMs: number,
* parse?: (text: string) => unknown}} options
* @returns {Promise<unknown[]>} The parsed bodies (texts without `parse`), in
* feed order.
*/
export async function readCatalogsTogether(
feeds,
{ accept, maxBytes, timeoutMs, parse = (text) => text },
) {
const sibling = new AbortController();
const signal = AbortSignal.any([
sibling.signal,
AbortSignal.timeout(timeoutMs),
]);
return Promise.all(
feeds.map(async ({ url, label }) => {
let response;
try {
response = await fetch(url, {
headers: { Accept: accept },
redirect: 'manual',
signal,
});
signal.throwIfAborted();
if (!response.ok) {
throw new Error(
response.status >= 300 && response.status < 400
? `${label} catalog redirected; redirects are not followed`
: `${label} download failed: ${response.status}`,
);
}
return parse(await readResponseTextCapped(response, maxBytes, signal));
} catch (error) {
sibling.abort();
await response?.body?.cancel().catch(() => {});
throw error;
}
}),
);
}
+13 -38
View File
@@ -4,7 +4,7 @@ import {
fallbackHeadingFromId,
prioritizeSources,
} from './normalize.js';
import { readResponseJsonCapped } from '../common/http.js';
import { readCatalogsTogether } from './catalogPair.js';
const BASE = 'https://eismoinfo.lt/eismoinfo-backend';
export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`;
@@ -81,46 +81,21 @@ export function lithuaniaCameraToSource(row, feature) {
};
}
async function readCatalog(url, sibling) {
const signal = AbortSignal.any([
AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
sibling.signal,
]);
try {
const response = await fetch(url, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `Catalog HTTP ${response.status}`,
);
}
return await readResponseJsonCapped(
response,
LITHUANIA_MAX_CATALOG_BYTES,
signal,
);
} catch (error) {
// The join needs both catalogs; release the other request at once
// rather than leaving its body open until the timeout.
sibling.abort();
throw error;
}
}
/** Two bounded requests per shared catalog refresh, no API key required. */
export async function loadLithuaniaSources() {
try {
const sibling = new AbortController();
const [rows, layers] = await Promise.all([
readCatalog(LITHUANIA_CAMERAS_URL, sibling),
readCatalog(LITHUANIA_LOCATIONS_URL, sibling),
]);
const [rows, layers] = await readCatalogsTogether(
[
{ url: LITHUANIA_CAMERAS_URL, label: 'cameras' },
{ url: LITHUANIA_LOCATIONS_URL, label: 'locations' },
],
{
accept: 'application/json',
maxBytes: LITHUANIA_MAX_CATALOG_BYTES,
timeoutMs: CCTV_SOURCE_FETCH_TIMEOUT_MS,
parse: JSON.parse,
},
);
if (!Array.isArray(rows) || !Array.isArray(layers)) return [];
const features = layers.find((layer) => layer?.layer === 'VKR')?.features;
if (!Array.isArray(features)) return [];
+92
View File
@@ -127,3 +127,95 @@ for (const status of [429, 500, 503]) {
assert.doesNotMatch(lines[0], /secret-key-value/);
});
}
for (const failure of ['transport', 'invalid-json']) {
test(`Ontario ${failure} failures cannot log reflected keys or request URLs`, async (t) => {
const lines = captureConsole(t);
const key = 'synthetic secret&/=%value';
t.mock.method(globalThis, 'fetch', async (url) => {
if (failure === 'transport')
throw new Error(`request failed: ${url} ${key}`);
return new Response(`not JSON: ${url} ${key}`);
});
assert.deepEqual(
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: key }),
[],
);
assert.equal(lines.length, 1);
assert.match(lines[0], /Ontario 511 camera download error/);
assert.equal(lines[0].includes(key), false);
assert.equal(lines[0].includes('synthetic'), false);
assert.equal(lines[0].includes('https://'), false);
});
}
test('Ontario encodes reserved key characters and refuses credential-bearing redirects', async (t) => {
const lines = captureConsole(t);
const key = 'synthetic &/=% value';
let request;
t.mock.method(globalThis, 'fetch', async (url, init) => {
request = { url: new URL(url), init };
return Response.json([ROW]);
});
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: key },
);
assert.equal(request.url.searchParams.get('key'), key);
assert.equal(request.init.redirect, 'error');
assert.equal(cameras.length, 1);
assert.equal(JSON.stringify(cameras).includes('synthetic'), false);
assert.equal(lines.join(' ').includes('synthetic'), false);
});
test('Ontario names a refused redirect and a system code without the request text', async (t) => {
const lines = captureConsole(t);
const key = 'synthetic-secret';
const failures = [
// The shape Node's fetch gives a refused redirect and a DNS failure.
new TypeError('fetch failed', {
cause: new Error('unexpected redirect'),
}),
new TypeError('fetch failed', {
cause: Object.assign(new Error(`getaddrinfo ENOTFOUND ${key}`), {
code: 'ENOTFOUND',
}),
}),
];
t.mock.method(globalThis, 'fetch', async () => {
throw failures.shift();
});
for (let i = 0; i < 2; i++) {
assert.deepEqual(
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: key }),
[],
);
}
assert.deepEqual(lines, [
'[CCTV] Ontario 511 camera download error (TypeError, redirect refused)',
'[CCTV] Ontario 511 camera download error (TypeError ENOTFOUND)',
]);
});
test('Ontario releases the body of an error response', async (t) => {
captureConsole(t);
let cancelled = false;
t.mock.method(
globalThis,
'fetch',
async () =>
new Response(
new ReadableStream({
cancel() {
cancelled = true;
},
}),
{ status: 503 },
),
);
assert.deepEqual(
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: 'k' }),
[],
);
assert.equal(cancelled, true);
});
+34 -70
View File
@@ -43,6 +43,7 @@ import {
TARKTEE_IMAGE_ORIGIN,
DEFAULT_TARKTEE_MAX_SOURCES,
TARKTEE_MAX_CATALOG_BYTES,
TARKTEE_FETCH_TIMEOUT_MS,
TARKTEE_ANCHORS,
DEFAULT_WARENDORF_SOURCE_FILE,
WARENDORF_IMAGE_ORIGINS,
@@ -82,7 +83,6 @@ import {
HK_ANCHORS,
HK_MAX_CATALOG_BYTES,
CCTV_SOURCE_FETCH_TIMEOUT_MS,
TARKTEE_FETCH_TIMEOUT_MS,
DELDOT_CCTV_URL,
DEFAULT_DELDOT_MAX_SOURCES,
DELDOT_ANCHORS,
@@ -172,6 +172,7 @@ import {
pinnedCameraUrl,
} from './normalize.js';
import { directionToHeading } from '../../../src/data/directionText.js';
import { readCatalogsTogether } from './catalogPair.js';
import {
readResponseBytesCapped,
readResponseJsonCapped,
@@ -561,6 +562,22 @@ export function _resetOntarioKeyWarningForTest() {
ontarioKeyMissingLogged = false;
}
/**
* What failed in an Ontario download, without upstream text: the error class,
* plus the system code (`ENOTFOUND`) or a refused redirect when there is one.
* @param {unknown} error
* @returns {string}
*/
function ontarioFailureKind(error) {
const name = /^[A-Za-z]{1,40}$/.test(error?.name) ? error.name : 'Error';
if (error?.cause?.message === 'unexpected redirect')
return `${name}, redirect refused`;
const code = error?.cause?.code;
return typeof code === 'string' && /^[A-Z][A-Z0-9_]{1,40}$/.test(code)
? `${name} ${code}`
: name;
}
/**
* Fetch Ontario 511 CCTV cameras. The camera list comes from the 511 API,
* which now requires a free developer key (511on.ca/developers/doc, "Requires a
@@ -593,6 +610,7 @@ export async function loadOntarioSourcesFromOpenData(
url.searchParams.set('key', key);
const resp = await fetch(url, {
headers: { Accept: 'application/json' },
redirect: 'error',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) {
@@ -604,6 +622,7 @@ export async function loadOntarioSourcesFromOpenData(
console.warn(
`[CCTV] Ontario 511 camera download failed: ${resp.status}${keyHint}`,
);
await resp.body?.cancel().catch(() => {});
return [];
}
const rows = await resp.json();
@@ -658,7 +677,7 @@ export async function loadOntarioSourcesFromOpenData(
url: view.url,
snapshotUrl: view.url,
sourceKind: 'ontario-511-open-data',
license: 'Open Government Licence - Ontario',
license: 'Open Government Licence – Ontario',
});
}
@@ -677,9 +696,10 @@ export async function loadOntarioSourcesFromOpenData(
);
return prioritized;
} catch (error) {
// Transport and JSON errors may reflect the credential-bearing request,
// so only the error class and a system code are logged, never the text.
console.warn(
'[CCTV] Ontario 511 camera download error:',
error?.message || error,
`[CCTV] Ontario 511 camera download error (${ontarioFailureKind(error)})`,
);
return [];
}
@@ -1279,73 +1299,17 @@ export function parseTarkteeDatexImages(xml) {
*/
export async function loadTarkteeSourcesFromDatex() {
try {
const responses = await Promise.allSettled([
fetch(TARKTEE_LOCATIONS_URL, {
headers: { Accept: 'application/xml,text/xml,*/*' },
redirect: 'manual',
signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS),
}),
fetch(TARKTEE_IMAGES_URL, {
headers: { Accept: 'application/xml,text/xml,*/*' },
redirect: 'manual',
signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS),
}),
]);
const [locResp, imgResp] = responses.map((result) =>
result.status === 'fulfilled' ? result.value : undefined,
const [locXml, imgXml] = await readCatalogsTogether(
[
{ url: TARKTEE_LOCATIONS_URL, label: 'locations' },
{ url: TARKTEE_IMAGES_URL, label: 'images' },
],
{
accept: 'application/xml,text/xml,*/*',
maxBytes: TARKTEE_MAX_CATALOG_BYTES,
timeoutMs: TARKTEE_FETCH_TIMEOUT_MS,
},
);
// Both responses are fetched in parallel; if either is rejected or overflows
// its byte cap, both bodies must be released so neither transport leaks.
const discardBoth = async () => {
await Promise.allSettled([
locResp?.body?.cancel(),
imgResp?.body?.cancel(),
]);
return [];
};
const rejected = responses.find((result) => result.status === 'rejected');
if (rejected) {
await discardBoth();
throw rejected.reason;
}
if (
(locResp.status >= 300 && locResp.status < 400) ||
(imgResp.status >= 300 && imgResp.status < 400)
) {
console.warn(
'[CCTV] Tarktee DATEX catalog redirected; redirects are not followed',
);
return discardBoth();
}
if (!locResp.ok) {
console.warn('[CCTV] Tarktee locations download failed:', locResp.status);
return discardBoth();
}
if (!imgResp.ok) {
console.warn('[CCTV] Tarktee images download failed:', imgResp.status);
return discardBoth();
}
let locXml;
let imgXml;
const readAbort = new AbortController();
try {
[locXml, imgXml] = await Promise.all([
readResponseTextCapped(
locResp,
TARKTEE_MAX_CATALOG_BYTES,
readAbort.signal,
),
readResponseTextCapped(
imgResp,
TARKTEE_MAX_CATALOG_BYTES,
readAbort.signal,
),
]);
} catch (error) {
readAbort.abort();
await discardBoth();
throw error;
}
const locations = parseTarkteeDatexLocations(locXml);
const images = parseTarkteeDatexImages(imgXml);
if (!locations.size || !images.size) {