mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-21 02:32:18 +00:00
Merge main into codex/fix-cctv-session-admission
Main's Street View budget tests in the media feed-type file now mount the provider through the shared CCTV test harness, which also resets the Google rate-limit setting.
This commit is contained in:
commit
23efd3b343
196 files changed
+21064
-562
No files matched your search
@@ -0,0 +1,53 @@
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Read catalogs that are only useful together, because the pack joins them.
|
||||
*
|
||||
* The requests share one deadline and refuse redirects. Whichever fails first
|
||||
* (before its headers, on its status, or while its capped body is read or
|
||||
* parsed) aborts the others at once, so a stalled feed is not waited on until
|
||||
* the deadline. Every response body is released, including one that arrives
|
||||
* after another feed has already failed.
|
||||
*
|
||||
* @param {Array<{url: string, label: string}>} feeds `label` names the feed
|
||||
* in the error ("locations download failed: 503").
|
||||
* @param {{accept: string, maxBytes: number, timeoutMs: number,
|
||||
* parse?: (text: string) => unknown}} options
|
||||
* @returns {Promise<unknown[]>} The parsed bodies (texts without `parse`), in
|
||||
* feed order.
|
||||
*/
|
||||
export async function readCatalogsTogether(
|
||||
feeds,
|
||||
{ accept, maxBytes, timeoutMs, parse = (text) => text },
|
||||
) {
|
||||
const sibling = new AbortController();
|
||||
const signal = AbortSignal.any([
|
||||
sibling.signal,
|
||||
AbortSignal.timeout(timeoutMs),
|
||||
]);
|
||||
return Promise.all(
|
||||
feeds.map(async ({ url, label }) => {
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
headers: { Accept: accept },
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
});
|
||||
signal.throwIfAborted();
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
response.status >= 300 && response.status < 400
|
||||
? `${label} catalog redirected; redirects are not followed`
|
||||
: `${label} download failed: ${response.status}`,
|
||||
);
|
||||
}
|
||||
return parse(await readResponseTextCapped(response, maxBytes, signal));
|
||||
} catch (error) {
|
||||
sibling.abort();
|
||||
await response?.body?.cancel().catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
fallbackHeadingFromId,
|
||||
prioritizeSources,
|
||||
} from './normalize.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import { readCatalogsTogether } from './catalogPair.js';
|
||||
|
||||
const BASE = 'https://eismoinfo.lt/eismoinfo-backend';
|
||||
export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`;
|
||||
@@ -81,46 +81,21 @@ export function lithuaniaCameraToSource(row, feature) {
|
||||
};
|
||||
}
|
||||
|
||||
async function readCatalog(url, sibling) {
|
||||
const signal = AbortSignal.any([
|
||||
AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
sibling.signal,
|
||||
]);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw new Error(
|
||||
response.status >= 300 && response.status < 400
|
||||
? 'catalog redirected; redirects are not followed'
|
||||
: `Catalog HTTP ${response.status}`,
|
||||
);
|
||||
}
|
||||
return await readResponseJsonCapped(
|
||||
response,
|
||||
LITHUANIA_MAX_CATALOG_BYTES,
|
||||
signal,
|
||||
);
|
||||
} catch (error) {
|
||||
// The join needs both catalogs; release the other request at once
|
||||
// rather than leaving its body open until the timeout.
|
||||
sibling.abort();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Two bounded requests per shared catalog refresh, no API key required. */
|
||||
export async function loadLithuaniaSources() {
|
||||
try {
|
||||
const sibling = new AbortController();
|
||||
const [rows, layers] = await Promise.all([
|
||||
readCatalog(LITHUANIA_CAMERAS_URL, sibling),
|
||||
readCatalog(LITHUANIA_LOCATIONS_URL, sibling),
|
||||
]);
|
||||
const [rows, layers] = await readCatalogsTogether(
|
||||
[
|
||||
{ url: LITHUANIA_CAMERAS_URL, label: 'cameras' },
|
||||
{ url: LITHUANIA_LOCATIONS_URL, label: 'locations' },
|
||||
],
|
||||
{
|
||||
accept: 'application/json',
|
||||
maxBytes: LITHUANIA_MAX_CATALOG_BYTES,
|
||||
timeoutMs: CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
parse: JSON.parse,
|
||||
},
|
||||
);
|
||||
if (!Array.isArray(rows) || !Array.isArray(layers)) return [];
|
||||
const features = layers.find((layer) => layer?.layer === 'VKR')?.features;
|
||||
if (!Array.isArray(features)) return [];
|
||||
|
||||
@@ -127,3 +127,95 @@ for (const status of [429, 500, 503]) {
|
||||
assert.doesNotMatch(lines[0], /secret-key-value/);
|
||||
});
|
||||
}
|
||||
|
||||
for (const failure of ['transport', 'invalid-json']) {
|
||||
test(`Ontario ${failure} failures cannot log reflected keys or request URLs`, async (t) => {
|
||||
const lines = captureConsole(t);
|
||||
const key = 'synthetic secret&/=%value';
|
||||
t.mock.method(globalThis, 'fetch', async (url) => {
|
||||
if (failure === 'transport')
|
||||
throw new Error(`request failed: ${url} ${key}`);
|
||||
return new Response(`not JSON: ${url} ${key}`);
|
||||
});
|
||||
assert.deepEqual(
|
||||
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: key }),
|
||||
[],
|
||||
);
|
||||
assert.equal(lines.length, 1);
|
||||
assert.match(lines[0], /Ontario 511 camera download error/);
|
||||
assert.equal(lines[0].includes(key), false);
|
||||
assert.equal(lines[0].includes('synthetic'), false);
|
||||
assert.equal(lines[0].includes('https://'), false);
|
||||
});
|
||||
}
|
||||
|
||||
test('Ontario encodes reserved key characters and refuses credential-bearing redirects', async (t) => {
|
||||
const lines = captureConsole(t);
|
||||
const key = 'synthetic &/=% value';
|
||||
let request;
|
||||
t.mock.method(globalThis, 'fetch', async (url, init) => {
|
||||
request = { url: new URL(url), init };
|
||||
return Response.json([ROW]);
|
||||
});
|
||||
const cameras = await loadOntarioSourcesFromOpenData(
|
||||
{},
|
||||
{ ONTARIO_511_API_KEY: key },
|
||||
);
|
||||
assert.equal(request.url.searchParams.get('key'), key);
|
||||
assert.equal(request.init.redirect, 'error');
|
||||
assert.equal(cameras.length, 1);
|
||||
assert.equal(JSON.stringify(cameras).includes('synthetic'), false);
|
||||
assert.equal(lines.join(' ').includes('synthetic'), false);
|
||||
});
|
||||
|
||||
test('Ontario names a refused redirect and a system code without the request text', async (t) => {
|
||||
const lines = captureConsole(t);
|
||||
const key = 'synthetic-secret';
|
||||
const failures = [
|
||||
// The shape Node's fetch gives a refused redirect and a DNS failure.
|
||||
new TypeError('fetch failed', {
|
||||
cause: new Error('unexpected redirect'),
|
||||
}),
|
||||
new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error(`getaddrinfo ENOTFOUND ${key}`), {
|
||||
code: 'ENOTFOUND',
|
||||
}),
|
||||
}),
|
||||
];
|
||||
t.mock.method(globalThis, 'fetch', async () => {
|
||||
throw failures.shift();
|
||||
});
|
||||
for (let i = 0; i < 2; i++) {
|
||||
assert.deepEqual(
|
||||
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: key }),
|
||||
[],
|
||||
);
|
||||
}
|
||||
assert.deepEqual(lines, [
|
||||
'[CCTV] Ontario 511 camera download error (TypeError, redirect refused)',
|
||||
'[CCTV] Ontario 511 camera download error (TypeError ENOTFOUND)',
|
||||
]);
|
||||
});
|
||||
|
||||
test('Ontario releases the body of an error response', async (t) => {
|
||||
captureConsole(t);
|
||||
let cancelled = false;
|
||||
t.mock.method(
|
||||
globalThis,
|
||||
'fetch',
|
||||
async () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
cancel() {
|
||||
cancelled = true;
|
||||
},
|
||||
}),
|
||||
{ status: 503 },
|
||||
),
|
||||
);
|
||||
assert.deepEqual(
|
||||
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: 'k' }),
|
||||
[],
|
||||
);
|
||||
assert.equal(cancelled, true);
|
||||
});
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
TARKTEE_IMAGE_ORIGIN,
|
||||
DEFAULT_TARKTEE_MAX_SOURCES,
|
||||
TARKTEE_MAX_CATALOG_BYTES,
|
||||
TARKTEE_FETCH_TIMEOUT_MS,
|
||||
TARKTEE_ANCHORS,
|
||||
DEFAULT_WARENDORF_SOURCE_FILE,
|
||||
WARENDORF_IMAGE_ORIGINS,
|
||||
@@ -82,7 +83,6 @@ import {
|
||||
HK_ANCHORS,
|
||||
HK_MAX_CATALOG_BYTES,
|
||||
CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
TARKTEE_FETCH_TIMEOUT_MS,
|
||||
DELDOT_CCTV_URL,
|
||||
DEFAULT_DELDOT_MAX_SOURCES,
|
||||
DELDOT_ANCHORS,
|
||||
@@ -172,6 +172,7 @@ import {
|
||||
pinnedCameraUrl,
|
||||
} from './normalize.js';
|
||||
import { directionToHeading } from '../../../src/data/directionText.js';
|
||||
import { readCatalogsTogether } from './catalogPair.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseJsonCapped,
|
||||
@@ -561,6 +562,22 @@ export function _resetOntarioKeyWarningForTest() {
|
||||
ontarioKeyMissingLogged = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* What failed in an Ontario download, without upstream text: the error class,
|
||||
* plus the system code (`ENOTFOUND`) or a refused redirect when there is one.
|
||||
* @param {unknown} error
|
||||
* @returns {string}
|
||||
*/
|
||||
function ontarioFailureKind(error) {
|
||||
const name = /^[A-Za-z]{1,40}$/.test(error?.name) ? error.name : 'Error';
|
||||
if (error?.cause?.message === 'unexpected redirect')
|
||||
return `${name}, redirect refused`;
|
||||
const code = error?.cause?.code;
|
||||
return typeof code === 'string' && /^[A-Z][A-Z0-9_]{1,40}$/.test(code)
|
||||
? `${name} ${code}`
|
||||
: name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Ontario 511 CCTV cameras. The camera list comes from the 511 API,
|
||||
* which now requires a free developer key (511on.ca/developers/doc, "Requires a
|
||||
@@ -593,6 +610,7 @@ export async function loadOntarioSourcesFromOpenData(
|
||||
url.searchParams.set('key', key);
|
||||
const resp = await fetch(url, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
@@ -604,6 +622,7 @@ export async function loadOntarioSourcesFromOpenData(
|
||||
console.warn(
|
||||
`[CCTV] Ontario 511 camera download failed: ${resp.status}${keyHint}`,
|
||||
);
|
||||
await resp.body?.cancel().catch(() => {});
|
||||
return [];
|
||||
}
|
||||
const rows = await resp.json();
|
||||
@@ -658,7 +677,7 @@ export async function loadOntarioSourcesFromOpenData(
|
||||
url: view.url,
|
||||
snapshotUrl: view.url,
|
||||
sourceKind: 'ontario-511-open-data',
|
||||
license: 'Open Government Licence - Ontario',
|
||||
license: 'Open Government Licence – Ontario',
|
||||
});
|
||||
}
|
||||
|
||||
@@ -677,9 +696,10 @@ export async function loadOntarioSourcesFromOpenData(
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
// Transport and JSON errors may reflect the credential-bearing request,
|
||||
// so only the error class and a system code are logged, never the text.
|
||||
console.warn(
|
||||
'[CCTV] Ontario 511 camera download error:',
|
||||
error?.message || error,
|
||||
`[CCTV] Ontario 511 camera download error (${ontarioFailureKind(error)})`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
@@ -1279,73 +1299,17 @@ export function parseTarkteeDatexImages(xml) {
|
||||
*/
|
||||
export async function loadTarkteeSourcesFromDatex() {
|
||||
try {
|
||||
const responses = await Promise.allSettled([
|
||||
fetch(TARKTEE_LOCATIONS_URL, {
|
||||
headers: { Accept: 'application/xml,text/xml,*/*' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS),
|
||||
}),
|
||||
fetch(TARKTEE_IMAGES_URL, {
|
||||
headers: { Accept: 'application/xml,text/xml,*/*' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS),
|
||||
}),
|
||||
]);
|
||||
const [locResp, imgResp] = responses.map((result) =>
|
||||
result.status === 'fulfilled' ? result.value : undefined,
|
||||
const [locXml, imgXml] = await readCatalogsTogether(
|
||||
[
|
||||
{ url: TARKTEE_LOCATIONS_URL, label: 'locations' },
|
||||
{ url: TARKTEE_IMAGES_URL, label: 'images' },
|
||||
],
|
||||
{
|
||||
accept: 'application/xml,text/xml,*/*',
|
||||
maxBytes: TARKTEE_MAX_CATALOG_BYTES,
|
||||
timeoutMs: TARKTEE_FETCH_TIMEOUT_MS,
|
||||
},
|
||||
);
|
||||
// Both responses are fetched in parallel; if either is rejected or overflows
|
||||
// its byte cap, both bodies must be released so neither transport leaks.
|
||||
const discardBoth = async () => {
|
||||
await Promise.allSettled([
|
||||
locResp?.body?.cancel(),
|
||||
imgResp?.body?.cancel(),
|
||||
]);
|
||||
return [];
|
||||
};
|
||||
const rejected = responses.find((result) => result.status === 'rejected');
|
||||
if (rejected) {
|
||||
await discardBoth();
|
||||
throw rejected.reason;
|
||||
}
|
||||
if (
|
||||
(locResp.status >= 300 && locResp.status < 400) ||
|
||||
(imgResp.status >= 300 && imgResp.status < 400)
|
||||
) {
|
||||
console.warn(
|
||||
'[CCTV] Tarktee DATEX catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discardBoth();
|
||||
}
|
||||
if (!locResp.ok) {
|
||||
console.warn('[CCTV] Tarktee locations download failed:', locResp.status);
|
||||
return discardBoth();
|
||||
}
|
||||
if (!imgResp.ok) {
|
||||
console.warn('[CCTV] Tarktee images download failed:', imgResp.status);
|
||||
return discardBoth();
|
||||
}
|
||||
let locXml;
|
||||
let imgXml;
|
||||
const readAbort = new AbortController();
|
||||
try {
|
||||
[locXml, imgXml] = await Promise.all([
|
||||
readResponseTextCapped(
|
||||
locResp,
|
||||
TARKTEE_MAX_CATALOG_BYTES,
|
||||
readAbort.signal,
|
||||
),
|
||||
readResponseTextCapped(
|
||||
imgResp,
|
||||
TARKTEE_MAX_CATALOG_BYTES,
|
||||
readAbort.signal,
|
||||
),
|
||||
]);
|
||||
} catch (error) {
|
||||
readAbort.abort();
|
||||
await discardBoth();
|
||||
throw error;
|
||||
}
|
||||
const locations = parseTarkteeDatexLocations(locXml);
|
||||
const images = parseTarkteeDatexImages(imgXml);
|
||||
if (!locations.size || !images.size) {
|
||||
|
||||
Reference in new issue
Block a user