mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-17 08:42:21 +00:00
Merge upstream/main into feat/cctv-a22-autobrennero
Resolve conflicts with the NYC, Toronto, Lithuania, Florida, Taiwan and Kartverket work on main. The CHANGELOG entry moves to changelog.d/956-a22-brennero-webcams.md, as main now asks.
This commit is contained in:
commit
38302a27a4
1369 files changed
+189595
-25591
No files matched your search
@@ -4,6 +4,8 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
|
||||
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
|
||||
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
|
||||
import { cameraRegion, normalizeSourceItem } from './normalize.js';
|
||||
import { loadLithuaniaSources } from './lithuania.js';
|
||||
import { loadTaiwanSources } from './taiwan.js';
|
||||
import {
|
||||
loadAustinSourcesFromOpenData,
|
||||
loadCaltransSourcesFromOpenData,
|
||||
@@ -17,20 +19,33 @@ import {
|
||||
loadWarendorfSourcesFromCatalog,
|
||||
loadNswSourcesFromOpenData,
|
||||
loadCalgarySourcesFromOpenData,
|
||||
loadCataloniaSourcesFromOpenData,
|
||||
loadHkSourcesFromOpenData,
|
||||
loadDelDOTSourcesFromOpenData,
|
||||
loadKingCountySourcesFromOpenData,
|
||||
loadWsdotSourcesFromOpenData,
|
||||
loadVegvesenSourcesFromOpenData,
|
||||
loadNycSourcesFromOpenData,
|
||||
loadTorontoSourcesFromOpenData,
|
||||
loadFloridaSourcesFromOpenData,
|
||||
loadA22SourcesFromOpenDataHub,
|
||||
} from './sources.js';
|
||||
|
||||
/** Env kill switch: unset or anything but "0" means enabled. */
|
||||
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
|
||||
/** Env opt-in: only "1" enables. For packs whose feed access needs an
|
||||
* operator agreement or whose terms are unconfirmed, so an unset flag never
|
||||
* reaches the upstream. */
|
||||
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
|
||||
|
||||
/**
|
||||
* Live open-data packs, in merge order. Adding a region is one entry here
|
||||
* plus its loader in sources.js; the catalog cap is shared across entries
|
||||
* round-robin (cap.js), so a new pack never silently evicts an older one.
|
||||
* plus its loader (in sources.js or its own module); the catalog cap is
|
||||
* shared across entries round-robin (cap.js), so a new pack never silently
|
||||
* evicts an older one.
|
||||
* Each pack fails independently (allSettled) and is gated by its own env
|
||||
* kill switch.
|
||||
* kill switch, or by an opt-in flag where access needs an agreement or the
|
||||
* terms are unconfirmed.
|
||||
*/
|
||||
const LIVE_PACKS = [
|
||||
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
|
||||
@@ -89,16 +104,68 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_CALGARY_ENABLED'),
|
||||
load: loadCalgarySourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'catalonia',
|
||||
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
|
||||
load: loadCataloniaSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'hk',
|
||||
enabled: () => envEnabled('CCTV_HK_ENABLED'),
|
||||
load: loadHkSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'deldot',
|
||||
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
|
||||
load: loadDelDOTSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'kingcounty',
|
||||
enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'),
|
||||
load: loadKingCountySourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'wsdot',
|
||||
enabled: () => envEnabled('CCTV_WSDOT_ENABLED'),
|
||||
load: loadWsdotSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'vegvesen',
|
||||
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
|
||||
load: loadVegvesenSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
|
||||
// off unless the operator opts in (see DATA_SOURCES.md).
|
||||
name: 'nyc',
|
||||
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
|
||||
load: loadNycSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'toronto',
|
||||
enabled: () => envEnabled('CCTV_TORONTO_ENABLED'),
|
||||
load: loadTorontoSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
// Opt-in until the Via Lietuva open-data terms are confirmed to cover the
|
||||
// camera photos (see DATA_SOURCES.md).
|
||||
name: 'lithuania',
|
||||
enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'),
|
||||
load: loadLithuaniaSources,
|
||||
},
|
||||
{
|
||||
// FL511's terms limit its content to individual, personal use (other use
|
||||
// needs FDOT's written consent), so the pack is off unless the operator
|
||||
// opts in (see DATA_SOURCES.md).
|
||||
name: 'florida',
|
||||
enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
|
||||
load: loadFloridaSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'taiwan',
|
||||
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
|
||||
load: loadTaiwanSources,
|
||||
},
|
||||
{
|
||||
name: 'a22',
|
||||
enabled: () => envEnabled('CCTV_A22_ENABLED'),
|
||||
@@ -155,16 +222,19 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
/** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
|
||||
* callers so a post-TTL burst launches ONE refetch, not one per request. */
|
||||
let _cctvSourceInflight = null;
|
||||
/** @type {Array<{pack: string, available: number, served: number}>} Packs
|
||||
* that serve fewer cameras than they offer. */
|
||||
let _cctvTrimmedPacks = [];
|
||||
/** @type {WeakMap<Array<object>, Array<{pack: string, available: number, served: number}>>}
|
||||
* Packs that serve fewer cameras than they offer, keyed by the source list
|
||||
* they describe, so a request served the old list during a background
|
||||
* refresh never pairs it with the new list's trim notes. */
|
||||
const _cctvTrimmedPacks = new WeakMap();
|
||||
|
||||
/**
|
||||
* Assemble and cache the merged CCTV source list.
|
||||
*
|
||||
* Merges every source pack (live open-data packs, local file, env
|
||||
* variable), deduplicates by ID, shares the catalog cap fairly across
|
||||
* packs, and caches for CCTV_SOURCE_CACHE_MS.
|
||||
* packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is
|
||||
* returned while a background refresh replaces it; only a first load waits.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Deduplicated, capped source list.
|
||||
*/
|
||||
@@ -179,11 +249,31 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
// Single-flight: a burst of requests arriving past the TTL shares ONE refresh
|
||||
// instead of each launching the full multi-provider refetch. The `.finally`
|
||||
// clears the ref so the next post-TTL cycle starts fresh.
|
||||
if (_cctvSourceInflight) return _cctvSourceInflight;
|
||||
_cctvSourceInflight = refreshCctvSources().finally(() => {
|
||||
_cctvSourceInflight = null;
|
||||
});
|
||||
return _cctvSourceInflight;
|
||||
if (!_cctvSourceInflight) {
|
||||
_cctvSourceInflight = refreshCctvSources().finally(() => {
|
||||
_cctvSourceInflight = null;
|
||||
});
|
||||
// A background refresh has no caller waiting on it: report its failure
|
||||
// here, once per refresh rather than once per request it serves. Keep
|
||||
// the old catalog for another TTL, as an empty refresh does, so a refresh
|
||||
// that keeps failing is retried once per TTL, not back to back.
|
||||
if (_cctvSourceCache.length) {
|
||||
_cctvSourceInflight.catch((error) => {
|
||||
_cctvSourceCacheAt = Date.now();
|
||||
console.warn(
|
||||
'[CCTV] background catalog refresh failed; serving',
|
||||
_cctvSourceCache.length,
|
||||
'cameras from the last catalog:',
|
||||
error?.message || error,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
// With a catalog in hand, answer with it while the refresh runs: the
|
||||
// refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up
|
||||
// to its 30 s timeout), and every /api/cctv request would otherwise wait
|
||||
// for it. Only the first load, with nothing to serve yet, waits.
|
||||
return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -273,7 +363,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
}
|
||||
if (capped.length > 0 || _cctvSourceCache.length === 0) {
|
||||
_cctvSourceCache = capped;
|
||||
_cctvTrimmedPacks = trimmedPacks;
|
||||
_cctvTrimmedPacks.set(capped, trimmedPacks);
|
||||
} else {
|
||||
// Every source came back empty (all live packs timed out / upstream outage)
|
||||
// but a good catalog is already cached — serve it stale rather than blanking
|
||||
@@ -287,7 +377,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
return _cctvSourceCache;
|
||||
}
|
||||
|
||||
/** Packs serving fewer cameras than they offer, as of the last refresh. */
|
||||
getCctvSources.trimmedPacks = () => _cctvTrimmedPacks;
|
||||
/** Packs serving fewer cameras than they offer in `sources` (by default the
|
||||
* current catalog). Pass the list a request was served to keep the two in
|
||||
* step while a background refresh swaps the catalog. */
|
||||
getCctvSources.trimmedPacks = (sources = _cctvSourceCache) =>
|
||||
_cctvTrimmedPacks.get(sources) ?? [];
|
||||
return getCctvSources;
|
||||
}
|
||||
Reference in new issue
Block a user