mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 02:14:01 +00:00
fix(cctv): FL511 written-consent wording and robust catalog paging
- State FL511's terms the same way everywhere: individual, personal use only, and any other use (a public deployment included) needs FDOT's express written consent. .env.example, README, dataCredits, the source licence string, the catalog comment and the changelog fragment said only "individual, non-commercial use". - Page the ArcGIS catalog on exceededTransferLimit alone and advance the offset by the rows each page returned, so a lower server-side max record count no longer cuts the catalog short at the first short page. - Order pages by OBJECTID_1 (this layer's object id; a plain OBJECTID is rejected with HTTP 400) so offset paging is stable. - Raise the page ceiling to 40 and warn when it is hit. - Tests: short pages with exceededTransferLimit keep paging, orderByFields is sent, and a service that ignores resultOffset stops at the ceiling. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
c09865e582
commit
c174742e2c
9 files changed
+80
-21
No files matched your search
@@ -146,8 +146,9 @@ const LIVE_PACKS = [
|
||||
load: loadLithuaniaSources,
|
||||
},
|
||||
{
|
||||
// FL511's terms limit its content to individual, non-commercial use, so
|
||||
// the pack is off unless the operator opts in (see DATA_SOURCES.md).
|
||||
// FL511's terms limit its content to individual, personal use (other use
|
||||
// needs FDOT's written consent), so the pack is off unless the operator
|
||||
// opts in (see DATA_SOURCES.md).
|
||||
name: 'florida',
|
||||
enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
|
||||
load: loadFloridaSourcesFromOpenData,
|
||||
|
||||
@@ -122,6 +122,8 @@ export const FLORIDA_IMAGE_HOSTS = [
|
||||
export const DEFAULT_FLORIDA_MAX_SOURCES = 500;
|
||||
/** Byte cap per catalog page (2,000 rows, about 650 KiB measured). */
|
||||
export const FLORIDA_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
|
||||
/** Catalog page ceiling: 40 pages covers 4,000+ cameras even at 100 rows a page. */
|
||||
export const FLORIDA_MAX_CATALOG_PAGES = 40;
|
||||
export const FLORIDA_ANCHORS = [
|
||||
{ lat: 25.7617, lon: -80.1918 }, // Miami
|
||||
{ lat: 28.5384, lon: -81.3789 }, // Orlando
|
||||
|
||||
@@ -60,6 +60,7 @@ import {
|
||||
DEFAULT_FLORIDA_MAX_SOURCES,
|
||||
FLORIDA_ANCHORS,
|
||||
FLORIDA_MAX_CATALOG_BYTES,
|
||||
FLORIDA_MAX_CATALOG_PAGES,
|
||||
CATALONIA_CAMERAS_URL,
|
||||
CATALONIA_MAX_CATALOG_BYTES,
|
||||
CATALONIA_IMAGE_HOSTS,
|
||||
@@ -2883,14 +2884,18 @@ export function floridaCameraToSource(feature) {
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'fl511-open-data',
|
||||
license: 'FL511 (FDOT), individual non-commercial use only',
|
||||
license:
|
||||
'FL511 (FDOT), individual personal use only; other use needs FDOT written consent',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch FL511 (Florida DOT) cameras, keyless: an ArcGIS FeatureServer paged
|
||||
* 2,000 rows at a time. Frames are stills on the Divas Cloud hosts. A redirect
|
||||
* on any page drops the pack; any other failed page keeps the rows already read.
|
||||
* up to 2,000 rows at a time, ordered by the layer's object id so offsets are
|
||||
* stable. Paging follows `exceededTransferLimit` and advances by the rows each
|
||||
* page returned, so a lower server-side max record count still reads the whole
|
||||
* layer. Frames are stills on the Divas Cloud hosts. A redirect on any page
|
||||
* drops the pack; any other failed page keeps the rows already read.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
@@ -2899,9 +2904,17 @@ export async function loadFloridaSourcesFromOpenData() {
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
const pageSize = 2000;
|
||||
let offset = 0;
|
||||
// Page ceiling, in case the service ignores resultOffset.
|
||||
for (let offset = 0, page = 0; page < 10; offset += pageSize, page += 1) {
|
||||
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
|
||||
for (let page = 0; ; page += 1) {
|
||||
if (page >= FLORIDA_MAX_CATALOG_PAGES) {
|
||||
console.warn(
|
||||
`[CCTV] Florida catalog stopped at the ${FLORIDA_MAX_CATALOG_PAGES}-page ceiling; the catalog may be incomplete`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
// OBJECTID_1 is this layer's object id field (a plain OBJECTID is a 400).
|
||||
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&orderByFields=OBJECTID_1&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
|
||||
let resp;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
@@ -2953,9 +2966,10 @@ export async function loadFloridaSourcesFromOpenData() {
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
// ArcGIS sets exceededTransferLimit while more rows remain.
|
||||
if (feats.length < pageSize || body?.exceededTransferLimit !== true)
|
||||
break;
|
||||
// ArcGIS sets exceededTransferLimit while more rows remain. A page may
|
||||
// be shorter than pageSize when the layer's max record count is lower.
|
||||
if (body?.exceededTransferLimit !== true || feats.length === 0) break;
|
||||
offset += feats.length;
|
||||
}
|
||||
|
||||
const maxRaw = Number(
|
||||
|
||||
Reference in new issue
Block a user