mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
fix(cctv): FL511 written-consent wording and robust catalog paging
- State FL511's terms the same way everywhere: individual, personal use only, and any other use (a public deployment included) needs FDOT's express written consent. .env.example, README, dataCredits, the source licence string, the catalog comment and the changelog fragment said only "individual, non-commercial use". - Page the ArcGIS catalog on exceededTransferLimit alone and advance the offset by the rows each page returned, so a lower server-side max record count no longer cuts the catalog short at the first short page. - Order pages by OBJECTID_1 (this layer's object id; a plain OBJECTID is rejected with HTTP 400) so offset paging is stable. - Raise the page ceiling to 40 and warn when it is hit. - Tests: short pages with exceededTransferLimit keep paging, orderByFields is sent, and a service that ignores resultOffset stops at the ceiling. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
c09865e582
commit
c174742e2c
9 files changed
+80
-21
No files matched your search
@@ -60,6 +60,7 @@ import {
|
||||
DEFAULT_FLORIDA_MAX_SOURCES,
|
||||
FLORIDA_ANCHORS,
|
||||
FLORIDA_MAX_CATALOG_BYTES,
|
||||
FLORIDA_MAX_CATALOG_PAGES,
|
||||
CATALONIA_CAMERAS_URL,
|
||||
CATALONIA_MAX_CATALOG_BYTES,
|
||||
CATALONIA_IMAGE_HOSTS,
|
||||
@@ -2883,14 +2884,18 @@ export function floridaCameraToSource(feature) {
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'fl511-open-data',
|
||||
license: 'FL511 (FDOT), individual non-commercial use only',
|
||||
license:
|
||||
'FL511 (FDOT), individual personal use only; other use needs FDOT written consent',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch FL511 (Florida DOT) cameras, keyless: an ArcGIS FeatureServer paged
|
||||
* 2,000 rows at a time. Frames are stills on the Divas Cloud hosts. A redirect
|
||||
* on any page drops the pack; any other failed page keeps the rows already read.
|
||||
* up to 2,000 rows at a time, ordered by the layer's object id so offsets are
|
||||
* stable. Paging follows `exceededTransferLimit` and advances by the rows each
|
||||
* page returned, so a lower server-side max record count still reads the whole
|
||||
* layer. Frames are stills on the Divas Cloud hosts. A redirect on any page
|
||||
* drops the pack; any other failed page keeps the rows already read.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
@@ -2899,9 +2904,17 @@ export async function loadFloridaSourcesFromOpenData() {
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
const pageSize = 2000;
|
||||
let offset = 0;
|
||||
// Page ceiling, in case the service ignores resultOffset.
|
||||
for (let offset = 0, page = 0; page < 10; offset += pageSize, page += 1) {
|
||||
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
|
||||
for (let page = 0; ; page += 1) {
|
||||
if (page >= FLORIDA_MAX_CATALOG_PAGES) {
|
||||
console.warn(
|
||||
`[CCTV] Florida catalog stopped at the ${FLORIDA_MAX_CATALOG_PAGES}-page ceiling; the catalog may be incomplete`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
// OBJECTID_1 is this layer's object id field (a plain OBJECTID is a 400).
|
||||
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&orderByFields=OBJECTID_1&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
|
||||
let resp;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
@@ -2953,9 +2966,10 @@ export async function loadFloridaSourcesFromOpenData() {
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
// ArcGIS sets exceededTransferLimit while more rows remain.
|
||||
if (feats.length < pageSize || body?.exceededTransferLimit !== true)
|
||||
break;
|
||||
// ArcGIS sets exceededTransferLimit while more rows remain. A page may
|
||||
// be shorter than pageSize when the layer's max record count is lower.
|
||||
if (body?.exceededTransferLimit !== true || feats.length === 0) break;
|
||||
offset += feats.length;
|
||||
}
|
||||
|
||||
const maxRaw = Number(
|
||||
|
||||
Reference in new issue
Block a user