fix(cctv): FL511 written-consent wording and robust catalog paging

- State FL511's terms the same way everywhere: individual, personal use
  only, and any other use (a public deployment included) needs FDOT's
  express written consent. .env.example, README, dataCredits, the source
  licence string, the catalog comment and the changelog fragment said only
  "individual, non-commercial use".
- Page the ArcGIS catalog on exceededTransferLimit alone and advance the
  offset by the rows each page returned, so a lower server-side max
  record count no longer cuts the catalog short at the first short page.
- Order pages by OBJECTID_1 (this layer's object id; a plain OBJECTID is
  rejected with HTTP 400) so offset paging is stable.
- Raise the page ceiling to 40 and warn when it is hit.
- Tests: short pages with exceededTransferLimit keep paging, orderByFields
  is sent, and a service that ignores resultOffset stops at the ceiling.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Said TurkseverandClaude Opus 5.5 committed 2026-10-19 14:32:58 +01:00
1 parent c09865e582
commit c174742e2c
9 files changed
+80 -21

No files matched your search

+22 -8
View File
@@ -60,6 +60,7 @@ import {
DEFAULT_FLORIDA_MAX_SOURCES,
FLORIDA_ANCHORS,
FLORIDA_MAX_CATALOG_BYTES,
FLORIDA_MAX_CATALOG_PAGES,
CATALONIA_CAMERAS_URL,
CATALONIA_MAX_CATALOG_BYTES,
CATALONIA_IMAGE_HOSTS,
@@ -2883,14 +2884,18 @@ export function floridaCameraToSource(feature) {
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'fl511-open-data',
license: 'FL511 (FDOT), individual non-commercial use only',
license:
'FL511 (FDOT), individual personal use only; other use needs FDOT written consent',
};
}
/**
* Fetch FL511 (Florida DOT) cameras, keyless: an ArcGIS FeatureServer paged
* 2,000 rows at a time. Frames are stills on the Divas Cloud hosts. A redirect
* on any page drops the pack; any other failed page keeps the rows already read.
* up to 2,000 rows at a time, ordered by the layer's object id so offsets are
* stable. Paging follows `exceededTransferLimit` and advances by the rows each
* page returned, so a lower server-side max record count still reads the whole
* layer. Frames are stills on the Divas Cloud hosts. A redirect on any page
* drops the pack; any other failed page keeps the rows already read.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
@@ -2899,9 +2904,17 @@ export async function loadFloridaSourcesFromOpenData() {
const cameras = [];
const seen = new Set();
const pageSize = 2000;
let offset = 0;
// Page ceiling, in case the service ignores resultOffset.
for (let offset = 0, page = 0; page < 10; offset += pageSize, page += 1) {
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
for (let page = 0; ; page += 1) {
if (page >= FLORIDA_MAX_CATALOG_PAGES) {
console.warn(
`[CCTV] Florida catalog stopped at the ${FLORIDA_MAX_CATALOG_PAGES}-page ceiling; the catalog may be incomplete`,
);
break;
}
// OBJECTID_1 is this layer's object id field (a plain OBJECTID is a 400).
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&orderByFields=OBJECTID_1&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
let resp;
try {
resp = await fetch(url, {
@@ -2953,9 +2966,10 @@ export async function loadFloridaSourcesFromOpenData() {
seen.add(camera.id);
cameras.push(camera);
}
// ArcGIS sets exceededTransferLimit while more rows remain.
if (feats.length < pageSize || body?.exceededTransferLimit !== true)
break;
// ArcGIS sets exceededTransferLimit while more rows remain. A page may
// be shorter than pageSize when the layer's max record count is lower.
if (body?.exceededTransferLimit !== true || feats.length === 0) break;
offset += feats.length;
}
const maxRaw = Number(