fix(cctv): FL511 written-consent wording and robust catalog paging

- State FL511's terms the same way everywhere: individual, personal use
  only, and any other use (a public deployment included) needs FDOT's
  express written consent. .env.example, README, dataCredits, the source
  licence string, the catalog comment and the changelog fragment said only
  "individual, non-commercial use".
- Page the ArcGIS catalog on exceededTransferLimit alone and advance the
  offset by the rows each page returned, so a lower server-side max
  record count no longer cuts the catalog short at the first short page.
- Order pages by OBJECTID_1 (this layer's object id; a plain OBJECTID is
  rejected with HTTP 400) so offset paging is stable.
- Raise the page ceiling to 40 and warn when it is hit.
- Tests: short pages with exceededTransferLimit keep paging, orderByFields
  is sent, and a service that ignores resultOffset stops at the ceiling.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Said TurkseverandClaude Opus 5.5 committed 2026-10-19 14:32:58 +01:00
1 parent c09865e582
commit c174742e2c
9 files changed
+80 -21

No files matched your search

+3 -2
View File
@@ -146,8 +146,9 @@ const LIVE_PACKS = [
load: loadLithuaniaSources,
},
{
// FL511's terms limit its content to individual, non-commercial use, so
// the pack is off unless the operator opts in (see DATA_SOURCES.md).
// FL511's terms limit its content to individual, personal use (other use
// needs FDOT's written consent), so the pack is off unless the operator
// opts in (see DATA_SOURCES.md).
name: 'florida',
enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
load: loadFloridaSourcesFromOpenData,
+2
View File
@@ -122,6 +122,8 @@ export const FLORIDA_IMAGE_HOSTS = [
export const DEFAULT_FLORIDA_MAX_SOURCES = 500;
/** Byte cap per catalog page (2,000 rows, about 650 KiB measured). */
export const FLORIDA_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Catalog page ceiling: 40 pages covers 4,000+ cameras even at 100 rows a page. */
export const FLORIDA_MAX_CATALOG_PAGES = 40;
export const FLORIDA_ANCHORS = [
{ lat: 25.7617, lon: -80.1918 }, // Miami
{ lat: 28.5384, lon: -81.3789 }, // Orlando
+22 -8
View File
@@ -60,6 +60,7 @@ import {
DEFAULT_FLORIDA_MAX_SOURCES,
FLORIDA_ANCHORS,
FLORIDA_MAX_CATALOG_BYTES,
FLORIDA_MAX_CATALOG_PAGES,
CATALONIA_CAMERAS_URL,
CATALONIA_MAX_CATALOG_BYTES,
CATALONIA_IMAGE_HOSTS,
@@ -2883,14 +2884,18 @@ export function floridaCameraToSource(feature) {
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'fl511-open-data',
license: 'FL511 (FDOT), individual non-commercial use only',
license:
'FL511 (FDOT), individual personal use only; other use needs FDOT written consent',
};
}
/**
* Fetch FL511 (Florida DOT) cameras, keyless: an ArcGIS FeatureServer paged
* 2,000 rows at a time. Frames are stills on the Divas Cloud hosts. A redirect
* on any page drops the pack; any other failed page keeps the rows already read.
* up to 2,000 rows at a time, ordered by the layer's object id so offsets are
* stable. Paging follows `exceededTransferLimit` and advances by the rows each
* page returned, so a lower server-side max record count still reads the whole
* layer. Frames are stills on the Divas Cloud hosts. A redirect on any page
* drops the pack; any other failed page keeps the rows already read.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
@@ -2899,9 +2904,17 @@ export async function loadFloridaSourcesFromOpenData() {
const cameras = [];
const seen = new Set();
const pageSize = 2000;
let offset = 0;
// Page ceiling, in case the service ignores resultOffset.
for (let offset = 0, page = 0; page < 10; offset += pageSize, page += 1) {
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
for (let page = 0; ; page += 1) {
if (page >= FLORIDA_MAX_CATALOG_PAGES) {
console.warn(
`[CCTV] Florida catalog stopped at the ${FLORIDA_MAX_CATALOG_PAGES}-page ceiling; the catalog may be incomplete`,
);
break;
}
// OBJECTID_1 is this layer's object id field (a plain OBJECTID is a 400).
const url = `${FLORIDA_CCTV_URL}?where=1%3D1&outFields=*&orderByFields=OBJECTID_1&f=json&resultRecordCount=${pageSize}&resultOffset=${offset}`;
let resp;
try {
resp = await fetch(url, {
@@ -2953,9 +2966,10 @@ export async function loadFloridaSourcesFromOpenData() {
seen.add(camera.id);
cameras.push(camera);
}
// ArcGIS sets exceededTransferLimit while more rows remain.
if (feats.length < pageSize || body?.exceededTransferLimit !== true)
break;
// ArcGIS sets exceededTransferLimit while more rows remain. A page may
// be shorter than pageSize when the layer's max record count is lower.
if (body?.exceededTransferLimit !== true || feats.length === 0) break;
offset += feats.length;
}
const maxRaw = Number(