Files
gods-eye-view/server/providers/inaturalist/client.js
T
Yehe LiuandClaude Opus 5.5 14f88b3e08 feat: Wildlife Sightings layer from iNaturalist and Acartia
Recent community wildlife observations, as a new Wildlife group in the
layer panel.

- Server: keyless iNaturalist proxy (v2 observations with only the
  projected fields; undocumented v1 UTFGrid density tiles, decoded on the
  server) sharing one paced queue (burst 4, ~1/s, 60/min, 8,000/day budget,
  up to four in flight), in-memory caches and upstream-only cooldowns;
  Acartia Salish Sea marine-mammal feed rounded to 0.01 degrees with
  comments dropped.
- Layer: grid counts drawn as a soft view-ranked density glow; real sampled
  sightings as taxon badges at their true positions with ambient photo
  thumbnails and labels; obscured records stay privacy cells; a clicked
  sighting stays pinned through camera moves and refreshes.
- Anchored photo card per selection with the single iNaturalist link;
  attribution in the layer row and the Data attribution popover.
- Filters (CONFIRMED, taxa presets) round-trip in share links with token 9.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-11-05 16:17:12 -05:00

371 lines
13 KiB
JavaScript

import path from 'node:path';
import { promises as fs } from 'node:fs';
import { randomUUID } from 'node:crypto';
import { normalizeInatObservations } from '../../../src/layers/wildlifeSightings/records.js';
import { readResponseJsonCapped } from '../common/http.js';
import { clampInt } from '../common/query.js';
import { providerUserAgent } from '../common/userAgent.js';
import { projectInatGrid, validInatGridTile } from './grid.js';
import { createInatQueue } from './queue.js';
const DAILY_BUDGET = 8000;
const DAY_MS = 24 * 60 * 60_000;
// One page is capped independently of upstream Content-Length.
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
const TIMEOUT_MS = 20_000;
const API_URL = 'https://api.inaturalist.org/v1';
// Request only the fields used by the public observation projection.
const OBSERVATIONS_URL = 'https://api.inaturalist.org/v2/observations';
const OBSERVATION_FIELDS =
'(id:!t,uri:!t,location:!t,geojson:all,geoprivacy:!t,taxon_geoprivacy:!t,' +
'obscured:!t,quality_grade:!t,observed_on:!t,' +
'time_observed_at:!t,faves_count:!t,' +
'num_identification_agreements:!t,' +
'photos:(url:!t,attribution:!t,license_code:!t),' +
'taxon:(name:!t,preferred_common_name:!t,iconic_taxon_name:!t,' +
'threatened:!t))';
const COOLDOWN_MS = 60_000;
const FAILURE_COOLDOWN_MS = 15_000;
/** Seconds and HTTP dates are accepted; a missing value earns one minute. */
export function inatRetryCooldownMs(raw, nowMs) {
const value = typeof raw === 'string' ? raw.trim() : '';
if (value) {
const seconds = /^\d+(?:\.\d+)?$/.test(value) ? Number(value) : NaN;
const deadline =
!Number.isFinite(seconds) && /[a-z]/i.test(value)
? Date.parse(value)
: NaN;
const delay = Number.isFinite(seconds) ? seconds * 1000 : deadline - nowMs;
if (Number.isFinite(delay) && delay >= 0) return delay;
}
return COOLDOWN_MS;
}
function upstreamError(code, status, retryAfterS, upstreamStatus) {
return Object.assign(new Error(code), {
code,
status,
retryAfterS,
upstreamStatus,
});
}
function observationFilters(params) {
const query = new URLSearchParams({
verifiable: 'true',
captive: 'false',
// d1 is an observation DATE, unlike creation-time created_d1.
d1: params.observedD1,
});
if (params.taxa !== 'whales') query.set('photos', 'true');
if (params.quality === 'research') query.set('quality_grade', 'research');
if (params.taxa === 'whales') query.set('taxon_id', '152871');
else if (['Aves', 'Mammalia', 'Insecta', 'Plantae'].includes(params.taxa))
query.set('iconic_taxa', params.taxa);
return query;
}
/**
* Shared iNaturalist governor. Only request counters are persisted; observation
* payloads are projected immediately and are never written to disk.
*/
export function createInatClient({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
cacheDir = path.join(process.cwd(), '.gev-cache', 'inaturalist'),
warn = (...args) => console.warn(...args),
queueOptions,
} = {}) {
const enqueue = createInatQueue(queueOptions);
let budget = null;
let budgetLoaded = false;
let budgetTail = Promise.resolve();
let cooldownUntil = 0;
// An unavailable UTFGrid tile must never place observation queries into a
// 502 cooldown. Only an upstream 429 is a provider-wide restriction.
const failures = new Map();
function dailyLimit() {
return clampInt(
process.env.INATURALIST_DAILY_BUDGET,
1,
Number.MAX_SAFE_INTEGER,
DAILY_BUDGET,
);
}
function day() {
return new Date(now()).toISOString().slice(0, 10);
}
function currentBudget() {
const date = day();
if (budget?.date !== date) budget = { date, count: 0 };
return budget;
}
async function readBudgetOnce() {
if (budgetLoaded) return;
budgetLoaded = true;
if (cacheDir === null) return;
try {
const value = JSON.parse(
await fs.readFile(path.join(cacheDir, 'budget.json'), 'utf8'),
);
if (
typeof value?.date === 'string' &&
/^\d{4}-\d{2}-\d{2}$/.test(value.date) &&
Number.isSafeInteger(value.count) &&
value.count >= 0
)
budget = { date: value.date, count: value.count };
} catch (error) {
if (error.code !== 'ENOENT')
warn('[inaturalist-proxy] budget read failed');
}
}
async function persistBudget(value) {
if (cacheDir === null) return;
const destination = path.join(cacheDir, 'budget.json');
const temporary = `${destination}.${randomUUID()}.tmp`;
try {
await fs.mkdir(cacheDir, { recursive: true });
await fs.writeFile(temporary, JSON.stringify(value), 'utf8');
await fs.rename(temporary, destination);
} catch {
await fs.rm(temporary, { force: true }).catch(() => {});
warn('[inaturalist-proxy] budget write failed');
}
}
function retrySeconds(until = cooldownUntil) {
return Math.max(0, Math.ceil((until - now()) / 1000));
}
function checkCooldown(resource) {
if (now() < cooldownUntil)
throw upstreamError('inaturalist_rate_limited', 429, retrySeconds(), 429);
const failure = failures.get(resource);
if (failure && now() < failure.until)
throw upstreamError(
'inaturalist_unavailable',
502,
retrySeconds(failure.until),
failure.status,
);
failures.delete(resource);
}
// Reserve and atomically persist one attempt before issuing it. A recovered
// tail prevents one disk/reservation failure from poisoning future calls.
function reserveAttempt(signal, resource) {
const result = budgetTail.then(async () => {
signal?.throwIfAborted();
checkCooldown(resource);
await readBudgetOnce();
signal?.throwIfAborted();
checkCooldown(resource);
const value = currentBudget();
if (value.count >= dailyLimit()) {
const retry = Math.max(
1,
Math.ceil((Date.parse(`${day()}T00:00:00Z`) + DAY_MS - now()) / 1000),
);
throw upstreamError('inaturalist_budget_exhausted', 503, retry);
}
value.count += 1;
await persistBudget({ ...value });
});
budgetTail = result.catch(() => {});
return result;
}
async function requestProjected(url, project, resource, { signal } = {}) {
signal?.throwIfAborted();
checkCooldown(resource);
return enqueue(() => dispatch(url, project, resource, signal), {
signal,
priority: resource === 'observations' ? 0 : 1,
});
}
async function dispatch(url, project, resource, signal) {
signal?.throwIfAborted();
await reserveAttempt(signal, resource);
signal?.throwIfAborted();
checkCooldown(resource);
const timeout = AbortSignal.timeout(TIMEOUT_MS);
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
let upstreamStatus;
let retryDelay = FAILURE_COOLDOWN_MS;
try {
const response = await fetchImpl(url, {
signal: requestSignal,
redirect: 'error',
headers: {
Accept: 'application/json',
'User-Agent': providerUserAgent('wildlife-sightings'),
},
});
upstreamStatus = response.status;
if (!response.ok) {
await response.body?.cancel().catch(() => {});
if (response.status === 429) {
const delay = inatRetryCooldownMs(
response.headers.get('retry-after'),
now(),
);
cooldownUntil = Math.max(cooldownUntil, now() + delay);
throw upstreamError(
'inaturalist_rate_limited',
429,
retrySeconds(),
429,
);
}
if (response.headers.has('retry-after'))
retryDelay = inatRetryCooldownMs(
response.headers.get('retry-after'),
now(),
);
throw upstreamError('inaturalist_unavailable', 502);
}
const payload = await readResponseJsonCapped(
response,
MAX_RESPONSE_BYTES,
requestSignal,
);
return project(payload);
} catch (error) {
// Log only safe diagnostic categories, never a URL, body, status text or
// exception message (each can contain request/private information).
const reason = signal?.aborted
? 'abort'
: timeout.aborted || error.name === 'TimeoutError'
? 'timeout'
: upstreamStatus && upstreamStatus >= 400
? 'http'
: error.code === 'RESPONSE_TOO_LARGE'
? 'response-too-large'
: upstreamStatus
? 'invalid-payload'
: 'network';
warn(
`[inaturalist-proxy] upstream refresh failed resource=${resource} status=${upstreamStatus ?? 'none'} reason=${reason}`,
);
if (signal?.aborted) throw error;
// Network, abort, schema and size failures are not upstream cooldown
// signals. A later healthy request may recover immediately.
if (upstreamStatus >= 500)
failures.set(resource, {
until: now() + retryDelay,
status: upstreamStatus,
});
const failure = failures.get(resource);
throw upstreamError(
error.status === 429
? 'inaturalist_rate_limited'
: 'inaturalist_unavailable',
error.status === 429 ? 429 : 502,
error.status === 429
? retrySeconds()
: failure
? retrySeconds(failure.until)
: undefined,
upstreamStatus === 429 || upstreamStatus >= 500
? upstreamStatus
: undefined,
);
}
}
/** Request one bounded observation sample and project its public fields. */
async function getObservations(params, options = {}) {
const query = observationFilters(params);
// View samples stay stable in the route cache until their TTL expires.
query.set('order_by', params.scope === 'view' ? 'random' : 'observed_on');
if (params.scope !== 'view') query.set('order', 'desc');
query.set('per_page', '200');
query.set('fields', OBSERVATION_FIELDS);
if (params.scope === 'view') {
query.set('swlat', String(params.box.south));
query.set('swlng', String(params.box.west));
query.set('nelat', String(params.box.north));
query.set('nelng', String(params.box.east));
}
return requestProjected(
`${OBSERVATIONS_URL}?${query}`,
(payload) => {
if (
!Array.isArray(payload?.results) ||
!Number.isSafeInteger(payload.total_results) ||
payload.total_results < 0
)
throw upstreamError('inaturalist_unavailable', 502);
const rows = normalizeInatObservations(payload.results, {
nowMs: now(),
windowMs: (params.taxa === 'whales' ? 7 : 2) * DAY_MS,
});
// Empty views need total_results: 0. A wholly invalid page must not
// replace good data. Valid rows excluded by the finer rolling-time
// cutoff are a healthy empty result of the upstream DATE over-fetch.
if (
!rows ||
(!rows.length &&
(payload.results.length || payload.total_results) &&
!normalizeInatObservations(payload.results)?.length)
)
throw upstreamError('inaturalist_unavailable', 502);
return {
rows,
totalResults: payload.total_results,
...(payload.total_results > payload.results.length
? { partial: true, rawRowCount: payload.total_results }
: {}),
};
},
'observations',
options,
);
}
/** Decode one validated density tile without observation identifiers. */
async function getGrid(params, options = {}) {
if (!validInatGridTile(params)) throw new TypeError('Invalid grid tile');
const query = observationFilters(params);
return requestProjected(
`${API_URL}/grid/${params.z}/${params.x}/${params.y}.grid.json?${query}`,
(payload) => projectInatGrid(payload, params),
'grid',
options,
);
}
return {
getObservations,
getGrid,
/** Current provider-wide or resource-specific retry information. */
cooldownState(resource) {
if (now() < cooldownUntil)
return {
upstreamStatus: 429,
retryAfterS: retrySeconds(),
};
const failure = failures.get(resource);
if (failure && now() < failure.until)
return {
upstreamStatus: failure.status,
retryAfterS: retrySeconds(failure.until),
};
return undefined;
},
/** The current UTC day's reserved upstream attempts and configured limit. */
budgetState() {
const value = currentBudget();
return { ...value, budget: dailyLimit() };
},
};
}