mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
Recent community wildlife observations, as a new Wildlife group in the layer panel. - Server: keyless iNaturalist proxy (v2 observations with only the projected fields; undocumented v1 UTFGrid density tiles, decoded on the server) sharing one paced queue (burst 4, ~1/s, 60/min, 8,000/day budget, up to four in flight), in-memory caches and upstream-only cooldowns; Acartia Salish Sea marine-mammal feed rounded to 0.01 degrees with comments dropped. - Layer: grid counts drawn as a soft view-ranked density glow; real sampled sightings as taxon badges at their true positions with ambient photo thumbnails and labels; obscured records stay privacy cells; a clicked sighting stays pinned through camera moves and refreshes. - Anchored photo card per selection with the single iNaturalist link; attribution in the layer row and the Data attribution popover. - Filters (CONFIRMED, taxa presets) round-trip in share links with token 9. Co-Authored-By: Claude Opus 5.5 <[email protected]>
371 lines
13 KiB
JavaScript
371 lines
13 KiB
JavaScript
import path from 'node:path';
|
|
import { promises as fs } from 'node:fs';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { normalizeInatObservations } from '../../../src/layers/wildlifeSightings/records.js';
|
|
import { readResponseJsonCapped } from '../common/http.js';
|
|
import { clampInt } from '../common/query.js';
|
|
import { providerUserAgent } from '../common/userAgent.js';
|
|
import { projectInatGrid, validInatGridTile } from './grid.js';
|
|
import { createInatQueue } from './queue.js';
|
|
|
|
const DAILY_BUDGET = 8000;
|
|
const DAY_MS = 24 * 60 * 60_000;
|
|
// One page is capped independently of upstream Content-Length.
|
|
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
|
const TIMEOUT_MS = 20_000;
|
|
const API_URL = 'https://api.inaturalist.org/v1';
|
|
// Request only the fields used by the public observation projection.
|
|
const OBSERVATIONS_URL = 'https://api.inaturalist.org/v2/observations';
|
|
const OBSERVATION_FIELDS =
|
|
'(id:!t,uri:!t,location:!t,geojson:all,geoprivacy:!t,taxon_geoprivacy:!t,' +
|
|
'obscured:!t,quality_grade:!t,observed_on:!t,' +
|
|
'time_observed_at:!t,faves_count:!t,' +
|
|
'num_identification_agreements:!t,' +
|
|
'photos:(url:!t,attribution:!t,license_code:!t),' +
|
|
'taxon:(name:!t,preferred_common_name:!t,iconic_taxon_name:!t,' +
|
|
'threatened:!t))';
|
|
const COOLDOWN_MS = 60_000;
|
|
const FAILURE_COOLDOWN_MS = 15_000;
|
|
|
|
/** Seconds and HTTP dates are accepted; a missing value earns one minute. */
|
|
export function inatRetryCooldownMs(raw, nowMs) {
|
|
const value = typeof raw === 'string' ? raw.trim() : '';
|
|
if (value) {
|
|
const seconds = /^\d+(?:\.\d+)?$/.test(value) ? Number(value) : NaN;
|
|
const deadline =
|
|
!Number.isFinite(seconds) && /[a-z]/i.test(value)
|
|
? Date.parse(value)
|
|
: NaN;
|
|
const delay = Number.isFinite(seconds) ? seconds * 1000 : deadline - nowMs;
|
|
if (Number.isFinite(delay) && delay >= 0) return delay;
|
|
}
|
|
return COOLDOWN_MS;
|
|
}
|
|
|
|
function upstreamError(code, status, retryAfterS, upstreamStatus) {
|
|
return Object.assign(new Error(code), {
|
|
code,
|
|
status,
|
|
retryAfterS,
|
|
upstreamStatus,
|
|
});
|
|
}
|
|
|
|
function observationFilters(params) {
|
|
const query = new URLSearchParams({
|
|
verifiable: 'true',
|
|
captive: 'false',
|
|
// d1 is an observation DATE, unlike creation-time created_d1.
|
|
d1: params.observedD1,
|
|
});
|
|
if (params.taxa !== 'whales') query.set('photos', 'true');
|
|
if (params.quality === 'research') query.set('quality_grade', 'research');
|
|
if (params.taxa === 'whales') query.set('taxon_id', '152871');
|
|
else if (['Aves', 'Mammalia', 'Insecta', 'Plantae'].includes(params.taxa))
|
|
query.set('iconic_taxa', params.taxa);
|
|
return query;
|
|
}
|
|
|
|
/**
|
|
* Shared iNaturalist governor. Only request counters are persisted; observation
|
|
* payloads are projected immediately and are never written to disk.
|
|
*/
|
|
export function createInatClient({
|
|
fetchImpl = (...args) => globalThis.fetch(...args),
|
|
now = () => Date.now(),
|
|
cacheDir = path.join(process.cwd(), '.gev-cache', 'inaturalist'),
|
|
warn = (...args) => console.warn(...args),
|
|
queueOptions,
|
|
} = {}) {
|
|
const enqueue = createInatQueue(queueOptions);
|
|
let budget = null;
|
|
let budgetLoaded = false;
|
|
let budgetTail = Promise.resolve();
|
|
let cooldownUntil = 0;
|
|
// An unavailable UTFGrid tile must never place observation queries into a
|
|
// 502 cooldown. Only an upstream 429 is a provider-wide restriction.
|
|
const failures = new Map();
|
|
|
|
function dailyLimit() {
|
|
return clampInt(
|
|
process.env.INATURALIST_DAILY_BUDGET,
|
|
1,
|
|
Number.MAX_SAFE_INTEGER,
|
|
DAILY_BUDGET,
|
|
);
|
|
}
|
|
|
|
function day() {
|
|
return new Date(now()).toISOString().slice(0, 10);
|
|
}
|
|
|
|
function currentBudget() {
|
|
const date = day();
|
|
if (budget?.date !== date) budget = { date, count: 0 };
|
|
return budget;
|
|
}
|
|
|
|
async function readBudgetOnce() {
|
|
if (budgetLoaded) return;
|
|
budgetLoaded = true;
|
|
if (cacheDir === null) return;
|
|
try {
|
|
const value = JSON.parse(
|
|
await fs.readFile(path.join(cacheDir, 'budget.json'), 'utf8'),
|
|
);
|
|
if (
|
|
typeof value?.date === 'string' &&
|
|
/^\d{4}-\d{2}-\d{2}$/.test(value.date) &&
|
|
Number.isSafeInteger(value.count) &&
|
|
value.count >= 0
|
|
)
|
|
budget = { date: value.date, count: value.count };
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT')
|
|
warn('[inaturalist-proxy] budget read failed');
|
|
}
|
|
}
|
|
|
|
async function persistBudget(value) {
|
|
if (cacheDir === null) return;
|
|
const destination = path.join(cacheDir, 'budget.json');
|
|
const temporary = `${destination}.${randomUUID()}.tmp`;
|
|
try {
|
|
await fs.mkdir(cacheDir, { recursive: true });
|
|
await fs.writeFile(temporary, JSON.stringify(value), 'utf8');
|
|
await fs.rename(temporary, destination);
|
|
} catch {
|
|
await fs.rm(temporary, { force: true }).catch(() => {});
|
|
warn('[inaturalist-proxy] budget write failed');
|
|
}
|
|
}
|
|
|
|
function retrySeconds(until = cooldownUntil) {
|
|
return Math.max(0, Math.ceil((until - now()) / 1000));
|
|
}
|
|
|
|
function checkCooldown(resource) {
|
|
if (now() < cooldownUntil)
|
|
throw upstreamError('inaturalist_rate_limited', 429, retrySeconds(), 429);
|
|
const failure = failures.get(resource);
|
|
if (failure && now() < failure.until)
|
|
throw upstreamError(
|
|
'inaturalist_unavailable',
|
|
502,
|
|
retrySeconds(failure.until),
|
|
failure.status,
|
|
);
|
|
failures.delete(resource);
|
|
}
|
|
|
|
// Reserve and atomically persist one attempt before issuing it. A recovered
|
|
// tail prevents one disk/reservation failure from poisoning future calls.
|
|
function reserveAttempt(signal, resource) {
|
|
const result = budgetTail.then(async () => {
|
|
signal?.throwIfAborted();
|
|
checkCooldown(resource);
|
|
await readBudgetOnce();
|
|
signal?.throwIfAborted();
|
|
checkCooldown(resource);
|
|
const value = currentBudget();
|
|
if (value.count >= dailyLimit()) {
|
|
const retry = Math.max(
|
|
1,
|
|
Math.ceil((Date.parse(`${day()}T00:00:00Z`) + DAY_MS - now()) / 1000),
|
|
);
|
|
throw upstreamError('inaturalist_budget_exhausted', 503, retry);
|
|
}
|
|
value.count += 1;
|
|
await persistBudget({ ...value });
|
|
});
|
|
budgetTail = result.catch(() => {});
|
|
return result;
|
|
}
|
|
|
|
async function requestProjected(url, project, resource, { signal } = {}) {
|
|
signal?.throwIfAborted();
|
|
checkCooldown(resource);
|
|
return enqueue(() => dispatch(url, project, resource, signal), {
|
|
signal,
|
|
priority: resource === 'observations' ? 0 : 1,
|
|
});
|
|
}
|
|
|
|
async function dispatch(url, project, resource, signal) {
|
|
signal?.throwIfAborted();
|
|
await reserveAttempt(signal, resource);
|
|
signal?.throwIfAborted();
|
|
checkCooldown(resource);
|
|
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
|
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
let upstreamStatus;
|
|
let retryDelay = FAILURE_COOLDOWN_MS;
|
|
try {
|
|
const response = await fetchImpl(url, {
|
|
signal: requestSignal,
|
|
redirect: 'error',
|
|
headers: {
|
|
Accept: 'application/json',
|
|
'User-Agent': providerUserAgent('wildlife-sightings'),
|
|
},
|
|
});
|
|
upstreamStatus = response.status;
|
|
if (!response.ok) {
|
|
await response.body?.cancel().catch(() => {});
|
|
if (response.status === 429) {
|
|
const delay = inatRetryCooldownMs(
|
|
response.headers.get('retry-after'),
|
|
now(),
|
|
);
|
|
cooldownUntil = Math.max(cooldownUntil, now() + delay);
|
|
throw upstreamError(
|
|
'inaturalist_rate_limited',
|
|
429,
|
|
retrySeconds(),
|
|
429,
|
|
);
|
|
}
|
|
if (response.headers.has('retry-after'))
|
|
retryDelay = inatRetryCooldownMs(
|
|
response.headers.get('retry-after'),
|
|
now(),
|
|
);
|
|
throw upstreamError('inaturalist_unavailable', 502);
|
|
}
|
|
const payload = await readResponseJsonCapped(
|
|
response,
|
|
MAX_RESPONSE_BYTES,
|
|
requestSignal,
|
|
);
|
|
return project(payload);
|
|
} catch (error) {
|
|
// Log only safe diagnostic categories, never a URL, body, status text or
|
|
// exception message (each can contain request/private information).
|
|
const reason = signal?.aborted
|
|
? 'abort'
|
|
: timeout.aborted || error.name === 'TimeoutError'
|
|
? 'timeout'
|
|
: upstreamStatus && upstreamStatus >= 400
|
|
? 'http'
|
|
: error.code === 'RESPONSE_TOO_LARGE'
|
|
? 'response-too-large'
|
|
: upstreamStatus
|
|
? 'invalid-payload'
|
|
: 'network';
|
|
warn(
|
|
`[inaturalist-proxy] upstream refresh failed resource=${resource} status=${upstreamStatus ?? 'none'} reason=${reason}`,
|
|
);
|
|
if (signal?.aborted) throw error;
|
|
// Network, abort, schema and size failures are not upstream cooldown
|
|
// signals. A later healthy request may recover immediately.
|
|
if (upstreamStatus >= 500)
|
|
failures.set(resource, {
|
|
until: now() + retryDelay,
|
|
status: upstreamStatus,
|
|
});
|
|
const failure = failures.get(resource);
|
|
throw upstreamError(
|
|
error.status === 429
|
|
? 'inaturalist_rate_limited'
|
|
: 'inaturalist_unavailable',
|
|
error.status === 429 ? 429 : 502,
|
|
error.status === 429
|
|
? retrySeconds()
|
|
: failure
|
|
? retrySeconds(failure.until)
|
|
: undefined,
|
|
upstreamStatus === 429 || upstreamStatus >= 500
|
|
? upstreamStatus
|
|
: undefined,
|
|
);
|
|
}
|
|
}
|
|
|
|
/** Request one bounded observation sample and project its public fields. */
|
|
async function getObservations(params, options = {}) {
|
|
const query = observationFilters(params);
|
|
// View samples stay stable in the route cache until their TTL expires.
|
|
query.set('order_by', params.scope === 'view' ? 'random' : 'observed_on');
|
|
if (params.scope !== 'view') query.set('order', 'desc');
|
|
query.set('per_page', '200');
|
|
query.set('fields', OBSERVATION_FIELDS);
|
|
if (params.scope === 'view') {
|
|
query.set('swlat', String(params.box.south));
|
|
query.set('swlng', String(params.box.west));
|
|
query.set('nelat', String(params.box.north));
|
|
query.set('nelng', String(params.box.east));
|
|
}
|
|
return requestProjected(
|
|
`${OBSERVATIONS_URL}?${query}`,
|
|
(payload) => {
|
|
if (
|
|
!Array.isArray(payload?.results) ||
|
|
!Number.isSafeInteger(payload.total_results) ||
|
|
payload.total_results < 0
|
|
)
|
|
throw upstreamError('inaturalist_unavailable', 502);
|
|
const rows = normalizeInatObservations(payload.results, {
|
|
nowMs: now(),
|
|
windowMs: (params.taxa === 'whales' ? 7 : 2) * DAY_MS,
|
|
});
|
|
// Empty views need total_results: 0. A wholly invalid page must not
|
|
// replace good data. Valid rows excluded by the finer rolling-time
|
|
// cutoff are a healthy empty result of the upstream DATE over-fetch.
|
|
if (
|
|
!rows ||
|
|
(!rows.length &&
|
|
(payload.results.length || payload.total_results) &&
|
|
!normalizeInatObservations(payload.results)?.length)
|
|
)
|
|
throw upstreamError('inaturalist_unavailable', 502);
|
|
return {
|
|
rows,
|
|
totalResults: payload.total_results,
|
|
...(payload.total_results > payload.results.length
|
|
? { partial: true, rawRowCount: payload.total_results }
|
|
: {}),
|
|
};
|
|
},
|
|
'observations',
|
|
options,
|
|
);
|
|
}
|
|
|
|
/** Decode one validated density tile without observation identifiers. */
|
|
async function getGrid(params, options = {}) {
|
|
if (!validInatGridTile(params)) throw new TypeError('Invalid grid tile');
|
|
const query = observationFilters(params);
|
|
return requestProjected(
|
|
`${API_URL}/grid/${params.z}/${params.x}/${params.y}.grid.json?${query}`,
|
|
(payload) => projectInatGrid(payload, params),
|
|
'grid',
|
|
options,
|
|
);
|
|
}
|
|
|
|
return {
|
|
getObservations,
|
|
getGrid,
|
|
/** Current provider-wide or resource-specific retry information. */
|
|
cooldownState(resource) {
|
|
if (now() < cooldownUntil)
|
|
return {
|
|
upstreamStatus: 429,
|
|
retryAfterS: retrySeconds(),
|
|
};
|
|
const failure = failures.get(resource);
|
|
if (failure && now() < failure.until)
|
|
return {
|
|
upstreamStatus: failure.status,
|
|
retryAfterS: retrySeconds(failure.until),
|
|
};
|
|
return undefined;
|
|
},
|
|
/** The current UTC day's reserved upstream attempts and configured limit. */
|
|
budgetState() {
|
|
const value = currentBudget();
|
|
return { ...value, budget: dailyLimit() };
|
|
},
|
|
};
|
|
}
|