mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-17 08:42:21 +00:00
feat: Wildlife Sightings layer from iNaturalist and Acartia
Recent community wildlife observations, as a new Wildlife group in the layer panel. - Server: keyless iNaturalist proxy (v2 observations with only the projected fields; undocumented v1 UTFGrid density tiles, decoded on the server) sharing one paced queue (burst 4, ~1/s, 60/min, 8,000/day budget, up to four in flight), in-memory caches and upstream-only cooldowns; Acartia Salish Sea marine-mammal feed rounded to 0.01 degrees with comments dropped. - Layer: grid counts drawn as a soft view-ranked density glow; real sampled sightings as taxon badges at their true positions with ambient photo thumbnails and labels; obscured records stay privacy cells; a clicked sighting stays pinned through camera moves and refreshes. - Anchored photo card per selection with the single iNaturalist link; attribution in the layer row and the Data attribution popover. - Filters (CONFIRMED, taxa presets) round-trip in share links with token 9. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
edd6212793
commit
14f88b3e08
120 files changed
+21260
-285
No files matched your search
@@ -0,0 +1,370 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fs } from 'node:fs';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { normalizeInatObservations } from '../../../src/layers/wildlifeSightings/records.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import { clampInt } from '../common/query.js';
|
||||
import { providerUserAgent } from '../common/userAgent.js';
|
||||
import { projectInatGrid, validInatGridTile } from './grid.js';
|
||||
import { createInatQueue } from './queue.js';
|
||||
|
||||
const DAILY_BUDGET = 8000;
|
||||
const DAY_MS = 24 * 60 * 60_000;
|
||||
// One page is capped independently of upstream Content-Length.
|
||||
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 20_000;
|
||||
const API_URL = 'https://api.inaturalist.org/v1';
|
||||
// Request only the fields used by the public observation projection.
|
||||
const OBSERVATIONS_URL = 'https://api.inaturalist.org/v2/observations';
|
||||
const OBSERVATION_FIELDS =
|
||||
'(id:!t,uri:!t,location:!t,geojson:all,geoprivacy:!t,taxon_geoprivacy:!t,' +
|
||||
'obscured:!t,quality_grade:!t,observed_on:!t,' +
|
||||
'time_observed_at:!t,faves_count:!t,' +
|
||||
'num_identification_agreements:!t,' +
|
||||
'photos:(url:!t,attribution:!t,license_code:!t),' +
|
||||
'taxon:(name:!t,preferred_common_name:!t,iconic_taxon_name:!t,' +
|
||||
'threatened:!t))';
|
||||
const COOLDOWN_MS = 60_000;
|
||||
const FAILURE_COOLDOWN_MS = 15_000;
|
||||
|
||||
/** Seconds and HTTP dates are accepted; a missing value earns one minute. */
|
||||
export function inatRetryCooldownMs(raw, nowMs) {
|
||||
const value = typeof raw === 'string' ? raw.trim() : '';
|
||||
if (value) {
|
||||
const seconds = /^\d+(?:\.\d+)?$/.test(value) ? Number(value) : NaN;
|
||||
const deadline =
|
||||
!Number.isFinite(seconds) && /[a-z]/i.test(value)
|
||||
? Date.parse(value)
|
||||
: NaN;
|
||||
const delay = Number.isFinite(seconds) ? seconds * 1000 : deadline - nowMs;
|
||||
if (Number.isFinite(delay) && delay >= 0) return delay;
|
||||
}
|
||||
return COOLDOWN_MS;
|
||||
}
|
||||
|
||||
function upstreamError(code, status, retryAfterS, upstreamStatus) {
|
||||
return Object.assign(new Error(code), {
|
||||
code,
|
||||
status,
|
||||
retryAfterS,
|
||||
upstreamStatus,
|
||||
});
|
||||
}
|
||||
|
||||
function observationFilters(params) {
|
||||
const query = new URLSearchParams({
|
||||
verifiable: 'true',
|
||||
captive: 'false',
|
||||
// d1 is an observation DATE, unlike creation-time created_d1.
|
||||
d1: params.observedD1,
|
||||
});
|
||||
if (params.taxa !== 'whales') query.set('photos', 'true');
|
||||
if (params.quality === 'research') query.set('quality_grade', 'research');
|
||||
if (params.taxa === 'whales') query.set('taxon_id', '152871');
|
||||
else if (['Aves', 'Mammalia', 'Insecta', 'Plantae'].includes(params.taxa))
|
||||
query.set('iconic_taxa', params.taxa);
|
||||
return query;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared iNaturalist governor. Only request counters are persisted; observation
|
||||
* payloads are projected immediately and are never written to disk.
|
||||
*/
|
||||
export function createInatClient({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
cacheDir = path.join(process.cwd(), '.gev-cache', 'inaturalist'),
|
||||
warn = (...args) => console.warn(...args),
|
||||
queueOptions,
|
||||
} = {}) {
|
||||
const enqueue = createInatQueue(queueOptions);
|
||||
let budget = null;
|
||||
let budgetLoaded = false;
|
||||
let budgetTail = Promise.resolve();
|
||||
let cooldownUntil = 0;
|
||||
// An unavailable UTFGrid tile must never place observation queries into a
|
||||
// 502 cooldown. Only an upstream 429 is a provider-wide restriction.
|
||||
const failures = new Map();
|
||||
|
||||
function dailyLimit() {
|
||||
return clampInt(
|
||||
process.env.INATURALIST_DAILY_BUDGET,
|
||||
1,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
DAILY_BUDGET,
|
||||
);
|
||||
}
|
||||
|
||||
function day() {
|
||||
return new Date(now()).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function currentBudget() {
|
||||
const date = day();
|
||||
if (budget?.date !== date) budget = { date, count: 0 };
|
||||
return budget;
|
||||
}
|
||||
|
||||
async function readBudgetOnce() {
|
||||
if (budgetLoaded) return;
|
||||
budgetLoaded = true;
|
||||
if (cacheDir === null) return;
|
||||
try {
|
||||
const value = JSON.parse(
|
||||
await fs.readFile(path.join(cacheDir, 'budget.json'), 'utf8'),
|
||||
);
|
||||
if (
|
||||
typeof value?.date === 'string' &&
|
||||
/^\d{4}-\d{2}-\d{2}$/.test(value.date) &&
|
||||
Number.isSafeInteger(value.count) &&
|
||||
value.count >= 0
|
||||
)
|
||||
budget = { date: value.date, count: value.count };
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
warn('[inaturalist-proxy] budget read failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function persistBudget(value) {
|
||||
if (cacheDir === null) return;
|
||||
const destination = path.join(cacheDir, 'budget.json');
|
||||
const temporary = `${destination}.${randomUUID()}.tmp`;
|
||||
try {
|
||||
await fs.mkdir(cacheDir, { recursive: true });
|
||||
await fs.writeFile(temporary, JSON.stringify(value), 'utf8');
|
||||
await fs.rename(temporary, destination);
|
||||
} catch {
|
||||
await fs.rm(temporary, { force: true }).catch(() => {});
|
||||
warn('[inaturalist-proxy] budget write failed');
|
||||
}
|
||||
}
|
||||
|
||||
function retrySeconds(until = cooldownUntil) {
|
||||
return Math.max(0, Math.ceil((until - now()) / 1000));
|
||||
}
|
||||
|
||||
function checkCooldown(resource) {
|
||||
if (now() < cooldownUntil)
|
||||
throw upstreamError('inaturalist_rate_limited', 429, retrySeconds(), 429);
|
||||
const failure = failures.get(resource);
|
||||
if (failure && now() < failure.until)
|
||||
throw upstreamError(
|
||||
'inaturalist_unavailable',
|
||||
502,
|
||||
retrySeconds(failure.until),
|
||||
failure.status,
|
||||
);
|
||||
failures.delete(resource);
|
||||
}
|
||||
|
||||
// Reserve and atomically persist one attempt before issuing it. A recovered
|
||||
// tail prevents one disk/reservation failure from poisoning future calls.
|
||||
function reserveAttempt(signal, resource) {
|
||||
const result = budgetTail.then(async () => {
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
await readBudgetOnce();
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
const value = currentBudget();
|
||||
if (value.count >= dailyLimit()) {
|
||||
const retry = Math.max(
|
||||
1,
|
||||
Math.ceil((Date.parse(`${day()}T00:00:00Z`) + DAY_MS - now()) / 1000),
|
||||
);
|
||||
throw upstreamError('inaturalist_budget_exhausted', 503, retry);
|
||||
}
|
||||
value.count += 1;
|
||||
await persistBudget({ ...value });
|
||||
});
|
||||
budgetTail = result.catch(() => {});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function requestProjected(url, project, resource, { signal } = {}) {
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
return enqueue(() => dispatch(url, project, resource, signal), {
|
||||
signal,
|
||||
priority: resource === 'observations' ? 0 : 1,
|
||||
});
|
||||
}
|
||||
|
||||
async function dispatch(url, project, resource, signal) {
|
||||
signal?.throwIfAborted();
|
||||
await reserveAttempt(signal, resource);
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||
let upstreamStatus;
|
||||
let retryDelay = FAILURE_COOLDOWN_MS;
|
||||
try {
|
||||
const response = await fetchImpl(url, {
|
||||
signal: requestSignal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': providerUserAgent('wildlife-sightings'),
|
||||
},
|
||||
});
|
||||
upstreamStatus = response.status;
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (response.status === 429) {
|
||||
const delay = inatRetryCooldownMs(
|
||||
response.headers.get('retry-after'),
|
||||
now(),
|
||||
);
|
||||
cooldownUntil = Math.max(cooldownUntil, now() + delay);
|
||||
throw upstreamError(
|
||||
'inaturalist_rate_limited',
|
||||
429,
|
||||
retrySeconds(),
|
||||
429,
|
||||
);
|
||||
}
|
||||
if (response.headers.has('retry-after'))
|
||||
retryDelay = inatRetryCooldownMs(
|
||||
response.headers.get('retry-after'),
|
||||
now(),
|
||||
);
|
||||
throw upstreamError('inaturalist_unavailable', 502);
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
requestSignal,
|
||||
);
|
||||
return project(payload);
|
||||
} catch (error) {
|
||||
// Log only safe diagnostic categories, never a URL, body, status text or
|
||||
// exception message (each can contain request/private information).
|
||||
const reason = signal?.aborted
|
||||
? 'abort'
|
||||
: timeout.aborted || error.name === 'TimeoutError'
|
||||
? 'timeout'
|
||||
: upstreamStatus && upstreamStatus >= 400
|
||||
? 'http'
|
||||
: error.code === 'RESPONSE_TOO_LARGE'
|
||||
? 'response-too-large'
|
||||
: upstreamStatus
|
||||
? 'invalid-payload'
|
||||
: 'network';
|
||||
warn(
|
||||
`[inaturalist-proxy] upstream refresh failed resource=${resource} status=${upstreamStatus ?? 'none'} reason=${reason}`,
|
||||
);
|
||||
if (signal?.aborted) throw error;
|
||||
// Network, abort, schema and size failures are not upstream cooldown
|
||||
// signals. A later healthy request may recover immediately.
|
||||
if (upstreamStatus >= 500)
|
||||
failures.set(resource, {
|
||||
until: now() + retryDelay,
|
||||
status: upstreamStatus,
|
||||
});
|
||||
const failure = failures.get(resource);
|
||||
throw upstreamError(
|
||||
error.status === 429
|
||||
? 'inaturalist_rate_limited'
|
||||
: 'inaturalist_unavailable',
|
||||
error.status === 429 ? 429 : 502,
|
||||
error.status === 429
|
||||
? retrySeconds()
|
||||
: failure
|
||||
? retrySeconds(failure.until)
|
||||
: undefined,
|
||||
upstreamStatus === 429 || upstreamStatus >= 500
|
||||
? upstreamStatus
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Request one bounded observation sample and project its public fields. */
|
||||
async function getObservations(params, options = {}) {
|
||||
const query = observationFilters(params);
|
||||
// View samples stay stable in the route cache until their TTL expires.
|
||||
query.set('order_by', params.scope === 'view' ? 'random' : 'observed_on');
|
||||
if (params.scope !== 'view') query.set('order', 'desc');
|
||||
query.set('per_page', '200');
|
||||
query.set('fields', OBSERVATION_FIELDS);
|
||||
if (params.scope === 'view') {
|
||||
query.set('swlat', String(params.box.south));
|
||||
query.set('swlng', String(params.box.west));
|
||||
query.set('nelat', String(params.box.north));
|
||||
query.set('nelng', String(params.box.east));
|
||||
}
|
||||
return requestProjected(
|
||||
`${OBSERVATIONS_URL}?${query}`,
|
||||
(payload) => {
|
||||
if (
|
||||
!Array.isArray(payload?.results) ||
|
||||
!Number.isSafeInteger(payload.total_results) ||
|
||||
payload.total_results < 0
|
||||
)
|
||||
throw upstreamError('inaturalist_unavailable', 502);
|
||||
const rows = normalizeInatObservations(payload.results, {
|
||||
nowMs: now(),
|
||||
windowMs: (params.taxa === 'whales' ? 7 : 2) * DAY_MS,
|
||||
});
|
||||
// Empty views need total_results: 0. A wholly invalid page must not
|
||||
// replace good data. Valid rows excluded by the finer rolling-time
|
||||
// cutoff are a healthy empty result of the upstream DATE over-fetch.
|
||||
if (
|
||||
!rows ||
|
||||
(!rows.length &&
|
||||
(payload.results.length || payload.total_results) &&
|
||||
!normalizeInatObservations(payload.results)?.length)
|
||||
)
|
||||
throw upstreamError('inaturalist_unavailable', 502);
|
||||
return {
|
||||
rows,
|
||||
totalResults: payload.total_results,
|
||||
...(payload.total_results > payload.results.length
|
||||
? { partial: true, rawRowCount: payload.total_results }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
'observations',
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
/** Decode one validated density tile without observation identifiers. */
|
||||
async function getGrid(params, options = {}) {
|
||||
if (!validInatGridTile(params)) throw new TypeError('Invalid grid tile');
|
||||
const query = observationFilters(params);
|
||||
return requestProjected(
|
||||
`${API_URL}/grid/${params.z}/${params.x}/${params.y}.grid.json?${query}`,
|
||||
(payload) => projectInatGrid(payload, params),
|
||||
'grid',
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
getObservations,
|
||||
getGrid,
|
||||
/** Current provider-wide or resource-specific retry information. */
|
||||
cooldownState(resource) {
|
||||
if (now() < cooldownUntil)
|
||||
return {
|
||||
upstreamStatus: 429,
|
||||
retryAfterS: retrySeconds(),
|
||||
};
|
||||
const failure = failures.get(resource);
|
||||
if (failure && now() < failure.until)
|
||||
return {
|
||||
upstreamStatus: failure.status,
|
||||
retryAfterS: retrySeconds(failure.until),
|
||||
};
|
||||
return undefined;
|
||||
},
|
||||
/** The current UTC day's reserved upstream attempts and configured limit. */
|
||||
budgetState() {
|
||||
const value = currentBudget();
|
||||
return { ...value, budget: dailyLimit() };
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user