feat(layers): add shipwrecks layer (EMODnet + Wikidata)

A new Events layer maps shipwrecks with a card per wreck:
- EMODnet Human Activities heritage shipwrecks (about 6,500 in European
  waters, CC BY 4.0), skipping rows taken from wrecksite.eu.
- Wikidata shipwrecks with a Wikipedia article plus sinking events
  (CC0); an event folds into its ship for the date and death toll.
- A same-origin /api/shipwrecks proxy refreshes once a day and serves
  the last snapshot as stale for up to a week.

Markers are a boat icon coloured by era and sized by deaths; four era
chips filter the layer. The card follows the air-accidents layout
(deaths first). Text that names people is dropped; victims appear as
a count only.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Itallo BrandãoandClaude Opus 5.5 committed 2026-11-05 04:42:59 -03:00
1 parent 7d8b80db0a
commit dcd0f2cccd
41 files changed
+2168 -73

No files matched your search

+123
View File
@@ -0,0 +1,123 @@
import { formatDate, formatNumber, msg, t } from '../../i18n/index.js';
import { EMODNET_PAGE_URL } from './records.js';
export const ERA_LABELS = Object.freeze({
'before-1900': msg('Before 1900'),
'1900-1945': msg('1900–1945'),
'after-1945': msg('After 1945'),
undated: msg('Date unknown'),
});
export const ERA_COLORS = Object.freeze({
'before-1900': '#c79a5b',
'1900-1945': '#ff6b5e',
'after-1945': '#4fc3f7',
undated: '#9aa7b4',
});
function formatDay(date) {
const match = /^(-?\d{4})-(\d{2})-(\d{2})$/.exec(date || '');
if (!match) return '';
return formatDate(
Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3])),
{ day: 'numeric', month: 'short', year: 'numeric', timeZone: 'UTC' },
);
}
function meters(value) {
return formatNumber(value, {
maximumFractionDigits: value < 10 ? 1 : 0,
});
}
/** How well the position is known, as one card line (or null). */
export function positionLine(record) {
if (record.positionNote) return t('Position: centre of the protected area');
const accuracy = record.positionAccuracy;
if (!Number.isFinite(accuracy)) return null;
// 1852 m is EMODnet's "within a nautical mile" for old, reported positions.
if (accuracy >= 1000)
return t('Position approximate (within about {km} km)', {
km: formatNumber(accuracy / 1000, { maximumFractionDigits: 1 }),
});
return t('Position surveyed (within about {m} m)', { m: meters(accuracy) });
}
/**
* The link a click on the card opens: the Wikipedia (or Wikidata) page, the
* wreck's own page when EMODnet lists one, else EMODnet's portal.
* @param {object} record
* @returns {string}
*/
export function cardLink(record) {
return record?.url || EMODNET_PAGE_URL;
}
/**
* Title and detail lines for one wreck card, laid out like the air-accident
* card: deaths first, then when and where, then the ship facts. Only facts
* about the ship: the records carry no names of people, and victims appear as
* a count only.
* @param {object} record A normalized shipwreck record.
* @returns {{title: string, details: string[], link: string}}
*/
export function describeShipwreck(record) {
const titleParts = [t('Shipwreck')];
if (record.name) titleParts.push(record.name);
const details = [];
// Deaths first, right under the title: the fact readers look for.
const toll = [];
if (Number.isFinite(record.casualties))
toll.push(
t('Fatalities: {count}', { count: formatNumber(record.casualties) }),
);
if (record.survivors)
toll.push(
t('Survivors: {count}', { count: formatNumber(record.survivors) }),
);
if (toll.length) details.push(toll.join(' · '));
const when =
formatDay(record.date) ||
(Number.isInteger(record.year) ? String(record.year) : '') ||
record.dating ||
'';
const where = [when, record.country].filter(Boolean);
if (where.length) details.push(where.join(' · '));
if (record.vesselType)
details.push(t('Vessel: {value}', { value: record.vesselType }));
if (record.cause)
details.push(t('What happened: {value}', { value: record.cause }));
if (Number.isFinite(record.depth))
details.push(
record.depthQuality
? t('Depth: {m} m ({quality})', {
m: meters(record.depth),
quality: record.depthQuality,
})
: t('Depth: {m} m', { m: meters(record.depth) }),
);
const position = positionLine(record);
if (position) details.push(position);
if (record.notes) details.push(t('Notes: {value}', { value: record.notes }));
if (record.cargo)
details.push(t('Cargo or finds: {value}', { value: record.cargo }));
if (record.protected) details.push(t('Protected wreck'));
const sources =
record.source === 'wikidata'
? record.alsoIn === 'emodnet'
? 'Wikidata · EMODnet'
: 'Wikidata'
: record.originator
? `EMODnet (${record.originator})`
: 'EMODnet';
details.push(t('Source: {value}', { value: sources }));
const link = cardLink(record);
details.push(
link === EMODNET_PAGE_URL
? t('Click the card to open EMODnet ↗')
: link.includes('.wikipedia.org/')
? t('Click the card to open the article ↗')
: t('Click the card to open the source ↗'),
);
return { title: titleParts.join(' · '), details, link };
}
+63
View File
@@ -0,0 +1,63 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { describeShipwreck, positionLine } from './cards.js';
import { EMODNET_PAGE_URL } from './records.js';
test('a Wikidata card shows the ship facts and links its article', () => {
const card = describeShipwreck({
source: 'wikidata',
alsoIn: 'emodnet',
name: 'HMS Havant',
vesselType: 'destroyer',
date: '1940-06-01',
year: 1940,
cause: 'Was sunk on 06/01/1940 by the German air force',
casualties: 8,
survivors: 120,
depth: 14.72,
depthQuality: 'Least depth known',
positionAccuracy: 3.1,
notes: '1939 G and H-class destroyer',
country: 'France',
url: 'https://en.wikipedia.org/wiki/HMS_Havant_(H32)',
});
assert.equal(card.title, 'Shipwreck · HMS Havant');
assert.equal(card.link, 'https://en.wikipedia.org/wiki/HMS_Havant_(H32)');
const text = card.details.join('\n');
assert.equal(card.details[0], 'Fatalities: 8 · Survivors: 120');
assert.match(card.details[1], /1940 · France$/);
assert.match(text, /Vessel: destroyer/);
assert.match(text, /What happened: Was sunk/);
assert.match(text, /Depth: 15 m \(Least depth known\)/);
assert.match(text, /Position surveyed \(within about 3\.1 m\)/);
assert.match(text, /Source: Wikidata · EMODnet/);
assert.match(text, /open the article/);
});
test('an EMODnet card without a page links the portal', () => {
const card = describeShipwreck({
source: 'emodnet',
name: null,
year: null,
dating: 'C1st AD',
originator: 'The Oxford Roman Economy Project',
protected: true,
url: null,
});
assert.equal(card.title, 'Shipwreck');
assert.equal(card.link, EMODNET_PAGE_URL);
const text = card.details.join('\n');
assert.equal(card.details[0], 'C1st AD');
assert.match(text, /Protected wreck/);
assert.match(text, /EMODnet \(The Oxford Roman Economy Project\)/);
assert.match(text, /open EMODnet/);
});
test('position lines read the accuracy', () => {
assert.equal(positionLine({}), null);
assert.match(positionLine({ positionAccuracy: 1852 }), /about 1\.9 km/);
assert.match(
positionLine({ positionNote: 'x' }),
/centre of the protected area/,
);
});
+33
View File
@@ -0,0 +1,33 @@
/** Era chips, oldest first; a wreck with no known year sits in its own chip. */
export const SHIPWRECK_ERAS = Object.freeze([
'before-1900',
'1900-1945',
'after-1945',
'undated',
]);
/**
* The era a wreck belongs to. Ancient wrecks dated only by period
* ("Roman Imperial", "C1st AD") count as before 1900.
* @param {{year?: number|null, ancient?: boolean}} record
* @returns {string}
*/
export function eraOf(record) {
const year = Number.isInteger(record?.year) ? record.year : null;
if (year === null) return record?.ancient ? 'before-1900' : 'undated';
if (year < 1900) return 'before-1900';
if (year <= 1945) return '1900-1945';
return 'after-1945';
}
/**
* Normalize a requested set of eras; unknown names are ignored and an empty
* or missing request keeps the current set.
* @param {unknown} value Array of era ids.
* @param {Set<string>} current
* @returns {Set<string>}
*/
export function normalizeEras(value, current) {
if (!Array.isArray(value)) return new Set(current);
return new Set(value.filter((era) => SHIPWRECK_ERAS.includes(era)));
}
+23
View File
@@ -0,0 +1,23 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { SHIPWRECK_ERAS, eraOf, normalizeEras } from './eras.js';
test('each wreck falls in one era', () => {
assert.equal(eraOf({ year: 1588 }), 'before-1900');
assert.equal(eraOf({ year: null, ancient: true }), 'before-1900');
assert.equal(eraOf({ year: 1900 }), '1900-1945');
assert.equal(eraOf({ year: 1945 }), '1900-1945');
assert.equal(eraOf({ year: 1946 }), 'after-1945');
assert.equal(eraOf({ year: null }), 'undated');
assert.equal(eraOf(null), 'undated');
});
test('era requests keep known eras only', () => {
const all = new Set(SHIPWRECK_ERAS);
assert.deepEqual(
[...normalizeEras(['after-1945', 'bogus'], all)],
['after-1945'],
);
assert.deepEqual([...normalizeEras('after-1945', all)], SHIPWRECK_ERAS);
assert.deepEqual([...normalizeEras([], all)], []);
});
+424
View File
@@ -0,0 +1,424 @@
import * as Cesium from 'cesium';
import { formatNumber, msg, t, tn } from '../../i18n/index.js';
import { createHorizonCull } from '../streetLevel/view.js';
import { ERA_COLORS, ERA_LABELS, describeShipwreck } from './cards.js';
import { SHIPWRECK_ERAS, eraOf, normalizeEras } from './eras.js';
export * from './records.js';
export * from './cards.js';
export * from './eras.js';
export { createShipwrecksSource } from './source.js';
export const SHIPWRECKS_LAYER_ID = 'shipwrecks';
export const SHIPWRECKS_OVERLAY_SOURCE_ID = 'shipwrecks';
const PICK_PREFIX = 'shipwreck:';
const CARD_HOST_OPTIONS = Object.freeze({
cohortLimit: 1,
collisionCapacity: 1,
moving: false,
});
const MAX_CARD_LINES = 14;
/**
* Marker size in pixels: grows with the death toll, as on the air-accidents
* layer, from 16 px (no deaths known) to 36 px (about a thousand or more).
*/
export function markerPixelSize(casualties) {
const n = Number.isFinite(casualties) && casualties > 0 ? casualties : 0;
return Math.round(16 + Math.min(20, Math.log2(1 + n) * 2));
}
/**
* A round marker in the era's colour with a sinking hull and mast in the
* middle, as an SVG data URL.
* @param {string} color CSS colour.
* @returns {string}
*/
export function wreckMarkerImage(color) {
const svg =
`<svg xmlns="http://www.w3.org/2000/svg" width="36" height="36" viewBox="0 0 36 36">` +
`<circle cx="18" cy="18" r="16" fill="${color}" stroke="#ffffff" stroke-width="2"/>` +
`<path d="M8 21h20l-3.5 5h-13z" fill="#0b1420"/>` +
`<path d="M17 8v12M17 9l6 7h-6" fill="none" stroke="#0b1420" stroke-width="2.2" stroke-linejoin="round"/>` +
`</svg>`;
return `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`;
}
/**
* Shipwrecks: EMODnet's European heritage wrecks and Wikidata's notable
* wrecks worldwide, one marker each, with a card and era chips.
*/
export function createShipwrecksLayer({
source,
overlayHost = null,
screenSpaceEventHandlerFactory = null,
picking = null,
pointer = null,
openExternal = null,
} = {}) {
if (typeof source?.getSnapshot !== 'function')
throw new TypeError('Shipwrecks require a snapshot source');
let _viewer = null;
let _enabled = false;
let _dataSource = null;
let _request = null;
let _records = new Map();
let _pins = [];
let _snapshot = null;
let _lastError = null;
let _clickHandler = null;
let _rowListener = null;
let _selectedId = null;
let _selectedCardId = null;
let _selectedLink = null;
let _eras = new Set(SHIPWRECK_ERAS);
const _images = Object.fromEntries(
SHIPWRECK_ERAS.map((era) => [era, wreckMarkerImage(ERA_COLORS[era])]),
);
const render = () => _viewer?.scene?.requestRender?.();
const notifyRow = () => _rowListener?.();
const canSelect = () =>
overlayHost && screenSpaceEventHandlerFactory && picking;
// Markers skip the depth test so terrain does not swallow them; the
// horizon cull hides the far side of the planet instead.
const _horizon = createHorizonCull({
getViewer: () => _viewer,
items: () => _pins,
onChange: render,
});
/** Shown: its era chip is on. */
function visible(record) {
return Boolean(record) && _eras.has(eraOf(record));
}
function applyPin(pin) {
pin.entity.show = pin.inFront && visible(pin.record);
}
/** Re-apply the era chips to every marker and the open card. */
function applyFilter() {
for (const pin of _pins) applyPin(pin);
if (_selectedId && !visible(_records.get(_selectedId))) clearSelection();
notifyRow();
render();
}
function rebuild(records) {
_records = new Map(records.map((record) => [record.id, record]));
_pins = [];
if (!_dataSource) return;
_dataSource.entities.suspendEvents();
_dataSource.entities.removeAll();
for (const record of records) {
const position = Cesium.Cartesian3.fromDegrees(record.lon, record.lat);
const entity = _dataSource.entities.add({
id: `${PICK_PREFIX}${record.id}`,
position,
billboard: {
image: _images[eraOf(record)],
width: markerPixelSize(record.casualties),
height: markerPixelSize(record.casualties),
heightReference: Cesium.HeightReference.CLAMP_TO_GROUND,
disableDepthTestDistance: Number.POSITIVE_INFINITY,
// Bigger toll, drawn on top of smaller neighbours.
eyeOffset: new Cesium.Cartesian3(
0,
0,
-Math.min(record.casualties || 0, 5000),
),
},
});
const pin = {
entity,
record,
position,
inFront: true,
get show() {
return this.inFront;
},
set show(value) {
this.inFront = value;
applyPin(this);
},
};
applyPin(pin);
_pins.push(pin);
}
_dataSource.entities.resumeEvents();
_horizon.update();
if (_selectedId && !_records.has(_selectedId)) clearSelection();
else if (_selectedId) publishCard();
render();
}
// ---- selection card ----------------------------------------------------
function publishCard() {
if (!canSelect()) return;
const record = _selectedId ? _records.get(_selectedId) : null;
if (!visible(record)) {
clearSelection();
return;
}
const { title, details, link } = describeShipwreck(record);
_selectedLink = link && openExternal ? link : null;
const card = {
id: `shipwreck-card:${record.id}`,
selected: true,
interactive: Boolean(_selectedLink),
title,
details: details.slice(0, MAX_CARD_LINES),
accent: ERA_COLORS[eraOf(record)],
priority: Number.MAX_SAFE_INTEGER,
gapPx: 15,
verticalOnly: true,
placement: 'above',
stateless: true,
horizonCull: true,
position: Cesium.Cartesian3.fromDegrees(record.lon, record.lat),
};
if (card.interactive) {
const target = _selectedLink;
card.accessibilityLabel = t('Open the source for {title}', { title });
card.activate = () => {
openExternal(target);
return true;
};
}
_selectedCardId = card.id;
overlayHost.setEntries(
SHIPWRECKS_OVERLAY_SOURCE_ID,
[card],
CARD_HOST_OPTIONS,
);
overlayHost.setVisible?.(SHIPWRECKS_OVERLAY_SOURCE_ID, true);
}
function clearSelection() {
_selectedId = null;
_selectedCardId = null;
_selectedLink = null;
overlayHost?.setEntries(
SHIPWRECKS_OVERLAY_SOURCE_ID,
[],
CARD_HOST_OPTIONS,
);
}
function pickedRecordId(picked) {
const pickId = picking.resolvePickId(picked);
if (typeof pickId !== 'string' || !pickId.startsWith(PICK_PREFIX))
return null;
const id = pickId.slice(PICK_PREFIX.length);
return _records.has(id) ? id : null;
}
function installClickHandler() {
if (!canSelect() || _clickHandler || !_viewer) return;
picking.registerPickOwner?.(
SHIPWRECKS_LAYER_ID,
(pickedId) =>
typeof pickedId === 'string' && pickedId.startsWith(PICK_PREFIX),
);
_clickHandler = screenSpaceEventHandlerFactory(_viewer);
_clickHandler.setInputAction((click) => {
if (pointer && !pointer.isPointerFree()) return;
const cardHit = overlayHost.hitTest?.(
click.position?.x,
click.position?.y,
{ sourceId: SHIPWRECKS_OVERLAY_SOURCE_ID },
);
if (cardHit && cardHit.entryId === _selectedCardId) {
if (_selectedLink) openExternal(_selectedLink);
return;
}
const picked = _viewer.scene.pick(click.position);
const id = picked ? pickedRecordId(picked) : null;
if (id) {
_selectedId = id;
publishCard();
render();
return;
}
if (picked) {
const pickId = picking.resolvePickId(picked);
if (pickId && picking.isOwnedByOtherLayer(SHIPWRECKS_LAYER_ID, pickId))
return;
}
if (_selectedId) {
clearSelection();
render();
}
}, Cesium.ScreenSpaceEventType.LEFT_CLICK);
}
function removeClickHandler() {
_clickHandler?.destroy();
_clickHandler = null;
picking?.unregisterPickOwner?.(SHIPWRECKS_LAYER_ID);
}
function toggleEra(era) {
if (_eras.has(era)) _eras.delete(era);
else _eras.add(era);
applyFilter();
}
function countOf(era) {
let count = 0;
for (const record of _records.values()) if (eraOf(record) === era) count++;
return count;
}
const layer = {
id: SHIPWRECKS_LAYER_ID,
name: 'Shipwrecks',
icon: '⚓',
source: 'EMODnet · Wikidata',
// The proxy refreshes once a day; this only notices a new snapshot.
updateInterval: 6 * 60 * 60 * 1000,
init(viewer) {
if (_viewer) throw new Error('Shipwrecks layer is already initialized');
_viewer = viewer;
_dataSource = new Cesium.CustomDataSource(SHIPWRECKS_LAYER_ID);
_dataSource.show = false;
viewer.dataSources.add(_dataSource);
},
enable() {
_enabled = true;
if (_dataSource) _dataSource.show = true;
installClickHandler();
if (_pins.length) _horizon.update();
},
disable() {
_enabled = false;
_request?.abort();
_request = null;
if (_dataSource) _dataSource.show = false;
removeClickHandler();
clearSelection();
_horizon.stop();
render();
},
async update() {
if (!_enabled || !_dataSource) return false;
_request?.abort();
const request = new AbortController();
_request = request;
try {
const snapshot = await source.getSnapshot({ signal: request.signal });
if (request.signal.aborted || _request !== request || !_enabled)
return false;
_snapshot = snapshot;
_lastError =
snapshot.sources.emodnet && snapshot.sources.wikidata
? null
: snapshot.sources.emodnet
? msg('Wikidata unavailable; showing EMODnet only')
: msg('EMODnet unavailable; showing Wikidata only');
rebuild(snapshot.records);
notifyRow();
return true;
} catch (error) {
if (request.signal.aborted || _request !== request || !_enabled)
return false;
console.warn('[Data:Shipwrecks] Fetch error:', error);
_lastError = error?.message || msg('Shipwrecks unavailable');
notifyRow();
return false;
} finally {
if (_request === request) _request = null;
}
},
destroy(viewer = _viewer) {
layer.disable();
if (_dataSource) {
viewer?.dataSources?.remove(_dataSource, true);
_dataSource = null;
}
_records = new Map();
_pins = [];
_snapshot = null;
_viewer = null;
_rowListener = null;
},
/** The era filter: `eras` lists the era chips that are on. */
getParams() {
return { eras: SHIPWRECK_ERAS.filter((era) => _eras.has(era)) };
},
setParams(params = {}) {
if (!('eras' in params)) return true;
const next = normalizeEras(params.eras, _eras);
if (next.size === _eras.size && [...next].every((era) => _eras.has(era)))
return true;
_eras = next;
applyFilter();
return true;
},
setRowControlsListener(listener) {
_rowListener = listener;
},
getRowControls() {
return {
chips: SHIPWRECK_ERAS.map((era) => ({
id: era,
label: t(ERA_LABELS[era]),
active: _eras.has(era),
color: ERA_COLORS[era],
title: tn(countOf(era), '{count} wreck', '{count} wrecks', {
count: formatNumber(countOf(era)),
}),
onClick: () => toggleEra(era),
})),
legend: [],
info: t('EMODnet (European waters) · Wikidata (worldwide)'),
infoTitle: t(
'European heritage wrecks come from EMODnet Human Activities; notable wrecks worldwide come from Wikidata. Colour shows when the ship sank; marker size grows with deaths. Click a marker for its card.',
),
};
},
getStats() {
let shown = 0;
for (const record of _records.values()) if (visible(record)) shown++;
return {
count: shown,
lastUpdate: _snapshot?.fetchedAt ?? null,
stale: _snapshot?.stale === true,
error: _lastError,
};
},
/** Loaded records as plain JSON-safe objects, for tools and QA. */
getAnalystRecords(maxCount = 2000) {
if (!_enabled) return [];
const out = [];
for (const record of _records.values()) {
if (out.length >= maxCount) break;
if (!visible(record)) continue;
out.push({
id: record.id,
source: record.source,
name: record.name,
vesselType: record.vesselType,
year: record.year,
era: eraOf(record),
lat: record.lat,
lon: record.lon,
depth: record.depth,
casualties: record.casualties,
});
}
return out;
},
};
return layer;
}
+428
View File
@@ -0,0 +1,428 @@
// Shared by the server proxy (server/providers/shipwrecks.js) and the browser
// layer, so it must not load the i18n catalogs: values stay data.
/** EMODnet Human Activities "Cultural Heritage – Ship wrecks" (CC BY 4.0). */
export const EMODNET_WFS_URL =
'https://ows.emodnet-humanactivities.eu/wfs?service=WFS&request=GetFeature' +
'&version=2.0.0&typeNames=emodnet:heritageshipwrecks' +
'&outputFormat=application/json&srsName=EPSG:4326';
/** Landing page shown when an EMODnet wreck has no web page of its own. */
export const EMODNET_PAGE_URL =
'https://emodnet.ec.europa.eu/en/human-activities';
export const SHIPWRECK_SOURCES = Object.freeze(['emodnet', 'wikidata']);
/**
* Wikidata (CC0): shipwreck items that have at least one Wikipedia article,
* so the worldwide set stays to notable wrecks and skips bulk register
* imports, plus sinking events ("sinking of the RMS Titanic"), which are where
* Wikidata keeps the death toll. Dates come with their precision so a
* month-precise date is not shown as the 1st. Only ship and event facts are
* read: no items about people.
*/
export const WIKIDATA_QUERY = `SELECT ?item ?itemLabel ?itemDescription ?coord ?sunk ?sunkPrecision ?end ?deaths ?survivors ?typeLabel ?enwiki ?ptwiki ?event WHERE {
{
?item wdt:P31/wdt:P279* wd:Q852190 ; wdt:P625 ?coord ; wikibase:sitelinks ?links .
FILTER(?links > 0)
OPTIONAL { ?item p:P793 ?wreck . ?wreck ps:P793 wd:Q906512 ; pqv:P585 [ wikibase:timeValue ?sunk ; wikibase:timePrecision ?sunkPrecision ] }
OPTIONAL { ?item wdt:P576 ?end }
OPTIONAL { ?item wdt:P31 ?type . FILTER(?type != wd:Q852190) }
} UNION {
?item wdt:P31/wdt:P279* wd:Q906512 ; wdt:P625 ?coord .
BIND("event" AS ?event)
OPTIONAL { ?item p:P585/psv:P585 [ wikibase:timeValue ?sunk ; wikibase:timePrecision ?sunkPrecision ] }
OPTIONAL { ?item wdt:P1561 ?survivors }
}
OPTIONAL { ?item wdt:P1120 ?deaths }
OPTIONAL { ?enwiki schema:about ?item ; schema:isPartOf <https://en.wikipedia.org/> }
OPTIONAL { ?ptwiki schema:about ?item ; schema:isPartOf <https://pt.wikipedia.org/> }
SERVICE wikibase:label { bd:serviceParam wikibase:language "en,pt,mul". }
}`;
const MAX_TEXT = 160;
const MAX_NOTE = 260;
/** Same name within this distance: one wreck seen by both sources. */
const SAME_WRECK_KM = 3;
// Free text sometimes names people (owners, masters, divers who reported a
// wreck). The layer shows ships, never people, so such text is dropped whole.
const PERSON_TEXT =
/\b(mr|mrs|ms|dr|capt|captain|master|skipper|owner|owned|crew|body|bodies|drowned|reported by|according to)\b/i;
/** EMODnet writes "n/a" or nothing where a value is unknown. */
function clean(value, max = MAX_TEXT) {
if (typeof value !== 'string') return null;
const text = value
.replace(/[\u0000-\u001f<>]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
if (!text || /^(n\/a|null|unknown|none|-)$/i.test(text)) return null;
return text.length > max ? `${text.slice(0, max - 1).trimEnd()}…` : text;
}
/** Free text with no personal names in it, or null. */
function impersonalText(value, max) {
const text = clean(value, max);
return text && !PERSON_TEXT.test(text) ? text : null;
}
function positiveNumber(value, max) {
const n = typeof value === 'number' ? value : Number(value);
return Number.isFinite(n) && n > 0 && n <= max ? n : null;
}
function count(value) {
const text = String(value ?? '').replace(/\.0+$/, '');
if (!/^\d{1,6}$/.test(text)) return null;
return Number(text);
}
/** A sinking year from "1917", "1917-1918" or "c. 1917"; null otherwise. */
export function parseYear(value, maxYear = new Date().getUTCFullYear()) {
const match = /(?:^|\D)(\d{3,4})(?:\D|$)/.exec(String(value ?? ''));
if (!match) return null;
const year = Number(match[1]);
return year >= 100 && year <= maxYear ? year : null;
}
function validPoint(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
Math.abs(lat) <= 90 &&
Math.abs(lon) <= 180 &&
!(lat === 0 && lon === 0)
);
}
function httpUrl(value) {
const text = clean(value, 400);
if (!text) return null;
try {
const url = new URL(text);
return url.protocol === 'https:' || url.protocol === 'http:'
? url.href
: null;
} catch {
return null;
}
}
/**
* Normalize EMODnet's shipwreck GeoJSON. Records the portal took from
* wrecksite.eu are skipped: that site's data is not openly licensed.
* @param {object} payload WFS GeoJSON FeatureCollection.
* @returns {object[]} Shipwreck records.
*/
export function normalizeEmodnetFeatures(payload) {
const features = payload?.features;
if (!Array.isArray(features)) return [];
const out = [];
const ids = new Set();
for (const feature of features) {
const p = feature?.properties || {};
const coords = feature?.geometry?.coordinates;
if (feature?.geometry?.type !== 'Point' || !Array.isArray(coords)) continue;
const lon = Number(coords[0]);
const lat = Number(coords[1]);
if (!validPoint(lat, lon)) continue;
const origin = clean(p.source_inf, 80);
if (origin && /wrecksite/i.test(origin)) continue;
const key = clean(String(feature.id ?? p.source_id ?? ''), 80);
if (!key) continue;
const id = `emodnet:${key}`;
if (ids.has(id)) continue;
ids.add(id);
const year = parseYear(p.sink_yr);
const dating = clean(p.dating, 60);
const period = clean(p.period, 60);
const vesselType =
clean(p.obj_type, 60) ||
(() => {
// ship_char is sometimes a short type ("Trawler"), sometimes a long
// history that can name people; only the short form is a type.
const text = clean(p.ship_char, 40);
return text && !/\d/.test(text) && !PERSON_TEXT.test(text)
? text
: null;
})();
out.push({
id,
source: 'emodnet',
lat,
lon,
name: clean(p.name, 100),
vesselType,
year,
date: null,
dating: year ? null : dating,
ancient: !year && Boolean(period || dating),
cause: impersonalText(p.sink_context, MAX_NOTE),
notes: period && dating ? `${period} (${dating})` : period,
cargo: impersonalText(p.artefacts, 120),
protected: /protected/i.test(p.statutory || ''),
depth: positiveNumber(p.least_depth, 11000),
depthQuality: clean(p.depth_prec, 80),
positionAccuracy: positiveNumber(p.loc_prec, 100000),
positionNote: /centroid/i.test(p.point_info || '')
? 'Centre of the protected area'
: null,
casualties: null,
survivors: null,
country: clean(p.country, 60),
originator: origin,
url: httpUrl(p.website1),
});
}
return out;
}
function wikiUrl(value) {
const url = httpUrl(value);
return url && /^https:\/\/[a-z]{2,3}\.wikipedia\.org\/wiki\//.test(url)
? url
: null;
}
/**
* A Wikidata time as a day ("1912-04-15") when it is day-precise, else as a
* year only: a month- or year-precise value reads back as the 1st.
* @returns {{date: string|null, year: number|null}}
*/
function wikidataTime(value, precision) {
const match = /^(-?\d{4})-(\d{2})-(\d{2})T/.exec(value || '');
if (!match) return { date: null, year: null };
const year = parseYear(match[1]);
const p = Number(precision);
// 11 = day; 9 = year; 10 = month. A missing precision (P576) counts as a day.
if (!Number.isFinite(p) || p >= 11)
return { date: `${match[1]}-${match[2]}-${match[3]}`, year };
return { date: null, year: p >= 9 ? year : null };
}
/** "sinking of the RMS Titanic" → "Sinking of the RMS Titanic". */
function sentenceCase(text) {
return text ? text.charAt(0).toUpperCase() + text.slice(1) : text;
}
/**
* Normalize Wikidata SPARQL results to shipwreck records (one per item).
* Sinking events are folded into the ship they sank when both are listed.
* @param {object} payload SPARQL JSON results.
* @returns {object[]}
*/
export function normalizeWikidataResults(payload) {
const bindings = payload?.results?.bindings;
if (!Array.isArray(bindings)) return [];
const byItem = new Map();
for (const binding of bindings) {
const value = (key) => binding?.[key]?.value;
const qid = /\/entity\/(Q\d{1,12})$/.exec(value('item') || '')?.[1];
if (!qid) continue;
const sunk = wikidataTime(value('sunk'), value('sunkPrecision'));
const ended = wikidataTime(value('end'), null);
const known = byItem.get(qid);
if (known) {
// Extra rows only add what the first one lacked.
known.vesselType ||= clean(value('typeLabel'), 60);
if (!known.date && !known.year) {
known.date = sunk.date || ended.date;
known.year = sunk.year ?? ended.year;
}
known.casualties ??= count(value('deaths'));
known.survivors ??= count(value('survivors'));
continue;
}
const point = /^Point\(([-\d.eE]+) ([-\d.eE]+)\)$/.exec(
value('coord') || '',
);
const lon = point ? Number(point[1]) : NaN;
const lat = point ? Number(point[2]) : NaN;
if (!validPoint(lat, lon)) continue;
const name = clean(value('itemLabel'), 100);
const event = value('event') === 'event';
const when = sunk.date || sunk.year ? sunk : ended;
byItem.set(qid, {
id: `wikidata:${qid}`,
source: 'wikidata',
event,
lat,
lon,
// An unlabelled item reads back as its own Q-id; that is no name.
name: name && name !== qid ? (event ? sentenceCase(name) : name) : null,
vesselType: event ? null : clean(value('typeLabel'), 60),
year: when.year,
date: when.date,
dating: null,
ancient: false,
cause: null,
notes: impersonalText(value('itemDescription'), MAX_TEXT),
cargo: null,
protected: false,
depth: null,
depthQuality: null,
positionAccuracy: null,
positionNote: null,
casualties: count(value('deaths')),
survivors: count(value('survivors')),
country: null,
originator: null,
url:
wikiUrl(value('enwiki')) ||
wikiUrl(value('ptwiki')) ||
`https://www.wikidata.org/wiki/${qid}`,
});
}
return foldSinkingEvents([...byItem.values()]);
}
/** An event this far from a listed ship whose name it contains is that ship's sinking. */
const SAME_SINKING_KM = 50;
/**
* "Sinking of the RMS Titanic" near the RMS Titanic wreck is one marker: the
* ship keeps its article and takes the event's date, deaths and survivors.
*/
function foldSinkingEvents(records) {
// Wikidata sometimes has both the ship and its wreck as items at one spot
// ("Titanic", "wreck of the RMS Titanic"): keep the ship, one marker.
const isWreckItem = (record) => (/^wreck\b/i.test(record.name || '') ? 1 : 0);
const ships = [];
const candidates = records
.filter((record) => !record.event)
.sort((a, b) => isWreckItem(a) - isWreckItem(b));
for (const record of candidates) {
const key = nameKey(record.name);
const twin =
key &&
ships.find(
(ship) =>
nameKey(ship.name) === key &&
distanceKm(ship, record) <= SAME_WRECK_KM,
);
if (!twin) {
ships.push(record);
continue;
}
twin.vesselType ||= record.vesselType;
if (!twin.date && !twin.year) {
twin.date = record.date;
twin.year = record.year;
}
twin.casualties ??= record.casualties;
twin.survivors ??= record.survivors;
}
const out = [...ships];
for (const event of records.filter((record) => record.event)) {
const eventKey = ` ${nameKey(event.name)} `;
const ship = ships.find((candidate) => {
const key = nameKey(candidate.name);
return (
key.length >= 3 &&
eventKey.includes(` ${key} `) &&
distanceKm(candidate, event) <= SAME_SINKING_KM
);
});
if (!ship) {
out.push(event);
continue;
}
if (!ship.date && !ship.year) {
ship.date = event.date;
ship.year = event.year;
} else if (!ship.date && event.date && event.year === ship.year) {
ship.date = event.date;
}
ship.casualties ??= event.casualties;
ship.survivors ??= event.survivors;
}
for (const record of out) delete record.event;
return out;
}
function nameKey(name) {
return String(name || '')
.normalize('NFD')
.replace(/[̀-ͯ]/g, '')
.toLowerCase()
.replace(/\([^)]*\)/g, ' ')
.replace(/\b(wreck|sinking|loss|disaster|of)\b/g, ' ')
.replace(
/\b(hms|hmt|hmhs|hmas|sms|ss|ms|mv|rms|uss|hm|s\/s|m\/v|the)\b/g,
' ',
)
.replace(/[^a-z0-9]+/g, ' ')
.trim();
}
function distanceKm(a, b) {
const rad = Math.PI / 180;
const dLat = (b.lat - a.lat) * rad;
const dLon = (b.lon - a.lon) * rad;
const h =
Math.sin(dLat / 2) ** 2 +
Math.cos(a.lat * rad) * Math.cos(b.lat * rad) * Math.sin(dLon / 2) ** 2;
return 12742 * Math.asin(Math.min(1, Math.sqrt(h)));
}
/**
* One list from both sources. A wreck in both (same name, a few km apart)
* keeps the Wikidata record, which links an article, and takes EMODnet's
* survey facts (depth, sinking text, year) that Wikidata lacks.
*/
export function mergeShipwreckRecords(emodnet = [], wikidata = []) {
const byName = new Map();
for (const record of wikidata) {
const key = nameKey(record.name);
if (!key) continue;
if (!byName.has(key)) byName.set(key, []);
byName.get(key).push(record);
}
const kept = [];
for (const record of emodnet) {
const twin = (byName.get(nameKey(record.name)) || []).find(
(candidate) => distanceKm(candidate, record) <= SAME_WRECK_KM,
);
if (!twin) {
kept.push(record);
continue;
}
twin.year ??= record.year;
twin.vesselType ||= record.vesselType;
twin.cause ||= record.cause;
twin.depth ??= record.depth;
twin.depthQuality ||= record.depthQuality;
twin.positionAccuracy ??= record.positionAccuracy;
twin.country ||= record.country;
twin.protected ||= record.protected;
twin.alsoIn = 'emodnet';
}
return [...wikidata, ...kept];
}
/**
* Validate the proxy's snapshot in the browser.
* @returns {object[]|null} Records, or null when the payload is malformed.
*/
export function normalizeShipwreckSnapshot(payload) {
if (!Array.isArray(payload?.records)) return null;
const rows = [];
const ids = new Set();
for (const record of payload.records) {
if (
!record ||
typeof record.id !== 'string' ||
ids.has(record.id) ||
!SHIPWRECK_SOURCES.includes(record.source) ||
!validPoint(record.lat, record.lon)
)
continue;
ids.add(record.id);
rows.push({
...record,
year: Number.isInteger(record.year) ? record.year : null,
url: httpUrl(record.url),
});
}
return rows;
}
+212
View File
@@ -0,0 +1,212 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
mergeShipwreckRecords,
normalizeEmodnetFeatures,
normalizeShipwreckSnapshot,
normalizeWikidataResults,
parseYear,
} from './records.js';
function feature(id, properties, coordinates = [-4.35, 54.13]) {
return {
type: 'Feature',
id,
geometry: { type: 'Point', coordinates },
properties,
};
}
test('EMODnet wrecks keep ship facts and drop text that names people', () => {
const [wreck, roman] = normalizeEmodnetFeatures({
features: [
feature('w.1', {
name: 'MAINA-NUI',
sink_yr: '1987',
sink_context: 'Sank on 07/02/1987',
obj_type: 'Pusher or tug',
least_depth: 12.5,
depth_prec: 'Least depth known',
loc_prec: 50,
country: 'France',
source_inf: 'SHOM',
statutory: 'n/a',
website1: 'n/a',
}),
feature('w.2', {
name: 'Lerici',
sink_yr: '',
dating: 'C1st-2nd AD',
period: 'Roman Imperial',
artefacts: 'Marble, Columns',
obj_type: 'n/a',
ship_char: 'Owned by J. Smith, master was A. Jones',
sink_context: 'Wrecked here, reported by Mr Smith',
}),
feature('w.3', { name: 'Copied', source_inf: 'www.wrecksite.eu' }),
feature('w.4', { name: 'Null island' }, [0, 0]),
],
});
assert.equal(wreck.id, 'emodnet:w.1');
assert.equal(wreck.year, 1987);
assert.equal(wreck.vesselType, 'Pusher or tug');
assert.equal(wreck.cause, 'Sank on 07/02/1987');
assert.equal(wreck.depth, 12.5);
assert.equal(wreck.positionAccuracy, 50);
assert.equal(wreck.url, null);
assert.equal(roman.year, null);
assert.equal(roman.ancient, true);
assert.equal(roman.dating, 'C1st-2nd AD');
assert.equal(roman.notes, 'Roman Imperial (C1st-2nd AD)');
assert.equal(roman.cargo, 'Marble, Columns');
assert.equal(roman.vesselType, null, 'a history naming people is no type');
assert.equal(roman.cause, null, 'text naming a person is dropped');
});
test('Wikidata rows fold into one record per item', () => {
const item = 'http://www.wikidata.org/entity/Q1471525';
const records = normalizeWikidataResults({
results: {
bindings: [
{
item: { value: item },
itemLabel: { value: 'HMS Havant' },
itemDescription: { value: '1939 G and H-class destroyer' },
coord: { value: 'Point(2.26 51.13)' },
sunk: { value: '1940-06-01T00:00:00Z' },
enwiki: { value: 'https://en.wikipedia.org/wiki/HMS_Havant_(H32)' },
},
{
item: { value: item },
typeLabel: { value: 'destroyer' },
deaths: { value: '8' },
coord: { value: 'Point(2.26 51.13)' },
},
{
item: { value: 'http://www.wikidata.org/entity/Q9' },
itemLabel: { value: 'Q9' },
coord: { value: 'Point(10 10)' },
},
],
},
});
assert.equal(records.length, 2);
const [havant, unnamed] = records;
assert.equal(havant.date, '1940-06-01');
assert.equal(havant.year, 1940);
assert.equal(havant.vesselType, 'destroyer');
assert.equal(havant.casualties, 8);
assert.equal(havant.url, 'https://en.wikipedia.org/wiki/HMS_Havant_(H32)');
assert.equal(unnamed.name, null);
assert.equal(unnamed.url, 'https://www.wikidata.org/wiki/Q9');
});
test('a wreck in both sources keeps Wikidata and takes EMODnet survey facts', () => {
const wikidata = [
{
id: 'wikidata:Q1',
source: 'wikidata',
name: 'HMS Havant',
lat: 51.13,
lon: 2.26,
year: null,
depth: null,
},
];
const emodnet = [
{
id: 'emodnet:a',
source: 'emodnet',
name: 'Havant (HMS)',
lat: 51.131,
lon: 2.262,
year: 1940,
depth: 14.7,
},
{
id: 'emodnet:b',
source: 'emodnet',
name: 'HMS Havant',
lat: 45,
lon: -10,
year: 1950,
},
];
const merged = mergeShipwreckRecords(emodnet, wikidata);
assert.deepEqual(
merged.map((record) => record.id),
['wikidata:Q1', 'emodnet:b'],
);
assert.equal(merged[0].year, 1940);
assert.equal(merged[0].depth, 14.7);
assert.equal(merged[0].alsoIn, 'emodnet');
});
test('years and snapshots are validated', () => {
assert.equal(parseYear('1917-1918'), 1917);
assert.equal(parseYear('c. 1588'), 1588);
assert.equal(parseYear(''), null);
assert.equal(parseYear('3000'), null);
assert.equal(normalizeShipwreckSnapshot({}), null);
const rows = normalizeShipwreckSnapshot({
records: [
{ id: 'a', source: 'emodnet', lat: 1, lon: 2, url: 'javascript:x' },
{ id: 'a', source: 'emodnet', lat: 1, lon: 2 },
{ id: 'b', source: 'other', lat: 1, lon: 2 },
{ id: 'c', source: 'wikidata', lat: 91, lon: 2 },
],
});
assert.equal(rows.length, 1);
assert.equal(rows[0].url, null);
});
test('a sinking event folds into its ship; month-precise dates keep the year only', () => {
const entity = (q) => ({ value: `http://www.wikidata.org/entity/${q}` });
const records = normalizeWikidataResults({
results: {
bindings: [
{
item: entity('Q25173'),
itemLabel: { value: 'Titanic' },
coord: { value: 'Point(-49.9469 41.7325)' },
},
{
item: entity('Q3018259'),
itemLabel: { value: 'wreck of the RMS Titanic' },
coord: { value: 'Point(-49.9469 41.7325)' },
},
{
item: entity('Q2577588'),
itemLabel: { value: 'sinking of the RMS Titanic' },
coord: { value: 'Point(-49.9458 41.7319)' },
event: { value: 'event' },
sunk: { value: '1912-04-01T00:00:00Z' },
sunkPrecision: { value: '10' },
deaths: { value: '1501' },
survivors: { value: '712' },
},
{
item: entity('Q1245004'),
itemLabel: { value: 'Costa Concordia disaster' },
coord: { value: 'Point(10.92 42.37)' },
event: { value: 'event' },
sunk: { value: '2012-01-13T00:00:00Z' },
sunkPrecision: { value: '11' },
deaths: { value: '32' },
},
],
},
});
assert.deepEqual(
records.map((record) => record.id),
['wikidata:Q25173', 'wikidata:Q1245004'],
);
const [titanic, concordia] = records;
assert.equal(titanic.date, null);
assert.equal(titanic.year, 1912);
assert.equal(titanic.casualties, 1501);
assert.equal(titanic.survivors, 712);
assert.equal(concordia.name, 'Costa Concordia disaster');
assert.equal(concordia.date, '2012-01-13');
assert.equal('event' in concordia, false);
});
+35
View File
@@ -0,0 +1,35 @@
import { readResponseJsonCapped } from '../../sources/httpBody.js';
import { normalizeShipwreckSnapshot } from './records.js';
/** Same-origin source for the proxy's EMODnet + Wikidata shipwreck snapshot. */
export function createShipwrecksSource({
fetchImpl = (...args) => globalThis.fetch(...args),
} = {}) {
return {
async getSnapshot({ signal } = {}) {
signal?.throwIfAborted();
const response = await fetchImpl('/api/shipwrecks', { signal });
if (!response.ok)
throw new Error(`Shipwrecks unavailable (HTTP ${response.status})`);
const payload = await readResponseJsonCapped(
response,
16 * 1024 * 1024,
signal,
);
signal?.throwIfAborted();
const records = normalizeShipwreckSnapshot(payload);
if (!records) throw new Error('Malformed shipwrecks response');
return {
records,
fetchedAt: Number.isFinite(payload.fetchedAt)
? payload.fetchedAt
: null,
stale: payload.stale === true,
sources: {
emodnet: payload.sources?.emodnet !== false,
wikidata: payload.sources?.wikidata !== false,
},
};
},
};
}