fix(shipwrecks): skip every wrecksite.eu row, even when the name comes after the 80-character cut or only in the link

The exclusion tested the provenance after it was cut to 80 characters, so a
long source list ending in wrecksite.eu kept the row, and a row whose only
trace was a wrecksite.eu page link was kept too. Both raw values are now
checked before anything is cut.
This commit is contained in:
Sameh Khamis committed 2026-11-10 13:16:42 -08:00
1 parent 584138decf
commit 32e7a1a913
2 files changed
+22 -2

No files matched your search

+10 -2
View File
@@ -108,9 +108,15 @@ function httpUrl(value) {
}
}
/** Whether a provenance or link value points at wrecksite.eu, in any form. */
function fromWrecksite(value) {
return typeof value === 'string' && /wrecksite/i.test(value);
}
/**
* Normalize EMODnet's shipwreck GeoJSON. Records the portal took from
* wrecksite.eu are skipped: that site's data is not openly licensed.
* wrecksite.eu, named in the provenance or linked as the record's page, are
* skipped: that site's data is not openly licensed.
* @param {object} payload WFS GeoJSON FeatureCollection.
* @returns {object[]} Shipwreck records.
*/
@@ -126,8 +132,10 @@ export function normalizeEmodnetFeatures(payload) {
const lon = Number(coords[0]);
const lat = Number(coords[1]);
if (!validPoint(lat, lon)) continue;
// Checked on the raw values: the shown origin is cut to 80 characters,
// and a long provenance list can name wrecksite.eu after the cut.
if (fromWrecksite(p.source_inf) || fromWrecksite(p.website1)) continue;
const origin = clean(p.source_inf, 80);
if (origin && /wrecksite/i.test(origin)) continue;
const key = clean(String(feature.id ?? p.source_id ?? ''), 80);
if (!key) continue;
const id = `emodnet:${key}`;
+12
View File
@@ -47,6 +47,18 @@ test('EMODnet wrecks keep ship facts and drop text that names people', () => {
feature('w.4', { name: 'Null island' }, [0, 0]),
],
});
const ids = normalizeEmodnetFeatures({
features: [
feature('w.3', { name: 'Copied', source_inf: 'www.wrecksite.eu' }),
feature('w.5', {
source_inf: `${'SHOM; '.repeat(20)}wrecksite.eu`,
}),
feature('w.6', { website1: 'https://www.wrecksite.eu/wreck.aspx?1' }),
feature('w.7', { website1: 'wrecksite.eu/wreck/1' }),
feature('w.8', { name: 'Kept', source_inf: 'SHOM' }),
],
}).map((record) => record.id);
assert.deepEqual(ids, ['emodnet:w.8'], 'wrecksite.eu rows are all skipped');
assert.equal(wreck.id, 'emodnet:w.1');
assert.equal(wreck.year, 1987);
assert.equal(wreck.vesselType, 'Pusher or tug');