mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-22 11:12:18 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
75fe5b5703 | ||
|
|
b7552fbc64 |
No files matched your search
@@ -0,0 +1,3 @@
|
||||
- Shorten the hitch when the Radio directory loads from about 450 ms to about
|
||||
180 ms. A station's tags and categories are worked out once instead of for
|
||||
every category check (mazy1998, #1709).
|
||||
@@ -1,2 +0,0 @@
|
||||
- `/api/firms` keeps its response bytes and sends them gzipped instead of
|
||||
serialising the whole fire set for every request (mazy1998, #1711).
|
||||
@@ -3297,15 +3297,6 @@ The `get_active_fires` tool reports acquisition times as UTC ISO timestamps,
|
||||
including zero-padded hours and minutes; missing or invalid dates and times
|
||||
remain `null`, including impossible calendar dates from a custom source.
|
||||
|
||||
`/api/firms` keeps its last response body as bytes and reuses it for requests
|
||||
that would get the same payload: the same cache entry and `stale` flag, while
|
||||
the clock stays inside the span over which the trailing-24-hour filter keeps
|
||||
the same detections. That span ends when the oldest kept acquisition minute
|
||||
turns 24 hours old, so requests minutes apart are still serialised afresh. A
|
||||
client that sends `Accept-Encoding: gzip` gets the body gzipped (compressed
|
||||
once per kept body, `Content-Encoding: gzip`); every response carries
|
||||
`Vary: Accept-Encoding`. The JSON is unchanged.
|
||||
|
||||
The FIRMS proxy records source success after appending its rows. If aggregation
|
||||
throws, that source reports failure without a contradictory success entry.
|
||||
The existing per-record append continues to support large feeds; sequential
|
||||
|
||||
@@ -376,7 +376,6 @@
|
||||
"modules": [
|
||||
"server/providers/firms.js",
|
||||
"server/providers/common/http.js",
|
||||
"server/providers/common/compress.js",
|
||||
"src/sources/httpBody.js",
|
||||
"src/i18n/marker.js",
|
||||
"src/data/firmsCsv.js"
|
||||
|
||||
+10
-58
@@ -3,12 +3,7 @@ import https from 'node:https';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import {
|
||||
filterTrailing24h,
|
||||
parseFirmsCsv,
|
||||
trailing24hWindow,
|
||||
} from '../../src/data/firmsCsv.js';
|
||||
import { acceptsGzip, encodedBody, gzipBody } from './common/compress.js';
|
||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
|
||||
@@ -184,11 +179,6 @@ export function firmsProxy() {
|
||||
let statusCache = null;
|
||||
/** @type {?Promise<?{used: number, limit: number}>} */
|
||||
let statusInflight = null;
|
||||
/**
|
||||
* The last /api/firms payload as bytes, with its gzip. One slot.
|
||||
* @type {?{entry: object, stale: boolean, from: number, until: number, body: Buffer, gzip: ?Promise<?Buffer>}}
|
||||
*/
|
||||
let served = null;
|
||||
|
||||
const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim();
|
||||
|
||||
@@ -269,8 +259,12 @@ export function firmsProxy() {
|
||||
return { at: now, sources, fires };
|
||||
}
|
||||
|
||||
/** Cache entry and the fires to serve from it → response payload. */
|
||||
function buildPayload(entry, stale, fires) {
|
||||
/**
|
||||
* Cache entry → response payload. Fires are RE-filtered to the trailing
|
||||
* 24 h at serve time so a stale cache never serves >24h-old detections.
|
||||
*/
|
||||
function buildPayload(entry, stale) {
|
||||
const fires = filterTrailing24h(entry.fires, Date.now());
|
||||
return {
|
||||
fetchedAt: entry.at,
|
||||
stale,
|
||||
@@ -281,32 +275,6 @@ export function firmsProxy() {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The payload as the bytes to send. Serialising the whole fire set is tens
|
||||
* of MB in a heavy season, so it is done once and kept, not once per request.
|
||||
*
|
||||
* Fires are RE-filtered to the trailing 24 h at serve time so a stale cache
|
||||
* never serves >24h-old detections. The kept bytes are therefore reused only
|
||||
* for the same entry and stale flag while the clock is inside the span over
|
||||
* which that filter keeps the same records: a request never gets a body that
|
||||
* serialising for it would not have produced.
|
||||
*/
|
||||
function payloadBody(entry, stale) {
|
||||
const now = Date.now();
|
||||
if (
|
||||
served?.entry === entry &&
|
||||
served.stale === stale &&
|
||||
now >= served.from &&
|
||||
now <= served.until
|
||||
) {
|
||||
return served;
|
||||
}
|
||||
const { kept, from, until } = trailing24hWindow(entry.fires, now);
|
||||
const body = Buffer.from(JSON.stringify(buildPayload(entry, stale, kept)));
|
||||
served = { entry, stale, from, until, body, gzip: null };
|
||||
return served;
|
||||
}
|
||||
|
||||
/** mapkey_status transactions, cached 5 min, best-effort (null on failure). */
|
||||
function getTransactions(key) {
|
||||
const now = Date.now();
|
||||
@@ -359,22 +327,6 @@ export function firmsProxy() {
|
||||
});
|
||||
res.end(JSON.stringify(obj));
|
||||
};
|
||||
const sendPayload = async (entry, stale) => {
|
||||
const kept = payloadBody(entry, stale);
|
||||
// Compressed once per kept body, on the first request that accepts
|
||||
// gzip, so an identity response never pays or waits for it.
|
||||
const gzip = acceptsGzip(req)
|
||||
? await (kept.gzip ??= gzipBody(kept.body).catch(() => null))
|
||||
: null;
|
||||
const encoded = encodedBody(req, kept.body, gzip);
|
||||
if (res.headersSent) return;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...encoded.headers,
|
||||
});
|
||||
res.end(encoded.body);
|
||||
};
|
||||
try {
|
||||
const subPath = String(req.url || '').split('?')[0];
|
||||
const key = mapKey();
|
||||
@@ -411,7 +363,7 @@ export function firmsProxy() {
|
||||
|
||||
const entry = mem;
|
||||
if (entry && Date.now() - entry.at < TTL_MS) {
|
||||
await sendPayload(entry, false);
|
||||
sendJson(200, buildPayload(entry, false));
|
||||
return;
|
||||
}
|
||||
// Stale or missing → refresh, single-flight (concurrent requests
|
||||
@@ -437,9 +389,9 @@ export function firmsProxy() {
|
||||
const pending = inflight;
|
||||
const fresh = await pending;
|
||||
if (fresh) {
|
||||
await sendPayload(fresh, false);
|
||||
sendJson(200, buildPayload(fresh, false));
|
||||
} else if (entry) {
|
||||
await sendPayload(entry, true); // upstream down — stale beats empty
|
||||
sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty
|
||||
} else {
|
||||
sendJson(502, {
|
||||
error: 'firms fetch failed and no cache available',
|
||||
|
||||
+2
-27
@@ -182,44 +182,19 @@ export function acquisitionMsUtc(acqDate, acqTime) {
|
||||
* @returns {Array<Object>} Filtered records (original objects, order preserved).
|
||||
*/
|
||||
export function filterTrailing24h(records, nowMs) {
|
||||
return trailing24hWindow(records, nowMs).kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link filterTrailing24h} with the span of `nowMs` values, inclusive, over
|
||||
* which it keeps exactly these records. A caller holding something built from
|
||||
* `kept` can reuse it while its clock stays inside `from..until`.
|
||||
* @param {Array<{acqDate: string, acqTime: string|number}>} records - Parser records.
|
||||
* @param {number} nowMs - Reference epoch milliseconds.
|
||||
* @returns {{kept: Array<Object>, from: number, until: number}} Span in epoch
|
||||
* ms; NaN (never reusable) for unusable input.
|
||||
*/
|
||||
export function trailing24hWindow(records, nowMs) {
|
||||
if (!Array.isArray(records) || !Number.isFinite(nowMs))
|
||||
return { kept: [], from: NaN, until: NaN };
|
||||
if (!Array.isArray(records) || !Number.isFinite(nowMs)) return [];
|
||||
const oldest = nowMs - WINDOW_MS;
|
||||
const newest = nowMs + FORWARD_SLACK_MS;
|
||||
let from = -Infinity;
|
||||
let until = Infinity;
|
||||
const memo = new Map();
|
||||
const kept = records.filter((record) => {
|
||||
return records.filter((record) => {
|
||||
const key = `${record.acqDate}:${record.acqTime}`;
|
||||
let ms = memo.get(key);
|
||||
if (ms === undefined) {
|
||||
ms = acquisitionMsUtc(record.acqDate, record.acqTime);
|
||||
memo.set(key, ms);
|
||||
// Each distinct time narrows the span to the clock values that leave
|
||||
// it on the same side of the window.
|
||||
if (ms < oldest) from = Math.max(from, ms + WINDOW_MS + 1);
|
||||
else if (ms > newest) until = Math.min(until, ms - FORWARD_SLACK_MS - 1);
|
||||
else if (Number.isFinite(ms)) {
|
||||
from = Math.max(from, ms - FORWARD_SLACK_MS);
|
||||
until = Math.min(until, ms + WINDOW_MS);
|
||||
}
|
||||
}
|
||||
return Number.isFinite(ms) && ms >= oldest && ms <= newest;
|
||||
});
|
||||
return { kept, from, until };
|
||||
}
|
||||
|
||||
/** Trimmed string cell at index, '' for missing columns. */
|
||||
|
||||
@@ -7,15 +7,11 @@ import dnsLookup from 'node:dns';
|
||||
import https from 'node:https';
|
||||
import net from 'node:net';
|
||||
import { Readable } from 'node:stream';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
import { filterTrailing24h, parseFirmsCsv, trailing24hWindow } from './firmsCsv.js';
|
||||
import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js';
|
||||
import {
|
||||
describeFetchError,
|
||||
fetchFirms,
|
||||
fetchIpv4,
|
||||
firmsProxy,
|
||||
isAddressSelectionFailure,
|
||||
} from '../../server/providers/firms.js';
|
||||
|
||||
@@ -320,119 +316,3 @@ test('FIRMS requests only the satellites NASA still delivers', () => {
|
||||
// a MAP_KEY transaction.
|
||||
assert.ok(!SOURCES.includes('VIIRS_SNPP_NRT'));
|
||||
});
|
||||
|
||||
// The /api/firms handler itself, over a disk cache seeded in a temp directory:
|
||||
// no server, no upstream, and a placeholder MAP_KEY that is never sent anywhere.
|
||||
const TTL_MS = 30 * 60_000;
|
||||
const fire = (acqDate, acqTime, lat = 1) => ({ lat, lon: 2, frp: 4.5, confidence: 'n', brightness: 330.1,
|
||||
brightnessTi5: 290.2, daynight: 'D', acqDate, acqTime, satellite: 'N20', instrument: 'VIIRS' });
|
||||
const SERVED_SOURCES = SOURCES.map((source, index) => ({ source, count: index === 0 ? 20_001 : 0, ok: true }));
|
||||
// 20,000 detections an hour old and one that turns 24 h old five minutes after NOW.
|
||||
const SERVED_FIRES = [fire('2026-09-10', '1205'), ...Array.from({ length: 20_000 }, (_, i) => fire('2026-09-11', '1100', i / 1000))];
|
||||
const LEAVES_AT = NOW + 5 * 60_000;
|
||||
/** What the handler sent before it kept a body: one JSON.stringify of the payload per request. */
|
||||
const unfixedBody = (at, stale, now) => {
|
||||
const fires = filterTrailing24h(SERVED_FIRES, now);
|
||||
return JSON.stringify({ fetchedAt: at, stale, ttlMs: TTL_MS, sources: SERVED_SOURCES, count: fires.length, fires });
|
||||
};
|
||||
|
||||
function createFirmsHandler(t, at) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'firms-proxy-'));
|
||||
fs.mkdirSync(path.join(dir, '.gev-cache'));
|
||||
fs.writeFileSync(path.join(dir, '.gev-cache', 'firms.json'), JSON.stringify({ at, sources: SERVED_SOURCES, fires: SERVED_FIRES }));
|
||||
const cwd = process.cwd();
|
||||
const key = process.env.FIRMS_MAP_KEY;
|
||||
const realFetch = globalThis.fetch;
|
||||
process.env.FIRMS_MAP_KEY = 'unit-test-placeholder';
|
||||
// A refresh must never reach NASA from a test: every upstream call fails.
|
||||
globalThis.fetch = async () => { throw new Error('upstream down'); };
|
||||
t.mock.method(console, 'warn', () => {});
|
||||
t.mock.timers.enable({ apis: ['Date'], now: NOW });
|
||||
t.after(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
if (key === undefined) delete process.env.FIRMS_MAP_KEY; else process.env.FIRMS_MAP_KEY = key;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
let handler;
|
||||
process.chdir(dir); // the proxy resolves its cache directory when created
|
||||
try {
|
||||
firmsProxy().configureServer({ middlewares: { use: (route, fn) => { handler = fn; } } });
|
||||
} finally {
|
||||
process.chdir(cwd);
|
||||
}
|
||||
return (acceptEncoding) => new Promise((resolve, reject) => {
|
||||
const res = {
|
||||
headersSent: false,
|
||||
writeHead(status, headers) { this.headersSent = true; this.status = status; this.headers = headers; },
|
||||
end(body) { resolve({ status: this.status, headers: this.headers, body }); },
|
||||
};
|
||||
handler({ url: '/', headers: acceptEncoding === undefined ? {} : { 'accept-encoding': acceptEncoding } }, res).catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
test('/api/firms serialises once for requests that share a payload', async (t) => {
|
||||
const request = createFirmsHandler(t, NOW);
|
||||
const first = await request();
|
||||
t.mock.timers.setTime(LEAVES_AT); // exactly 24 h old is still inside the window
|
||||
const second = await request();
|
||||
assert.equal(first.status, 200);
|
||||
assert.ok(Buffer.isBuffer(first.body), 'the body is kept as bytes');
|
||||
assert.equal(second.body, first.body, 'the second request sends the kept bytes, not a new serialisation');
|
||||
assert.equal(first.body.toString(), unfixedBody(NOW, false, NOW), 'byte-identical to the per-request serialisation');
|
||||
assert.equal(JSON.parse(first.body).count, 20_001);
|
||||
});
|
||||
|
||||
test('/api/firms sends gzip only to a client that accepts it', async (t) => {
|
||||
const request = createFirmsHandler(t, NOW);
|
||||
const identity = await request();
|
||||
const gzipped = await request('gzip, deflate, br');
|
||||
const refused = await request('gzip;q=0, br');
|
||||
assert.deepEqual(identity.headers,
|
||||
{ 'Content-Type': 'application/json', 'Cache-Control': 'no-store', Vary: 'Accept-Encoding' });
|
||||
assert.deepEqual(gzipped.headers, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store',
|
||||
'Content-Encoding': 'gzip', Vary: 'Accept-Encoding' });
|
||||
assert.deepEqual(refused.headers, identity.headers);
|
||||
assert.equal(refused.body, identity.body);
|
||||
assert.ok(gzipped.body.byteLength < identity.body.byteLength / 4, 'the fire set compresses');
|
||||
assert.equal(gunzipSync(gzipped.body).toString(), unfixedBody(NOW, false, NOW), 'decodes to the identity bytes');
|
||||
assert.equal((await request('gzip')).body, gzipped.body, 'compressed once');
|
||||
});
|
||||
|
||||
test('/api/firms never reuses a body the request would not have produced', async (t) => {
|
||||
const request = createFirmsHandler(t, NOW - TTL_MS + 6 * 60_000);
|
||||
const at = NOW - TTL_MS + 6 * 60_000;
|
||||
const before = await request();
|
||||
t.mock.timers.setTime(LEAVES_AT + 1); // one detection is now older than 24 h
|
||||
const after = await request('gzip');
|
||||
assert.equal(before.body.toString(), unfixedBody(at, false, NOW));
|
||||
assert.equal(gunzipSync(after.body).toString(), unfixedBody(at, false, LEAVES_AT + 1));
|
||||
assert.equal(JSON.parse(gunzipSync(after.body)).count, 20_000);
|
||||
t.mock.timers.setTime(NOW + 6 * 60_000); // the entry reaches its TTL and the refresh fails
|
||||
const stale = await request();
|
||||
assert.equal(stale.body.toString(), unfixedBody(at, true, NOW + 6 * 60_000));
|
||||
assert.equal((await request()).body, stale.body, 'the stale body is kept too');
|
||||
});
|
||||
|
||||
test('trailing24hWindow spans the clock values that keep the same records', () => {
|
||||
const records = [fire('2026-09-10', '1205'), fire('2026-09-10', '1300'), fire('2026-09-11', '1100'),
|
||||
fire('2026-09-11', '1430'), fire('2026-09-11', '1600'), fire('bad-date', '1006')];
|
||||
const edges = records.map((r) => Date.parse(`${r.acqDate}T${r.acqTime.slice(0, 2)}:${r.acqTime.slice(2)}Z`))
|
||||
.filter(Number.isFinite).flatMap((ms) => [ms - 2 * 3600_000, ms + 24 * 3600_000]);
|
||||
assert.equal(edges.length, 10);
|
||||
const clocks = [NOW, ...edges.flatMap((ms) => [ms - 1, ms, ms + 1])];
|
||||
for (const now of clocks) {
|
||||
const { kept, from, until } = trailing24hWindow(records, now);
|
||||
assert.deepEqual(kept, filterTrailing24h(records, now));
|
||||
assert.ok(from <= now && now <= until, 'the span holds its own clock');
|
||||
for (const other of clocks) {
|
||||
const then = filterTrailing24h(records, other);
|
||||
const same = then.length === kept.length && then.every((r, i) => r === kept[i]);
|
||||
// Before the first record and after the last both keep nothing; the span covers one side.
|
||||
const bothEmpty = then.length === 0 && kept.length === 0;
|
||||
if (other >= from && other <= until) assert.ok(same, `${other} is inside the span of ${now}`);
|
||||
else assert.ok(!same || bothEmpty, `${other} is outside the span of ${now}`);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(trailing24hWindow([], NOW), { kept: [], from: -Infinity, until: Infinity });
|
||||
assert.deepEqual(trailing24hWindow(records, NaN), { kept: [], from: NaN, until: NaN });
|
||||
});
|
||||
@@ -27,42 +27,52 @@ export function createCategories({
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function hasTag(station, needles) {
|
||||
const tags = stationTags(station);
|
||||
return needles.some((needle) =>
|
||||
const tagsInclude = (tags, needles) =>
|
||||
needles.some((needle) =>
|
||||
tags.some((tag) => tag === needle || tag.includes(needle)),
|
||||
);
|
||||
|
||||
// A station's tags decide its categories, and neither changes while the
|
||||
// station record lives. Matching used to normalise every tag again for
|
||||
// every needle of every category of every station: about 0.9 s and a
|
||||
// 445 ms freeze per directory load of 750 stations, repeated for the active
|
||||
// filter on every horizon scan. Keyed by the record and its tags value, so
|
||||
// a record given new tags is read again.
|
||||
const membershipByStation = new WeakMap();
|
||||
|
||||
function stationMembership(station) {
|
||||
const keyed = station !== null && typeof station === 'object';
|
||||
const known = keyed ? membershipByStation.get(station) : null;
|
||||
if (known && known.source === station.tags) return known;
|
||||
const tags = stationTags(station);
|
||||
const genres = MUSIC_GENRES.filter(([genre]) =>
|
||||
tagsInclude(tags, [genre]),
|
||||
).map(([genre]) => genre);
|
||||
const categories = new Set(genres.map((genre) => `genre:${genre}`));
|
||||
for (const [id, needles] of Object.entries(CATEGORY_MATCHERS))
|
||||
if (tagsInclude(tags, needles)) categories.add(id);
|
||||
const sorted = categories.size > genres.length;
|
||||
if (genres.length > 0 || tagsInclude(tags, ['music', 'hits', 'songs']))
|
||||
categories.add('music');
|
||||
else if (!sorted) categories.add('other');
|
||||
const membership = { source: station?.tags, tags, genres, categories };
|
||||
if (keyed) membershipByStation.set(station, membership);
|
||||
return membership;
|
||||
}
|
||||
|
||||
function hasTag(station, needles) {
|
||||
return tagsInclude(stationMembership(station).tags, needles);
|
||||
}
|
||||
|
||||
function detectedGenres(station) {
|
||||
return MUSIC_GENRES.filter(([genre]) => hasTag(station, [genre])).map(
|
||||
([genre]) => genre,
|
||||
);
|
||||
return [...stationMembership(station).genres];
|
||||
}
|
||||
|
||||
/** Return whether a station belongs in a station-tag category. */
|
||||
|
||||
function stationMatchesRadioCategory(station, categoryId) {
|
||||
if (categoryId === 'all') return true;
|
||||
if (categoryId.startsWith('genre:')) {
|
||||
return detectedGenres(station).includes(
|
||||
categoryId.slice('genre:'.length),
|
||||
);
|
||||
}
|
||||
if (categoryId === 'music') {
|
||||
return (
|
||||
detectedGenres(station).length > 0 ||
|
||||
hasTag(station, ['music', 'hits', 'songs'])
|
||||
);
|
||||
}
|
||||
if (categoryId === 'other') {
|
||||
return (
|
||||
!Object.entries(CATEGORY_MATCHERS).some(([id]) =>
|
||||
stationMatchesRadioCategory(station, id),
|
||||
) && !stationMatchesRadioCategory(station, 'music')
|
||||
);
|
||||
}
|
||||
return hasTag(station, CATEGORY_MATCHERS[categoryId] || []);
|
||||
return stationMembership(station).categories.has(categoryId);
|
||||
}
|
||||
|
||||
/** Build canonical and detected-genre categories from station-level tags. */
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createCategories } from './categories.js';
|
||||
import { CATEGORY_MATCHERS, MUSIC_GENRES } from './policy.js';
|
||||
|
||||
const categories = createCategories({
|
||||
state: {},
|
||||
services: {},
|
||||
parts: { model: { radioCategoryColor: (id) => `color:${id}` } },
|
||||
source: {},
|
||||
});
|
||||
|
||||
// Category matching as it was before memberships were remembered: every call
|
||||
// normalises the station's tags again. Counts and filters must stay equal.
|
||||
const normalize = (value) =>
|
||||
String(value ?? '')
|
||||
.trim()
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[_-]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.slice(0, 80);
|
||||
const tagsOf = (station) =>
|
||||
(Array.isArray(station?.tags)
|
||||
? station.tags
|
||||
: String(station?.tags ?? '').split(',')
|
||||
)
|
||||
.map(normalize)
|
||||
.filter(Boolean);
|
||||
const hasTag = (station, needles) => {
|
||||
const tags = tagsOf(station);
|
||||
return needles.some((needle) =>
|
||||
tags.some((tag) => tag === needle || tag.includes(needle)),
|
||||
);
|
||||
};
|
||||
const genresOf = (station) =>
|
||||
MUSIC_GENRES.filter(([genre]) => hasTag(station, [genre])).map(
|
||||
([genre]) => genre,
|
||||
);
|
||||
function matchesByRescan(station, categoryId) {
|
||||
if (categoryId === 'all') return true;
|
||||
if (categoryId.startsWith('genre:'))
|
||||
return genresOf(station).includes(categoryId.slice('genre:'.length));
|
||||
if (categoryId === 'music')
|
||||
return (
|
||||
genresOf(station).length > 0 ||
|
||||
hasTag(station, ['music', 'hits', 'songs'])
|
||||
);
|
||||
if (categoryId === 'other')
|
||||
return (
|
||||
!Object.keys(CATEGORY_MATCHERS).some((id) =>
|
||||
matchesByRescan(station, id),
|
||||
) && !matchesByRescan(station, 'music')
|
||||
);
|
||||
return hasTag(station, CATEGORY_MATCHERS[categoryId] || []);
|
||||
}
|
||||
|
||||
const CATEGORY_IDS = [
|
||||
'all',
|
||||
...Object.keys(CATEGORY_MATCHERS),
|
||||
'music',
|
||||
...MUSIC_GENRES.map(([genre]) => `genre:${genre}`),
|
||||
'other',
|
||||
'genre:not-a-genre',
|
||||
'not-a-category',
|
||||
];
|
||||
|
||||
function seeded(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
||||
return state / 2 ** 32;
|
||||
};
|
||||
}
|
||||
|
||||
/** Stations tagged the way the directory tags them, plus the odd shapes. */
|
||||
function directory(count = 1500) {
|
||||
const random = seeded(11);
|
||||
const vocabulary = [
|
||||
...MUSIC_GENRES.map(([genre]) => genre),
|
||||
...Object.values(CATEGORY_MATCHERS).flat(),
|
||||
'Hip-Hop',
|
||||
'R&B',
|
||||
'Top_Hits',
|
||||
'songs of praise',
|
||||
'community',
|
||||
'college',
|
||||
'varied',
|
||||
'Police Scanner',
|
||||
'FIREFIGHTERS',
|
||||
'metropolitan',
|
||||
'x'.repeat(100),
|
||||
'',
|
||||
' ',
|
||||
];
|
||||
const pick = () => vocabulary[Math.floor(random() * vocabulary.length)];
|
||||
const stations = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const tags = Array.from({ length: Math.floor(random() * 6) }, pick);
|
||||
stations.push({
|
||||
id: `station-${i}`,
|
||||
tags: i % 5 === 0 ? tags.join(',') : i % 7 === 0 ? Object.freeze(tags) : tags,
|
||||
});
|
||||
}
|
||||
stations.push(
|
||||
{ id: 'no-tags' },
|
||||
{ id: 'null-tags', tags: null },
|
||||
{ id: 'number-tags', tags: 42 },
|
||||
Object.freeze({ id: 'frozen', tags: Object.freeze(['Jazz', 'News']) }),
|
||||
null,
|
||||
undefined,
|
||||
'pop',
|
||||
);
|
||||
return stations;
|
||||
}
|
||||
|
||||
test('remembered memberships give the counts and filters of a full rescan', () => {
|
||||
const stations = directory();
|
||||
for (const id of CATEGORY_IDS) {
|
||||
const expected = stations.filter((station) => matchesByRescan(station, id));
|
||||
assert.deepEqual(
|
||||
categories.filterRadioStations(stations, id),
|
||||
expected,
|
||||
`filter ${id}`,
|
||||
);
|
||||
for (const station of stations)
|
||||
assert.equal(
|
||||
categories.stationMatchesRadioCategory(station, id),
|
||||
matchesByRescan(station, id),
|
||||
);
|
||||
}
|
||||
const built = categories.buildRadioCategories(stations);
|
||||
assert.deepEqual(
|
||||
built.map(({ id, count }) => [id, count]),
|
||||
CATEGORY_IDS.filter(
|
||||
(id) =>
|
||||
!id.startsWith('genre:') ||
|
||||
stations.some((station) => matchesByRescan(station, id)),
|
||||
)
|
||||
.filter((id) => id !== 'not-a-category')
|
||||
.map((id) => [
|
||||
id,
|
||||
stations.filter((station) => matchesByRescan(station, id)).length,
|
||||
]),
|
||||
);
|
||||
assert.ok(built.find(({ id }) => id === 'other').count > 0);
|
||||
assert.ok(built.find(({ id }) => id === 'genre:hip hop').count > 0);
|
||||
for (const station of stations) {
|
||||
assert.deepEqual(categories.detectedGenres(station), genresOf(station));
|
||||
assert.deepEqual(categories.stationTags(station), tagsOf(station));
|
||||
}
|
||||
});
|
||||
|
||||
test('a station has its tags normalised once, however often it is matched', () => {
|
||||
let reads = 0;
|
||||
const tag = (text) => ({
|
||||
toString() {
|
||||
reads++;
|
||||
return text;
|
||||
},
|
||||
});
|
||||
const stations = [
|
||||
{ id: 'a', tags: [tag('Jazz'), tag('News')] },
|
||||
{ id: 'b', tags: [tag('community')] },
|
||||
];
|
||||
categories.buildRadioCategories(stations);
|
||||
for (const id of CATEGORY_IDS) categories.filterRadioStations(stations, id);
|
||||
// A horizon scan asks once per station for the active filter.
|
||||
for (let scan = 0; scan < 50; scan++)
|
||||
for (const station of stations)
|
||||
categories.stationMatchesRadioCategory(station, 'news');
|
||||
assert.equal(reads, 3, 'one normalisation per tag');
|
||||
assert.deepEqual(
|
||||
categories.filterRadioStations(stations, 'genre:jazz').map(({ id }) => id),
|
||||
['a'],
|
||||
);
|
||||
});
|
||||
|
||||
test('a station record given new tags is matched on the new tags', () => {
|
||||
const station = { id: 'a', tags: ['jazz'] };
|
||||
assert.equal(categories.stationMatchesRadioCategory(station, 'music'), true);
|
||||
assert.equal(categories.stationMatchesRadioCategory(station, 'news'), false);
|
||||
station.tags = ['news'];
|
||||
assert.equal(categories.stationMatchesRadioCategory(station, 'music'), false);
|
||||
assert.equal(categories.stationMatchesRadioCategory(station, 'news'), true);
|
||||
assert.deepEqual(categories.detectedGenres(station), []);
|
||||
station.tags = 'community';
|
||||
assert.equal(categories.stationMatchesRadioCategory(station, 'other'), true);
|
||||
});
|
||||
|
||||
test('callers cannot alter a remembered genre list', () => {
|
||||
const station = { id: 'a', tags: ['jazz', 'soul'] };
|
||||
categories.detectedGenres(station).push('rock');
|
||||
assert.deepEqual(categories.detectedGenres(station), ['jazz', 'soul']);
|
||||
assert.equal(
|
||||
categories.stationMatchesRadioCategory(station, 'genre:rock'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user