Implement staged history loading with SSE push
Build and Push Docker Image / build (push) Has been cancelled
Licence Check / Licence compatibility and copyright header verification (push) Has been cancelled
CI / Tests & coverage (push) Has been cancelled
CI / Security audit (push) Has been cancelled

- Stage 1: Fetch 100 records immediately for fast display
- Stage 2+: Background fetch up to 1000 records in batches of 100
- Date-based cursor pagination to avoid race conditions
- Deduplication by record ID to prevent duplicates
- SSE push to clients when history cache is updated
- Shared background fetch state for concurrent user requests
This commit is contained in:
2026-05-21 01:23:11 +01:00
parent dbdfe3f329
commit 0acd452ebd
2 changed files with 291 additions and 4 deletions
+13
View File
@@ -10,6 +10,7 @@ const downloadClientRegistry = require('../utils/downloadClients');
const sanitizeError = require('../utils/sanitizeError');
const TagMatcher = require('../services/TagMatcher');
const { buildUserDownloads } = require('../services/DownloadBuilder');
const { onHistoryUpdate, offHistoryUpdate } = require('../utils/historyFetcher');
// Track active SSE clients for disconnect cleanup
@@ -202,6 +203,17 @@ router.get('/stream', requireAuth, async (req, res) => {
// Subscribe to poll-complete notifications
onPollComplete(sendDownloads);
// Subscribe to history update notifications
function sendHistoryUpdate(type) {
try {
res.write(`event: history-update\ndata: ${JSON.stringify({ type })}\n\n`);
console.log(`[SSE] Sent history update for ${type} to ${user.name}`);
} catch (err) {
console.error('[SSE] Error sending history update:', err.message);
}
}
onHistoryUpdate(sendHistoryUpdate);
// 25s heartbeat comment to keep the connection alive through proxies/load-balancers
const heartbeat = setInterval(() => {
try { res.write(': heartbeat\n\n'); } catch { /* ignore — cleanup below handles it */ }
@@ -211,6 +223,7 @@ router.get('/stream', requireAuth, async (req, res) => {
req.on('close', () => {
clearInterval(heartbeat);
offPollComplete(sendDownloads);
offHistoryUpdate(sendHistoryUpdate);
activeClients.delete(username);
console.log(`[SSE] Client disconnected: ${user.name}`);
});