perf: split into fast poll + slow-cached library fetches
Fast poll (every cycle): SABnzbd, Sonarr/Radarr queue + history, qBittorrent — all lightweight with no include* params. Slow cache (5 min TTL): Sonarr series, Radarr movies, tags — fetched only when cache expires. These rarely change. This eliminates the 2s+ includeSeries/includeMovie joins from every poll cycle. First poll is still slow (cold cache), but subsequent polls should complete in <500ms.
This commit is contained in:
@@ -140,14 +140,18 @@ router.get('/user-downloads', async (req, res) => {
|
||||
// Read all data from cache
|
||||
const sabQueueData = cache.get('poll:sab-queue') || { slots: [] };
|
||||
const sabHistoryData = cache.get('poll:sab-history') || { slots: [] };
|
||||
const sonarrTagsResults = cache.get('poll:sonarr-tags') || [];
|
||||
const sonarrQueueData = cache.get('poll:sonarr-queue') || { records: [] };
|
||||
const sonarrHistoryData = cache.get('poll:sonarr-history') || { records: [] };
|
||||
const radarrQueueData = cache.get('poll:radarr-queue') || { records: [] };
|
||||
const radarrHistoryData = cache.get('poll:radarr-history') || { records: [] };
|
||||
const radarrTagsData = cache.get('poll:radarr-tags') || [];
|
||||
const qbittorrentTorrents = cache.get('poll:qbittorrent') || [];
|
||||
|
||||
// Slow-cached data (series/movie libraries + tags, refreshed every 5 min)
|
||||
const sonarrSeriesData = cache.get('poll:sonarr-series') || [];
|
||||
const radarrMoviesData = cache.get('poll:radarr-movies') || [];
|
||||
const sonarrTagsResults = cache.get('poll:sonarr-tags') || [];
|
||||
const radarrTagsData = cache.get('poll:radarr-tags') || [];
|
||||
|
||||
// Wrap in the structure the rest of the code expects
|
||||
const sabnzbdQueue = { data: { queue: sabQueueData } };
|
||||
const sabnzbdHistory = { data: { history: sabHistoryData } };
|
||||
@@ -157,23 +161,9 @@ router.get('/user-downloads', async (req, res) => {
|
||||
const radarrHistory = { data: radarrHistoryData };
|
||||
const radarrTags = { data: radarrTagsData };
|
||||
|
||||
// Build series/movie maps from embedded objects in queue records
|
||||
// (history is fetched without includeSeries/includeMovie for speed;
|
||||
// history matches fall back to the queue-built map via seriesId/movieId)
|
||||
const seriesMap = new Map();
|
||||
for (const r of sonarrQueue.data.records) {
|
||||
if (r.series && r.seriesId) seriesMap.set(r.seriesId, r.series);
|
||||
}
|
||||
for (const r of sonarrHistory.data.records) {
|
||||
if (r.series && r.seriesId && !seriesMap.has(r.seriesId)) seriesMap.set(r.seriesId, r.series);
|
||||
}
|
||||
const moviesMap = new Map();
|
||||
for (const r of radarrQueue.data.records) {
|
||||
if (r.movie && r.movieId) moviesMap.set(r.movieId, r.movie);
|
||||
}
|
||||
for (const r of radarrHistory.data.records) {
|
||||
if (r.movie && r.movieId && !moviesMap.has(r.movieId)) moviesMap.set(r.movieId, r.movie);
|
||||
}
|
||||
// Build series/movie maps from the slow-cached full library
|
||||
const seriesMap = new Map(sonarrSeriesData.map(s => [s.id, s]));
|
||||
const moviesMap = new Map(radarrMoviesData.map(m => [m.id, m]));
|
||||
|
||||
// Create tag maps (id -> label)
|
||||
const sonarrTagMap = new Map(sonarrTagsResults.flatMap(t => t.data || []).map(t => [t.id, t.label]));
|
||||
|
||||
@@ -15,6 +15,7 @@ const POLLING_ENABLED = POLL_INTERVAL > 0;
|
||||
|
||||
let polling = false;
|
||||
let lastPollTimings = null;
|
||||
const SLOW_CACHE_TTL = 5 * 60 * 1000; // 5 minutes for series/movie library
|
||||
|
||||
// Timed fetch helper: runs a fetch and records how long it took
|
||||
async function timed(label, fn) {
|
||||
@@ -36,8 +37,59 @@ async function pollAllServices() {
|
||||
const sonarrInstances = getSonarrInstances();
|
||||
const radarrInstances = getRadarrInstances();
|
||||
|
||||
// All fetches in parallel, each individually timed
|
||||
const results = await Promise.all([
|
||||
// Slow-changing data: series/movie libraries + tags (only fetch when cache expired)
|
||||
const slowTasks = [];
|
||||
if (!cache.get('poll:sonarr-series')) {
|
||||
slowTasks.push(
|
||||
timed('Sonarr Series', () => Promise.all(sonarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/series`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Sonarr ${inst.id} series error:`, err.message);
|
||||
return { instance: inst.id, data: [] };
|
||||
})
|
||||
)))
|
||||
);
|
||||
}
|
||||
if (!cache.get('poll:radarr-movies')) {
|
||||
slowTasks.push(
|
||||
timed('Radarr Movies', () => Promise.all(radarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/movie`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Radarr ${inst.id} movies error:`, err.message);
|
||||
return { instance: inst.id, data: [] };
|
||||
})
|
||||
)))
|
||||
);
|
||||
}
|
||||
if (!cache.get('poll:sonarr-tags')) {
|
||||
slowTasks.push(
|
||||
timed('Sonarr Tags', () => Promise.all(sonarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/tag`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Sonarr ${inst.id} tags error:`, err.message);
|
||||
return { instance: inst.id, data: [] };
|
||||
})
|
||||
)))
|
||||
);
|
||||
}
|
||||
if (!cache.get('poll:radarr-tags')) {
|
||||
slowTasks.push(
|
||||
timed('Radarr Tags', () => Promise.all(radarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/tag`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Radarr ${inst.id} tags error:`, err.message);
|
||||
return { instance: inst.id, data: [] };
|
||||
})
|
||||
)))
|
||||
);
|
||||
}
|
||||
|
||||
// Fast-changing data: queues, history, torrents (every poll)
|
||||
const fastTasks = [
|
||||
timed('SABnzbd Queue', () => Promise.all(sabInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api`, {
|
||||
params: { mode: 'queue', apikey: inst.apiKey, output: 'json' }
|
||||
@@ -54,18 +106,9 @@ async function pollAllServices() {
|
||||
return { instance: inst.id, data: { history: { slots: [] } } };
|
||||
})
|
||||
))),
|
||||
timed('Sonarr Tags', () => Promise.all(sonarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/tag`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Sonarr ${inst.id} tags error:`, err.message);
|
||||
return { instance: inst.id, data: [] };
|
||||
})
|
||||
))),
|
||||
timed('Sonarr Queue', () => Promise.all(sonarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/queue`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey },
|
||||
params: { includeSeries: true }
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Sonarr ${inst.id} queue error:`, err.message);
|
||||
return { instance: inst.id, data: { records: [] } };
|
||||
@@ -82,8 +125,7 @@ async function pollAllServices() {
|
||||
))),
|
||||
timed('Radarr Queue', () => Promise.all(radarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/queue`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey },
|
||||
params: { includeMovie: true }
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Radarr ${inst.id} queue error:`, err.message);
|
||||
return { instance: inst.id, data: { records: [] } };
|
||||
@@ -98,41 +140,55 @@ async function pollAllServices() {
|
||||
return { instance: inst.id, data: { records: [] } };
|
||||
})
|
||||
))),
|
||||
timed('Radarr Tags', () => Promise.all(radarrInstances.map(inst =>
|
||||
axios.get(`${inst.url}/api/v3/tag`, {
|
||||
headers: { 'X-Api-Key': inst.apiKey }
|
||||
}).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
|
||||
console.error(`[Poller] Radarr ${inst.id} tags error:`, err.message);
|
||||
return { instance: inst.id, data: [] };
|
||||
})
|
||||
))),
|
||||
timed('qBittorrent', () => getTorrents().catch(err => {
|
||||
console.error(`[Poller] qBittorrent error:`, err.message);
|
||||
return [];
|
||||
}))
|
||||
]);
|
||||
];
|
||||
|
||||
// Run slow + fast in parallel
|
||||
const allResults = await Promise.all([...slowTasks, ...fastTasks]);
|
||||
const slowResults = allResults.slice(0, slowTasks.length);
|
||||
const fastResults = allResults.slice(slowTasks.length);
|
||||
|
||||
const [
|
||||
{ result: sabQueues }, { result: sabHistories },
|
||||
{ result: sonarrTagsResults }, { result: sonarrQueues },
|
||||
{ result: sonarrHistories },
|
||||
{ result: sonarrQueues }, { result: sonarrHistories },
|
||||
{ result: radarrQueues }, { result: radarrHistories },
|
||||
{ result: radarrTagsResults },
|
||||
{ result: qbittorrentTorrents }
|
||||
] = results;
|
||||
] = fastResults;
|
||||
|
||||
// Store per-task timings
|
||||
const totalMs = Date.now() - start;
|
||||
lastPollTimings = {
|
||||
totalMs,
|
||||
timestamp: new Date().toISOString(),
|
||||
tasks: results.map(r => ({ label: r.label, ms: r.ms }))
|
||||
tasks: allResults.map(r => ({ label: r.label, ms: r.ms }))
|
||||
};
|
||||
|
||||
// When polling is active, TTL is 3x interval to avoid gaps between polls
|
||||
// When polling is disabled (on-demand), use 30s so data refreshes on next request after expiry
|
||||
const cacheTTL = POLLING_ENABLED ? POLL_INTERVAL * 3 : 30000;
|
||||
|
||||
// Store slow-changing data (only if fetched this cycle)
|
||||
for (const sr of slowResults) {
|
||||
if (sr.label === 'Sonarr Series') {
|
||||
cache.set('poll:sonarr-series', sr.result.flatMap(s => {
|
||||
const inst = sonarrInstances.find(i => i.id === s.instance);
|
||||
return (s.data || []).map(item => ({ ...item, _instanceUrl: inst ? inst.url : null }));
|
||||
}), SLOW_CACHE_TTL);
|
||||
} else if (sr.label === 'Radarr Movies') {
|
||||
cache.set('poll:radarr-movies', sr.result.flatMap(m => {
|
||||
const inst = radarrInstances.find(i => i.id === m.instance);
|
||||
return (m.data || []).map(item => ({ ...item, _instanceUrl: inst ? inst.url : null }));
|
||||
}), SLOW_CACHE_TTL);
|
||||
} else if (sr.label === 'Sonarr Tags') {
|
||||
cache.set('poll:sonarr-tags', sr.result, SLOW_CACHE_TTL);
|
||||
} else if (sr.label === 'Radarr Tags') {
|
||||
cache.set('poll:radarr-tags', sr.result.flatMap(t => t.data || []), SLOW_CACHE_TTL);
|
||||
}
|
||||
}
|
||||
|
||||
// SABnzbd
|
||||
const firstSabQueue = sabQueues[0] && sabQueues[0].data && sabQueues[0].data.queue;
|
||||
cache.set('poll:sab-queue', {
|
||||
@@ -146,38 +202,21 @@ async function pollAllServices() {
|
||||
slots: sabHistories.flatMap(h => (h.data.history && h.data.history.slots) || [])
|
||||
}, cacheTTL);
|
||||
|
||||
// Sonarr
|
||||
cache.set('poll:sonarr-tags', sonarrTagsResults, cacheTTL);
|
||||
// Tag queue/history records with _instanceUrl so embedded series/movie objects can build links
|
||||
// Sonarr (lightweight — no embedded series/movie objects)
|
||||
cache.set('poll:sonarr-queue', {
|
||||
records: sonarrQueues.flatMap(q => {
|
||||
const inst = sonarrInstances.find(i => i.id === q.instance);
|
||||
const url = inst ? inst.url : null;
|
||||
return (q.data.records || []).map(r => {
|
||||
if (r.series) r.series._instanceUrl = url;
|
||||
return r;
|
||||
});
|
||||
})
|
||||
records: sonarrQueues.flatMap(q => q.data.records || [])
|
||||
}, cacheTTL);
|
||||
cache.set('poll:sonarr-history', {
|
||||
records: sonarrHistories.flatMap(h => h.data.records || [])
|
||||
}, cacheTTL);
|
||||
|
||||
// Radarr
|
||||
// Radarr (lightweight — no embedded series/movie objects)
|
||||
cache.set('poll:radarr-queue', {
|
||||
records: radarrQueues.flatMap(q => {
|
||||
const inst = radarrInstances.find(i => i.id === q.instance);
|
||||
const url = inst ? inst.url : null;
|
||||
return (q.data.records || []).map(r => {
|
||||
if (r.movie) r.movie._instanceUrl = url;
|
||||
return r;
|
||||
});
|
||||
})
|
||||
records: radarrQueues.flatMap(q => q.data.records || [])
|
||||
}, cacheTTL);
|
||||
cache.set('poll:radarr-history', {
|
||||
records: radarrHistories.flatMap(h => h.data.records || [])
|
||||
}, cacheTTL);
|
||||
cache.set('poll:radarr-tags', radarrTagsResults.flatMap(t => t.data || []), cacheTTL);
|
||||
|
||||
// qBittorrent
|
||||
cache.set('poll:qbittorrent', qbittorrentTorrents, cacheTTL);
|
||||
|
||||
Reference in New Issue
Block a user