perf: background poller for near-instant dashboard responses

- New poller.js polls all services on a configurable interval
- POLL_INTERVAL env var (default 5000ms / 5 seconds)
- All data stored in cache with TTL of 3x poll interval
- Dashboard endpoint now reads from cache only (no network calls)
- API responses are near-instant regardless of service count
- First poll runs immediately on server start

server/index.js

@@ -54,6 +54,7 @@ const radarrRoutes = require('./routes/radarr');
 const embyRoutes = require('./routes/emby');
 const dashboardRoutes = require('./routes/dashboard');
 const authRoutes = require('./routes/auth');
+const { startPoller, POLL_INTERVAL } = require('./utils/poller');
 
 const app = express();
 const PORT = process.env.PORT || 3001;
@@ -79,5 +80,7 @@ app.listen(PORT, () => {
   console.log(`  sofarr - Your Downloads Dashboard`);
   console.log(`  Server running on port ${PORT}`);
   console.log(`  Log level: ${process.env.LOG_LEVEL || 'info'}`);
+  console.log(`  Poll interval: ${POLL_INTERVAL}ms`);
   console.log(`=================================`);
+  startPoller();
 });

server/routes/dashboard.js

@@ -1,17 +1,9 @@
 const express = require('express');
-const axios = require('axios');
 const router = express.Router();
 
-const { getTorrents, mapTorrentToDownload } = require('../utils/qbittorrent');
+const { mapTorrentToDownload } = require('../utils/qbittorrent');
-const { 
-  getSABnzbdInstances, 
-  getSonarrInstances, 
-  getRadarrInstances 
-} = require('../utils/config');
 const cache = require('../utils/cache');
 
-const CACHE_TTL = 60 * 1000; // 60 seconds for slow-changing data (series, movies, tags)
-
 const EMBY_URL = process.env.EMBY_URL;
 const EMBY_API_KEY = process.env.EMBY_API_KEY;
 
@@ -97,19 +89,6 @@ function getRadarrLink(movie) {
   return `${movie._instanceUrl}/movie/${movie.titleSlug}`;
 }
 
-// Cached fetch helper - returns cached data if available, otherwise fetches and caches
-async function cachedFetch(cacheKey, fetchFn, ttl = CACHE_TTL) {
-  const cached = cache.get(cacheKey);
-  if (cached) {
-    console.log(`[Dashboard] Cache HIT: ${cacheKey}`);
-    return cached;
-  }
-  console.log(`[Dashboard] Cache MISS: ${cacheKey}`);
-  const data = await fetchFn();
-  cache.set(cacheKey, data, ttl);
-  return data;
-}
-
 // Get user downloads for authenticated user
 router.get('/user-downloads', async (req, res) => {
   try {
@@ -124,210 +103,33 @@ router.get('/user-downloads', async (req, res) => {
     const usernameSanitized = sanitizeTagLabel(user.name);
     const isAdmin = !!user.isAdmin;
     const showAll = isAdmin && req.query.showAll === 'true';
-    console.log(`[Dashboard] Fetching downloads for authenticated user: ${user.name} (${username}), isAdmin: ${isAdmin}, showAll: ${showAll}`);
+    console.log(`[Dashboard] Serving downloads for user: ${user.name} (${username}), isAdmin: ${isAdmin}, showAll: ${showAll}`);
 
-    // Get all service instances
+    // Read all data from poller cache
-    const sabInstances = getSABnzbdInstances();
+    const sabQueueData = cache.get('poll:sab-queue') || { slots: [] };
-    const sonarrInstances = getSonarrInstances();
+    const sabHistoryData = cache.get('poll:sab-history') || { slots: [] };
-    const radarrInstances = getRadarrInstances();
+    const sonarrTagsResults = cache.get('poll:sonarr-tags') || [];
+    const sonarrSeriesData = cache.get('poll:sonarr-series') || [];
+    const sonarrQueueData = cache.get('poll:sonarr-queue') || { records: [] };
+    const sonarrHistoryData = cache.get('poll:sonarr-history') || { records: [] };
+    const radarrMoviesData = cache.get('poll:radarr-movies') || [];
+    const radarrQueueData = cache.get('poll:radarr-queue') || { records: [] };
+    const radarrHistoryData = cache.get('poll:radarr-history') || { records: [] };
+    const radarrTagsData = cache.get('poll:radarr-tags') || [];
+    const qbittorrentTorrents = cache.get('poll:qbittorrent') || [];
 
-    console.log(`[Dashboard] Fetching data from all services...`);
+    // Wrap in the structure the rest of the code expects
-    console.log(`[Dashboard] SABnzbd instances: ${sabInstances.length}`);
+    const sabnzbdQueue = { data: { queue: sabQueueData } };
-    console.log(`[Dashboard] Sonarr instances: ${sonarrInstances.length}`);
+    const sabnzbdHistory = { data: { history: sabHistoryData } };
-    console.log(`[Dashboard] Radarr instances: ${radarrInstances.length}`);
+    const sonarrQueue = { data: sonarrQueueData };
+    const sonarrHistory = { data: sonarrHistoryData };
+    const sonarrSeries = { data: sonarrSeriesData };
+    const radarrQueue = { data: radarrQueueData };
+    const radarrHistory = { data: radarrHistoryData };
+    const radarrMovies = { data: radarrMoviesData };
+    const radarrTags = { data: radarrTagsData };
 
-    // Fetch from all SABnzbd instances
+    console.log(`[Dashboard] Cache data - Series: ${sonarrSeries.data.length}, Movies: ${radarrMovies.data.length}, qBit: ${qbittorrentTorrents.length}`);
-    const sabQueuePromises = sabInstances.map(inst => 
-      axios.get(`${inst.url}/api`, {
-        params: { mode: 'queue', apikey: inst.apiKey, output: 'json' }
-      }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-        console.error(`[Dashboard] SABnzbd ${inst.id} queue error:`, err.message);
-        return { instance: inst.id, data: { queue: { slots: [] } } };
-      })
-    );
-    
-    const sabHistoryPromises = sabInstances.map(inst => 
-      axios.get(`${inst.url}/api`, {
-        params: { mode: 'history', apikey: inst.apiKey, output: 'json', limit: 20 }
-      }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-        console.error(`[Dashboard] SABnzbd ${inst.id} history error:`, err.message);
-        return { instance: inst.id, data: { history: { slots: [] } } };
-      })
-    );
-    
-    // Fetch from all Sonarr instances (tags cached)
-    const sonarrTagsPromise = cachedFetch('sonarr-tags', () => 
-      Promise.all(sonarrInstances.map(inst => 
-        axios.get(`${inst.url}/api/v3/tag`, {
-          headers: { 'X-Api-Key': inst.apiKey }
-        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-          console.error(`[Dashboard] Sonarr ${inst.id} tags error:`, err.message);
-          return { instance: inst.id, data: [] };
-        })
-      ))
-    );
-    
-    const sonarrQueuePromises = sonarrInstances.map(inst => 
-      axios.get(`${inst.url}/api/v3/queue`, {
-        headers: { 'X-Api-Key': inst.apiKey },
-        params: { includeSeries: true, includeEpisode: true }
-      }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-        console.error(`[Dashboard] Sonarr ${inst.id} queue error:`, err.message);
-        return { instance: inst.id, data: { records: [] } };
-      })
-    );
-    
-    const sonarrHistoryPromises = sonarrInstances.map(inst => 
-      axios.get(`${inst.url}/api/v3/history`, {
-        headers: { 'X-Api-Key': inst.apiKey },
-        params: { pageSize: 20, includeSeries: true, includeEpisode: true }
-      }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-        console.error(`[Dashboard] Sonarr ${inst.id} history error:`, err.message);
-        return { instance: inst.id, data: { records: [] } };
-      })
-    );
-    
-    const sonarrSeriesPromise = cachedFetch('sonarr-series', () => 
-      Promise.all(sonarrInstances.map(inst => 
-        axios.get(`${inst.url}/api/v3/series`, {
-          headers: { 'X-Api-Key': inst.apiKey }
-        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-          console.error(`[Dashboard] Sonarr ${inst.id} series error:`, err.message);
-          return { instance: inst.id, data: [] };
-        })
-      ))
-    );
-    
-    // Fetch from all Radarr instances
-    const radarrQueuePromises = radarrInstances.map(inst => 
-      axios.get(`${inst.url}/api/v3/queue`, {
-        headers: { 'X-Api-Key': inst.apiKey },
-        params: { includeMovie: true }
-      }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-        console.error(`[Dashboard] Radarr ${inst.id} queue error:`, err.message);
-        return { instance: inst.id, data: { records: [] } };
-      })
-    );
-    
-    const radarrHistoryPromises = radarrInstances.map(inst => 
-      axios.get(`${inst.url}/api/v3/history`, {
-        headers: { 'X-Api-Key': inst.apiKey },
-        params: { pageSize: 20, includeMovie: true }
-      }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-        console.error(`[Dashboard] Radarr ${inst.id} history error:`, err.message);
-        return { instance: inst.id, data: { records: [] } };
-      })
-    );
-    
-    const radarrMoviesPromise = cachedFetch('radarr-movies', () => 
-      Promise.all(radarrInstances.map(inst => 
-        axios.get(`${inst.url}/api/v3/movie`, {
-          headers: { 'X-Api-Key': inst.apiKey }
-        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-          console.error(`[Dashboard] Radarr ${inst.id} movies error:`, err.message);
-          return { instance: inst.id, data: [] };
-        })
-      ))
-    );
-    
-    const radarrTagsPromise = cachedFetch('radarr-tags', () => 
-      Promise.all(radarrInstances.map(inst => 
-        axios.get(`${inst.url}/api/v3/tag`, {
-          headers: { 'X-Api-Key': inst.apiKey }
-        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
-          console.error(`[Dashboard] Radarr ${inst.id} tags error:`, err.message);
-          return { instance: inst.id, data: [] };
-        })
-      ))
-    );
-    
-    // Execute all requests (cached items resolve instantly on cache hit)
-    const [
-      sabQueues, sabHistories, sonarrTagsResults, sonarrQueues, sonarrHistories, sonarrSeriesResults,
-      radarrQueues, radarrHistories, radarrMoviesResults, radarrTagsResults,
-      qbittorrentTorrents
-    ] = await Promise.all([
-      Promise.all(sabQueuePromises),
-      Promise.all(sabHistoryPromises),
-      sonarrTagsPromise,
-      Promise.all(sonarrQueuePromises),
-      Promise.all(sonarrHistoryPromises),
-      sonarrSeriesPromise,
-      Promise.all(radarrQueuePromises),
-      Promise.all(radarrHistoryPromises),
-      radarrMoviesPromise,
-      radarrTagsPromise,
-      getTorrents().catch(err => {
-        console.error(`[Dashboard] qBittorrent error:`, err.message);
-        return [];
-      })
-    ]);
-    
-    // Aggregate data from all instances
-    const firstSabQueue = sabQueues[0] && sabQueues[0].data && sabQueues[0].data.queue;
-    const sabnzbdQueue = { 
-      data: { 
-        queue: { 
-          slots: sabQueues.flatMap(q => (q.data.queue && q.data.queue.slots) || []),
-          status: firstSabQueue && firstSabQueue.status,
-          speed: firstSabQueue && firstSabQueue.speed,
-          kbpersec: firstSabQueue && firstSabQueue.kbpersec
-        } 
-      } 
-    };
-    const sabnzbdHistory = { 
-      data: { 
-        history: { 
-          slots: sabHistories.flatMap(h => (h.data.history && h.data.history.slots) || []) 
-        } 
-      } 
-    };
-    const sonarrQueue = { 
-      data: { 
-        records: sonarrQueues.flatMap(q => q.data.records || []) 
-      } 
-    };
-    const sonarrHistory = { 
-      data: { 
-        records: sonarrHistories.flatMap(h => h.data.records || []) 
-      } 
-    };
-    const sonarrSeries = { 
-      data: sonarrSeriesResults.flatMap(s => {
-        const inst = sonarrInstances.find(i => i.id === s.instance);
-        return (s.data || []).map(item => ({ ...item, _instanceUrl: inst ? inst.url : null }));
-      })
-    };
-    const radarrQueue = { 
-      data: { 
-        records: radarrQueues.flatMap(q => q.data.records || []) 
-      } 
-    };
-    const radarrHistory = { 
-      data: { 
-        records: radarrHistories.flatMap(h => h.data.records || []) 
-      } 
-    };
-    const radarrMovies = { 
-      data: radarrMoviesResults.flatMap(m => {
-        const inst = radarrInstances.find(i => i.id === m.instance);
-        return (m.data || []).map(item => ({ ...item, _instanceUrl: inst ? inst.url : null }));
-      })
-    };
-    const radarrTags = { 
-      data: radarrTagsResults.flatMap(t => t.data || []) 
-    };
-    
-    console.log(`[Dashboard] Data fetched successfully`);
-    console.log(`[Dashboard] Sonarr series: ${sonarrSeries.data.length}`);
-    console.log(`[Dashboard] Radarr movies: ${radarrMovies.data.length}`);
-    console.log(`[Dashboard] Radarr queue records: ${radarrQueue.data.records.length}`);
-    console.log(`[Dashboard] Radarr history records: ${radarrHistory.data.records.length}`);
-    console.log(`[Dashboard] SABnzbd queue slots: ${sabnzbdQueue.data.queue ? sabnzbdQueue.data.queue.slots.length : 0}`);
-    console.log(`[Dashboard] SABnzbd history slots: ${sabnzbdHistory.data.history ? sabnzbdHistory.data.history.slots.length : 0}`);
-    console.log(`[Dashboard] qBittorrent torrents: ${qbittorrentTorrents.length}`);
-    console.log(`[Dashboard] Radarr queue:`, JSON.stringify(radarrQueue.data.records));
-    console.log(`[Dashboard] Radarr history:`, JSON.stringify(radarrHistory.data.records));
 
     // Create maps for quick lookup
     const seriesMap = new Map(sonarrSeries.data.map(s => [s.id, s]));

server/utils/poller.js

@@ -0,0 +1,199 @@
+const axios = require('axios');
+const cache = require('./cache');
+const { getTorrents } = require('./qbittorrent');
+const {
+  getSABnzbdInstances,
+  getSonarrInstances,
+  getRadarrInstances
+} = require('./config');
+
+const POLL_INTERVAL = parseInt(process.env.POLL_INTERVAL, 10) || 5000;
+
+let polling = false;
+
+async function pollAllServices() {
+  if (polling) {
+    console.log('[Poller] Previous poll still running, skipping');
+    return;
+  }
+  polling = true;
+  const start = Date.now();
+
+  try {
+    const sabInstances = getSABnzbdInstances();
+    const sonarrInstances = getSonarrInstances();
+    const radarrInstances = getRadarrInstances();
+
+    // All fetches in parallel
+    const [
+      sabQueues, sabHistories,
+      sonarrTagsResults, sonarrQueues, sonarrHistories, sonarrSeriesResults,
+      radarrQueues, radarrHistories, radarrMoviesResults, radarrTagsResults,
+      qbittorrentTorrents
+    ] = await Promise.all([
+      // SABnzbd
+      Promise.all(sabInstances.map(inst =>
+        axios.get(`${inst.url}/api`, {
+          params: { mode: 'queue', apikey: inst.apiKey, output: 'json' }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] SABnzbd ${inst.id} queue error:`, err.message);
+          return { instance: inst.id, data: { queue: { slots: [] } } };
+        })
+      )),
+      Promise.all(sabInstances.map(inst =>
+        axios.get(`${inst.url}/api`, {
+          params: { mode: 'history', apikey: inst.apiKey, output: 'json', limit: 20 }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] SABnzbd ${inst.id} history error:`, err.message);
+          return { instance: inst.id, data: { history: { slots: [] } } };
+        })
+      )),
+      // Sonarr
+      Promise.all(sonarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/tag`, {
+          headers: { 'X-Api-Key': inst.apiKey }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Sonarr ${inst.id} tags error:`, err.message);
+          return { instance: inst.id, data: [] };
+        })
+      )),
+      Promise.all(sonarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/queue`, {
+          headers: { 'X-Api-Key': inst.apiKey },
+          params: { includeSeries: true, includeEpisode: true }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Sonarr ${inst.id} queue error:`, err.message);
+          return { instance: inst.id, data: { records: [] } };
+        })
+      )),
+      Promise.all(sonarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/history`, {
+          headers: { 'X-Api-Key': inst.apiKey },
+          params: { pageSize: 20, includeSeries: true, includeEpisode: true }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Sonarr ${inst.id} history error:`, err.message);
+          return { instance: inst.id, data: { records: [] } };
+        })
+      )),
+      Promise.all(sonarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/series`, {
+          headers: { 'X-Api-Key': inst.apiKey }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Sonarr ${inst.id} series error:`, err.message);
+          return { instance: inst.id, data: [] };
+        })
+      )),
+      // Radarr
+      Promise.all(radarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/queue`, {
+          headers: { 'X-Api-Key': inst.apiKey },
+          params: { includeMovie: true }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Radarr ${inst.id} queue error:`, err.message);
+          return { instance: inst.id, data: { records: [] } };
+        })
+      )),
+      Promise.all(radarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/history`, {
+          headers: { 'X-Api-Key': inst.apiKey },
+          params: { pageSize: 20, includeMovie: true }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Radarr ${inst.id} history error:`, err.message);
+          return { instance: inst.id, data: { records: [] } };
+        })
+      )),
+      Promise.all(radarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/movie`, {
+          headers: { 'X-Api-Key': inst.apiKey }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Radarr ${inst.id} movies error:`, err.message);
+          return { instance: inst.id, data: [] };
+        })
+      )),
+      Promise.all(radarrInstances.map(inst =>
+        axios.get(`${inst.url}/api/v3/tag`, {
+          headers: { 'X-Api-Key': inst.apiKey }
+        }).then(res => ({ instance: inst.id, data: res.data })).catch(err => {
+          console.error(`[Poller] Radarr ${inst.id} tags error:`, err.message);
+          return { instance: inst.id, data: [] };
+        })
+      )),
+      // qBittorrent
+      getTorrents().catch(err => {
+        console.error(`[Poller] qBittorrent error:`, err.message);
+        return [];
+      })
+    ]);
+
+    // Aggregate and store in cache (TTL slightly longer than poll interval to avoid gaps)
+    const cacheTTL = POLL_INTERVAL * 3;
+
+    // SABnzbd
+    const firstSabQueue = sabQueues[0] && sabQueues[0].data && sabQueues[0].data.queue;
+    cache.set('poll:sab-queue', {
+      slots: sabQueues.flatMap(q => (q.data.queue && q.data.queue.slots) || []),
+      status: firstSabQueue && firstSabQueue.status,
+      speed: firstSabQueue && firstSabQueue.speed,
+      kbpersec: firstSabQueue && firstSabQueue.kbpersec
+    }, cacheTTL);
+
+    cache.set('poll:sab-history', {
+      slots: sabHistories.flatMap(h => (h.data.history && h.data.history.slots) || [])
+    }, cacheTTL);
+
+    // Sonarr
+    cache.set('poll:sonarr-tags', sonarrTagsResults, cacheTTL);
+    cache.set('poll:sonarr-queue', {
+      records: sonarrQueues.flatMap(q => q.data.records || [])
+    }, cacheTTL);
+    cache.set('poll:sonarr-history', {
+      records: sonarrHistories.flatMap(h => h.data.records || [])
+    }, cacheTTL);
+    cache.set('poll:sonarr-series', sonarrSeriesResults.flatMap(s => {
+      const inst = sonarrInstances.find(i => i.id === s.instance);
+      return (s.data || []).map(item => ({ ...item, _instanceUrl: inst ? inst.url : null }));
+    }), cacheTTL);
+
+    // Radarr
+    cache.set('poll:radarr-queue', {
+      records: radarrQueues.flatMap(q => q.data.records || [])
+    }, cacheTTL);
+    cache.set('poll:radarr-history', {
+      records: radarrHistories.flatMap(h => h.data.records || [])
+    }, cacheTTL);
+    cache.set('poll:radarr-movies', radarrMoviesResults.flatMap(m => {
+      const inst = radarrInstances.find(i => i.id === m.instance);
+      return (m.data || []).map(item => ({ ...item, _instanceUrl: inst ? inst.url : null }));
+    }), cacheTTL);
+    cache.set('poll:radarr-tags', radarrTagsResults.flatMap(t => t.data || []), cacheTTL);
+
+    // qBittorrent
+    cache.set('poll:qbittorrent', qbittorrentTorrents, cacheTTL);
+
+    const elapsed = Date.now() - start;
+    console.log(`[Poller] Poll complete in ${elapsed}ms`);
+  } catch (err) {
+    console.error(`[Poller] Poll error:`, err.message);
+  } finally {
+    polling = false;
+  }
+}
+
+let intervalHandle = null;
+
+function startPoller() {
+  console.log(`[Poller] Starting background poller (interval: ${POLL_INTERVAL}ms)`);
+  // Run immediately, then on interval
+  pollAllServices();
+  intervalHandle = setInterval(pollAllServices, POLL_INTERVAL);
+}
+
+function stopPoller() {
+  if (intervalHandle) {
+    clearInterval(intervalHandle);
+    intervalHandle = null;
+    console.log('[Poller] Stopped');
+  }
+}
+
+module.exports = { startPoller, stopPoller, POLL_INTERVAL };