Improve listening stats query efficiency

server/Database.js

@@ -512,9 +512,19 @@ class Database {
     return this.models.setting.updateSettingObj(settings.toJSON())
   }
 
-  getPlaybackSessions(where = null) {
+  getPlaybackSessions(where = null, options = null) {
     if (!this.sequelize) return false
-    return this.models.playbackSession.getOldPlaybackSessions(where)
+    return this.models.playbackSession.getOldPlaybackSessions(where, options || undefined)
+  }
+
+  countPlaybackSessions(where = null) {
+    if (!this.sequelize) return false
+    return this.models.playbackSession.countWithWhere(where)
+  }
+
+  getPlaybackSessionsForStats(where = null) {
+    if (!this.sequelize) return false
+    return this.models.playbackSession.getPlaybackSessionsForStats(where)
   }
 
   getPlaybackSession(sessionId) {

server/controllers/MeController.js

@@ -155,21 +155,23 @@ class MeController {
    * @param {Response} res
    */
   async getListeningSessions(req, res) {
-    const listeningSessions = await this.getUserListeningSessionsHelper(req.user.id)
-
+    const startedAt = Date.now()
     const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
     const page = toNumber(req.query.page, 0)
 
-    const start = page * itemsPerPage
-    const sessions = listeningSessions.slice(start, start + itemsPerPage)
+    Logger.debug(
+      `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start`
+    )
 
-    const payload = {
-      total: listeningSessions.length,
-      numPages: Math.ceil(listeningSessions.length / itemsPerPage),
+    const payload = await this.getUserListeningSessionsPageHelper(
+      req.user.id,
       page,
-      itemsPerPage,
-      sessions
-    }
+      itemsPerPage
+    )
+
+    Logger.debug(
+      `[MeController] /api/me/listening-sessions user="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
+    )
 
     res.json(payload)
   }
@@ -183,6 +185,7 @@ class MeController {
    * @param {Response} res
    */
   async getItemListeningSessions(req, res) {
+    const startedAt = Date.now()
     const libraryItem = await Database.libraryItemModel.getExpandedById(req.params.libraryItemId)
     const episode = await Database.podcastEpisodeModel.findByPk(req.params.episodeId)
 
@@ -198,21 +201,23 @@ class MeController {
     }
 
     const mediaItemId = episode?.id || libraryItem.mediaId
-    let listeningSessions = await this.getUserItemListeningSessionsHelper(req.user.id, mediaItemId)
-
     const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
     const page = toNumber(req.query.page, 0)
 
-    const start = page * itemsPerPage
-    const sessions = listeningSessions.slice(start, start + itemsPerPage)
+    Logger.debug(
+      `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} start`
+    )
 
-    const payload = {
-      total: listeningSessions.length,
-      numPages: Math.ceil(listeningSessions.length / itemsPerPage),
+    const payload = await this.getUserListeningSessionsPageHelper(
+      req.user.id,
       page,
       itemsPerPage,
-      sessions
-    }
+      mediaItemId
+    )
+
+    Logger.debug(
+      `[MeController] /api/me/item/listening-sessions user="${req.user.id}" libraryItem="${req.params.libraryItemId}" episode="${req.params.episodeId || ''}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
+    )
 
     res.json(payload)
   }
@@ -226,7 +231,12 @@ class MeController {
    * @param {Response} res
    */
   async getListeningStats(req, res) {
+    const startedAt = Date.now()
+    Logger.debug(`[MeController] /api/me/listening-stats user="${req.user.id}" start`)
     const listeningStats = await this.getUserListeningStatsHelpers(req.user.id)
+    Logger.debug(
+      `[MeController] /api/me/listening-stats user="${req.user.id}" totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms`
+    )
     res.json(listeningStats)
   }
 

server/controllers/UserController.js

@@ -432,14 +432,20 @@ class UserController {
    * @param {Response} res
    */
   async getListeningSessions(req, res) {
-    var listeningSessions = await this.getUserListeningSessionsHelper(req.params.id)
-
+    const startedAt = Date.now()
     const itemsPerPage = toNumber(req.query.itemsPerPage, 10) || 10
     const page = toNumber(req.query.page, 0)
+    Logger.debug(
+      `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} start`
+    )
+    const payload = await this.getUserListeningSessionsPageHelper(
+      req.params.id,
+      page,
+      itemsPerPage
+    )
 
-    const start = page * itemsPerPage
     // Map user to sessions to match the format of the sessions endpoint
-    const sessions = listeningSessions.slice(start, start + itemsPerPage).map((session) => {
+    payload.sessions = payload.sessions.map((session) => {
       return {
         ...session,
         user: {
@@ -449,13 +455,9 @@ class UserController {
       }
     })
 
-    const payload = {
-      total: listeningSessions.length,
-      numPages: Math.ceil(listeningSessions.length / itemsPerPage),
-      page,
-      itemsPerPage,
-      sessions
-    }
+    Logger.debug(
+      `[UserController] /api/users/${req.params.id}/listening-sessions reqUser="${req.user.id}" page=${page} itemsPerPage=${itemsPerPage} total=${payload.total} returned=${payload.sessions.length} in ${Date.now() - startedAt}ms`
+    )
 
     res.json(payload)
   }
@@ -469,7 +471,14 @@ class UserController {
    * @param {Response} res
    */
   async getListeningStats(req, res) {
+    const startedAt = Date.now()
+    Logger.debug(
+      `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" start`
+    )
     var listeningStats = await this.getUserListeningStatsHelpers(req.params.id)
+    Logger.debug(
+      `[UserController] /api/users/${req.params.id}/listening-stats reqUser="${req.user.id}" totalTime=${listeningStats?.totalTime || 0} recentSessions=${listeningStats?.recentSessions?.length || 0} items=${Object.keys(listeningStats?.items || {}).length} in ${Date.now() - startedAt}ms`
+    )
     res.json(listeningStats)
   }
 

server/models/PlaybackSession.js

@@ -52,18 +52,54 @@ class PlaybackSession extends Model {
     this.createdAt
   }
 
-  static async getOldPlaybackSessions(where = null) {
+  static async getOldPlaybackSessions(where = null, options = {}) {
+    const {
+      limit = null,
+      offset = null,
+      order = [['updatedAt', 'DESC']],
+      includeDevice = true
+    } = options
+
     const playbackSessions = await this.findAll({
       where,
-      include: [
+      limit,
+      offset,
+      order,
+      include: includeDevice
+        ? [
             {
               model: this.sequelize.models.device
             }
           ]
+        : undefined
     })
     return playbackSessions.map((session) => this.getOldPlaybackSession(session))
   }
 
+  static countWithWhere(where = null) {
+    return this.count({ where })
+  }
+
+  static getPlaybackSessionsForStats(where = null) {
+    return this.findAll({
+      where,
+      order: [['updatedAt', 'DESC']],
+      attributes: [
+        'mediaItemId',
+        'mediaItemType',
+        'displayTitle',
+        'displayAuthor',
+        'timeListening',
+        'mediaMetadata',
+        'date',
+        'dayOfWeek',
+        'updatedAt',
+        'extraData'
+      ],
+      raw: true
+    })
+  }
+
   static async getById(sessionId) {
     const playbackSession = await this.findByPk(sessionId, {
       include: [

server/routers/ApiRouter.js

@@ -527,24 +527,57 @@ class ApiRouter {
     return userSessions.sort((a, b) => b.updatedAt - a.updatedAt)
   }
 
+  async getUserListeningSessionsPageHelper(userId, page = 0, itemsPerPage = 10, mediaItemId = null) {
+    const where = { userId }
+    if (mediaItemId) where.mediaItemId = mediaItemId
+
+    const start = page * itemsPerPage
+    const [total, sessions] = await Promise.all([
+      Database.countPlaybackSessions(where),
+      Database.getPlaybackSessions(where, {
+        limit: itemsPerPage,
+        offset: start,
+        order: [['updatedAt', 'DESC']]
+      })
+    ])
+
+    return {
+      total,
+      numPages: Math.ceil(total / itemsPerPage),
+      page,
+      itemsPerPage,
+      sessions
+    }
+  }
+
   async getUserItemListeningSessionsHelper(userId, mediaItemId) {
     const userSessions = await Database.getPlaybackSessions({ userId, mediaItemId })
     return userSessions.sort((a, b) => b.updatedAt - a.updatedAt)
   }
 
   async getUserListeningStatsHelpers(userId) {
+    const startedAt = Date.now()
     const today = date.format(new Date(), 'YYYY-MM-DD')
 
-    const listeningSessions = await this.getUserListeningSessionsHelper(userId)
+    const [listeningSessions, recentSessions] = await Promise.all([
+      Database.getPlaybackSessionsForStats({ userId }),
+      Database.getPlaybackSessions({ userId }, {
+        limit: 10,
+        offset: 0,
+        order: [['updatedAt', 'DESC']]
+      })
+    ])
+
     const listeningStats = {
       totalTime: 0,
       items: {},
       days: {},
       dayOfWeek: {},
       today: 0,
-      recentSessions: listeningSessions.slice(0, 10)
+      recentSessions
     }
     listeningSessions.forEach((s) => {
+      const libraryItemId = s.extraData?.libraryItemId || null
       let sessionTimeListening = s.timeListening
       if (typeof sessionTimeListening == 'string') {
         sessionTimeListening = Number(sessionTimeListening)
@@ -562,19 +595,27 @@ class ApiRouter {
           listeningStats.today += sessionTimeListening
         }
       }
-      if (!listeningStats.items[s.libraryItemId]) {
-        listeningStats.items[s.libraryItemId] = {
-          id: s.libraryItemId,
+      if (!libraryItemId) {
+        listeningStats.totalTime += sessionTimeListening
+        return
+      }
+
+      if (!listeningStats.items[libraryItemId]) {
+        listeningStats.items[libraryItemId] = {
+          id: libraryItemId,
           timeListening: sessionTimeListening,
           mediaMetadata: s.mediaMetadata,
-          lastUpdate: s.lastUpdate
+          lastUpdate: s.updatedAt
         }
       } else {
-        listeningStats.items[s.libraryItemId].timeListening += sessionTimeListening
+        listeningStats.items[libraryItemId].timeListening += sessionTimeListening
       }
 
       listeningStats.totalTime += sessionTimeListening
     })
+    Logger.debug(
+      `[ApiRouter] Listening stats for user "${userId}" aggregated ${listeningSessions.length} sessions in ${Date.now() - startedAt}ms`
+    )
     return listeningStats
   }
 }