Init commit

server/core/controllers/sitemap.ts

@@ -0,0 +1,147 @@
+import { VideoFileStream, VideoInclude } from '@peertube/peertube-models'
+import { logger } from '@server/helpers/logger.js'
+import { ServerConfigManager } from '@server/lib/server-config-manager.js'
+import { getServerActor } from '@server/models/application/application.js'
+import express from 'express'
+import truncate from 'lodash-es/truncate.js'
+import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap'
+import { buildNSFWFilters } from '../helpers/express-utils.js'
+import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants.js'
+import { apiRateLimiter, asyncMiddleware, cacheRoute } from '../middlewares/index.js'
+import { AccountModel } from '../models/account/account.js'
+import { VideoChannelModel } from '../models/video/video-channel.js'
+import { VideoModel } from '../models/video/video.js'
+
+const sitemapRouter = express.Router()
+
+sitemapRouter.use('/sitemap.xml', apiRateLimiter, cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP), asyncMiddleware(getSitemap))
+
+// ---------------------------------------------------------------------------
+
+export {
+  sitemapRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function getSitemap (req: express.Request, res: express.Response) {
+  let urls = getSitemapBasicUrls()
+
+  urls = urls.concat(await getSitemapLocalVideoUrls())
+  urls = urls.concat(await getSitemapVideoChannelUrls())
+  urls = urls.concat(await getSitemapAccountUrls())
+
+  const sitemapStream = new SitemapStream({
+    hostname: WEBSERVER.URL,
+    errorHandler: (err: Error, level: ErrorLevel) => {
+      if (level === ErrorLevel.WARN) {
+        logger.warn('Warning in sitemap generation.', { err })
+      } else if (level === ErrorLevel.THROW) {
+        logger.error('Error in sitemap generation.', { err })
+
+        throw err
+      }
+    }
+  })
+
+  for (const urlObj of urls) {
+    sitemapStream.write(urlObj)
+  }
+  sitemapStream.end()
+
+  const xml = await streamToPromise(sitemapStream)
+
+  res.header('Content-Type', 'application/xml')
+  res.send(xml)
+}
+
+async function getSitemapVideoChannelUrls () {
+  const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
+
+  return rows.map(channel => ({ url: channel.getClientUrl() }))
+}
+
+async function getSitemapAccountUrls () {
+  const rows = await AccountModel.listLocalsForSitemap('createdAt')
+
+  return rows.map(account => ({ url: account.getClientUrl() }))
+}
+
+async function getSitemapLocalVideoUrls () {
+  const serverActor = await getServerActor()
+
+  let acc: { url: string, video: any[] }[] = []
+
+  const chunkSize = 200
+  let hasData = true
+  let i = 0
+
+  while (hasData && i < 1000) {
+    const { data } = await VideoModel.listForApi({
+      ...buildNSFWFilters(),
+
+      start: chunkSize * i,
+      count: chunkSize,
+      sort: 'createdAt',
+      displayOnlyForFollower: {
+        actorId: serverActor.id,
+        orLocalVideos: true
+      },
+      isLocal: true,
+      countVideos: false,
+      include: VideoInclude.FILES | VideoInclude.TAGS
+    })
+
+    hasData = data.length !== 0
+    i++
+
+    acc = acc.concat(
+      data.map(v => {
+        const contentLoc = v.getHLSPlaylist()?.getMasterPlaylistUrl(v) ||
+          v.getMaxQualityFile(VideoFileStream.VIDEO)?.getFileUrl(v) ||
+          v.getMaxQualityFile(VideoFileStream.AUDIO)?.getFileUrl(v)
+
+        return {
+          url: process.env.EXPERIMENTAL_SITEMAP_VIDEO_URL === 'true'
+            ? WEBSERVER.URL + '/videos/watch/' + v.uuid
+            : WEBSERVER.URL + v.getWatchStaticPath(),
+          video: [
+            {
+              // Sitemap title should be < 100 characters
+              'title': truncate(v.name, { length: 100, omission: '...' }),
+              // Sitemap description should be < 2000 characters
+              'description': truncate(v.description || v.name, { length: 2000, omission: '...' }),
+              'player_loc': WEBSERVER.URL + v.getEmbedStaticPath(),
+              'thumbnail_loc': WEBSERVER.URL + v.getMiniatureStaticPath(),
+              'content_loc': contentLoc,
+              'duration': v.duration,
+              'view_count': v.views,
+              'publication_date': v.publishedAt.toISOString(),
+              'uploader': v.VideoChannel.getDisplayName(),
+              'uploader:info': v.VideoChannel.getClientUrl(),
+              'live': v.isLive ? 'YES' : 'NO',
+              'family_friendly': v.nsfw ? 'NO' : 'YES',
+              'rating': (v.likes * 5) / (v.likes + v.dislikes) || 0, // Rating is between 0.0 and 5.0
+              'tag': v.Tags.map(t => t.name)
+            }
+          ]
+        }
+      })
+    )
+  }
+
+  return acc
+}
+
+function getSitemapBasicUrls () {
+  const paths = [
+    '/about/instance/home',
+    '/videos/browse?scope=local'
+  ]
+
+  if (ServerConfigManager.Instance.isHomepageEnabled()) {
+    paths.push('/home')
+  }
+
+  return paths.map(p => ({ url: WEBSERVER.URL + p }))
+}