Init commit

server/core/models/shared/abstract-list-query.ts

@@ -0,0 +1,124 @@
+import { AbstractRunQuery, ModelBuilder, parseRowCountResult } from '@server/models/shared/index.js'
+import { Model, Sequelize, Transaction } from 'sequelize'
+
+export interface AbstractListQueryOptions {
+  start?: number
+  count?: number
+  sort?: string
+  transaction?: Transaction
+}
+
+export interface ModelInfo {
+  tableName: string
+  modelName: string
+}
+
+export abstract class AbstractListQuery extends AbstractRunQuery {
+  protected attributes: string[] = []
+  protected join = ''
+
+  private subQuery: string
+
+  protected subQueryAttributes: string[] = []
+  protected subQueryLateralJoin = ''
+  protected subQueryJoin = ''
+  protected subQueryWhere = ''
+
+  protected abstract buildQueryAttributes (): void
+  protected abstract buildQueryJoin (): void
+
+  protected abstract buildSubQueryWhere (): void
+  protected abstract buildSubQueryAttributes (): void
+  protected abstract buildSubQueryJoin (): void
+
+  protected buildSubQueryLateralJoin () {
+    // Optional
+  }
+
+  protected getCalculatedAttributes () {
+    return []
+  }
+
+  constructor (
+    protected readonly sequelize: Sequelize,
+    protected readonly modelInfo: ModelInfo,
+    protected readonly options: AbstractListQueryOptions
+  ) {
+    super(sequelize)
+  }
+
+  async get<T extends Model> () {
+    this.options.start = 0
+    this.options.count = 1
+
+    const rows = await this.list<T>()
+
+    return rows.length === 1
+      ? rows[0]
+      : null
+  }
+
+  async list<T extends Model> () {
+    this.buildListQuery()
+
+    const results = await this.runQuery({ nest: true, transaction: this.options.transaction })
+    const modelBuilder = new ModelBuilder<T>(this.sequelize)
+
+    return modelBuilder.createModels(results, this.modelInfo.modelName.replace(/Model$/, ''))
+  }
+
+  async count () {
+    this.buildCountQuery()
+
+    const result = await this.runQuery({ transaction: this.options.transaction })
+
+    return parseRowCountResult(result)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private buildListQuery () {
+    this.buildSubQuery()
+
+    this.attributes = [ `"${this.modelInfo.modelName}".*` ]
+    this.buildQueryAttributes()
+    this.buildQueryJoin()
+
+    this.query = `${this.buildSelect(this.attributes)} ` +
+      `FROM (${this.subQuery}) AS "${this.modelInfo.modelName}" ` +
+      `${this.join} ` +
+      `${this.getOrder(this.modelInfo.modelName, this.options.sort)}`
+  }
+
+  private buildSubQuery () {
+    this.buildSubQueryWhere()
+    this.buildSubQueryJoin()
+    this.buildSubQueryLateralJoin()
+    this.buildSubQueryAttributes()
+
+    this.subQuery = `${this.buildSelect(this.subQueryAttributes)} ` +
+      `FROM "${this.modelInfo.tableName}" AS "${this.modelInfo.modelName}" ` +
+      `${this.subQueryJoin} ` +
+      `${this.subQueryLateralJoin} ` +
+      `${this.subQueryWhere} `
+
+    if (this.options.sort) {
+      this.subQuery += `${this.getOrder(this.modelInfo.modelName, this.options.sort, this.getCalculatedAttributes())} `
+    }
+
+    if (this.options.start !== undefined && this.options.count !== undefined) {
+      this.subQuery += this.getLimit(this.options.start, this.options.count)
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private buildCountQuery () {
+    this.buildSubQueryWhere()
+
+    this.query = `SELECT COUNT(*) AS "total" ` +
+      `FROM "${this.modelInfo.tableName}" AS "${this.modelInfo.modelName}" ` +
+      `${this.subQueryJoin} ` +
+      `${this.subQueryWhere}`
+  }
+}

server/core/models/shared/abstract-run-query.ts

@@ -0,0 +1,78 @@
+import { QueryTypes, Sequelize, Transaction } from 'sequelize'
+import { getSort } from './sort.js'
+import { Col } from 'sequelize/lib/utils'
+
+/**
+ * Abstract builder to run video SQL queries
+ */
+
+export class AbstractRunQuery {
+  protected query: string
+  protected replacements: any = {}
+
+  protected queryConfig = ''
+
+  constructor (protected readonly sequelize: Sequelize) {
+  }
+
+  protected async runQuery (options: { nest?: boolean, transaction?: Transaction, logging?: boolean } = {}) {
+    const queryOptions = {
+      transaction: options.transaction,
+      logging: options.logging,
+      replacements: this.replacements,
+      type: QueryTypes.SELECT as QueryTypes.SELECT,
+      nest: options.nest ?? false
+    }
+
+    if (this.queryConfig) {
+      await this.sequelize.query(this.queryConfig, queryOptions)
+    }
+
+    return this.sequelize.query<any>(this.query, queryOptions)
+  }
+
+  protected buildSelect (attributes: string[]) {
+    return `SELECT ${attributes.join(', ')} `
+  }
+
+  // ---------------------------------------------------------------------------
+
+  protected getOrder (tableName: string, sort: string, calculatedAttributes: string[] = []) {
+    if (!sort) return ''
+
+    const orders = this.getSort(sort)
+
+    const orderStr = orders.map(o => {
+      const columnName = o[0] instanceof Col
+        ? o[0].col
+        : o[0]
+
+      const direction = o[1]
+
+      // Prefix with the table name if the column name isn't a full path
+      // ("id", "displayName", etc. VS "ActorModel.id", "Server.redundancyAllowed", etc.)
+      const prefix = calculatedAttributes.includes(columnName) || columnName.includes('.')
+        ? ''
+        : `"${tableName}".`
+
+      return `${prefix}"${columnName}" ${direction}`
+    }).join(', ')
+
+    return 'ORDER BY ' + orderStr
+  }
+
+  protected getSort (sort: string) {
+    return getSort(sort)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  protected getLimit (start: number, count: number) {
+    if (!count) return ''
+
+    this.replacements.limit = count
+    this.replacements.offset = start || 0
+
+    return `LIMIT :limit OFFSET :offset `
+  }
+}

server/core/models/shared/delete.ts

@@ -0,0 +1,9 @@
+import { MAX_SQL_DELETE_ITEMS } from '@server/initializers/constants.js'
+
+export async function safeBulkDestroy (destroyFn: () => Promise<number>) {
+  const destroyedRows = await destroyFn()
+
+  if (destroyedRows === MAX_SQL_DELETE_ITEMS) {
+    return safeBulkDestroy(destroyFn)
+  }
+}

server/core/models/shared/index.ts

@@ -0,0 +1,11 @@
+export * from './abstract-run-query.js'
+export * from './model-builder.js'
+export * from './model-cache.js'
+export * from './query.js'
+export * from './sequelize-helpers.js'
+export * from './sequelize-type.js'
+export * from './sort.js'
+export * from './sql/common-helpers.js'
+export * from './update.js'
+export * from './delete.js'
+export * from './table.js'

server/core/models/shared/model-builder.ts

@@ -0,0 +1,154 @@
+import { logger } from '@server/helpers/logger.js'
+import isPlainObject from 'lodash-es/isPlainObject.js'
+import { ModelStatic, Sequelize, Model as SequelizeModel } from 'sequelize'
+
+/**
+ * Build Sequelize models from sequelize raw query (that must use { nest: true } options)
+ *
+ * In order to sequelize to correctly build the JSON this class will ingest,
+ * the columns selected in the raw query should be in the following form:
+ *   * All tables must be Pascal Cased (for example "VideoChannel")
+ *   * Root table must end with `Model` (for example "VideoCommentModel")
+ *   * Joined tables must contain the origin table name + '->JoinedTable'. For example:
+ *     * "Actor" is joined to "Account": "Actor" table must be renamed "Account->Actor"
+ *     * "Account->Actor" is joined to "Server": "Server" table must be renamed to "Account->Actor->Server"
+ *   * Selected columns must be renamed to contain the JSON path:
+ *     * "videoComment"."id": "VideoCommentModel"."id"
+ *     * "Account"."Actor"."Server"."id": "Account.Actor.Server.id"
+ *   * All tables must contain the row id
+ */
+
+export class ModelBuilder<T extends SequelizeModel> {
+  private readonly modelRegistry = new Map<string, T>()
+
+  private readonly aliasToTableName = {
+    ChannelCollab: 'VideoChannelCollaborator',
+    Collabs: 'VideoChannelCollaborators'
+  }
+
+  private readonly tableNameToModelName = {
+    Avatars: 'ActorImageModel',
+    Banners: 'ActorImageModel',
+    ActorFollowing: 'ActorModel',
+    ActorFollower: 'ActorModel',
+    FlaggedAccount: 'AccountModel',
+    CommentAutomaticTags: 'CommentAutomaticTagModel',
+    OwnerAccount: 'AccountModel',
+    Thumbnails: 'ThumbnailModel',
+    Channel: 'VideoChannelModel',
+    VideoChannels: 'VideoChannelModel',
+    VideoPlaylists: 'VideoPlaylistModel',
+    NotificationSetting: 'UserNotificationSettingModel',
+    VideoChannelCollaborators: 'VideoChannelCollaboratorModel',
+    Tags: 'TagModel'
+  }
+
+  constructor (private readonly sequelize: Sequelize) {
+  }
+
+  createModels (jsonArray: any[], baseModelName: string): T[] {
+    const result: T[] = []
+
+    for (const json of jsonArray) {
+      const { created, model } = this.createModel(json, baseModelName, json.id + '.' + baseModelName)
+
+      if (created) result.push(model)
+    }
+
+    return result
+  }
+
+  private createModel (json: any, rootTableName: string, keyPath: string) {
+    if (!json.id) return { created: false, model: null }
+
+    const { created, model } = this.createOrFindModel(json, rootTableName, keyPath)
+
+    for (const key of Object.keys(json)) {
+      const value = json[key]
+      if (!value) continue
+
+      const tableName = this.buildTableName(key)
+
+      // Child model
+      if (isPlainObject(value)) {
+        const Model = this.findModelBuilder(rootTableName)
+        const association = Model.associations[tableName]
+
+        if (!association) {
+          if (!Model.getAttributes()[tableName]) {
+            logger.error(`Cannot find association ${tableName} from key ${key} of model ${rootTableName}`, {
+              associations: Object.keys(Model.associations),
+              model: Model.getAttributes()
+            })
+          }
+
+          continue
+        }
+
+        const { created, model: subModel } = this.createModel(value, tableName, `${keyPath}.${json.id}.${key}`)
+
+        if (association.isMultiAssociation && !Array.isArray(model[tableName])) {
+          model[tableName] = []
+        }
+
+        if (!created || !subModel) continue
+
+        if (Array.isArray(model[tableName])) {
+          model[tableName].push(subModel)
+        } else {
+          model[tableName] = subModel
+        }
+      }
+    }
+
+    return { created, model }
+  }
+
+  private createOrFindModel (json: any, tableName: string, keyPath: string) {
+    const registryKey = this.getModelRegistryKey(json, keyPath)
+    if (this.modelRegistry.has(registryKey)) {
+      return {
+        created: false,
+        model: this.modelRegistry.get(registryKey)
+      }
+    }
+
+    const Model = this.findModelBuilder(tableName)
+    if (!Model) {
+      throw new Error(`Cannot find model builder for ${tableName}. You may have to add an alias in ModelBuilder class`)
+    }
+
+    if (!Model) {
+      logger.error(
+        'Cannot build model %s that does not exist',
+        this.buildSequelizeModelName(tableName),
+        { existing: this.sequelize.modelManager.all.map(m => m.name) }
+      )
+      return { created: false, model: null }
+    }
+
+    const model = Model.build(json, { raw: true, isNewRecord: false })
+
+    this.modelRegistry.set(registryKey, model)
+
+    return { created: true, model }
+  }
+
+  private findModelBuilder (tableName: string) {
+    return this.sequelize.modelManager.getModel(this.buildSequelizeModelName(tableName)) as ModelStatic<T>
+  }
+
+  private buildSequelizeModelName (tableNameArg: string) {
+    const tableName = this.buildTableName(tableNameArg)
+
+    return this.tableNameToModelName[tableName] ?? tableName + 'Model'
+  }
+
+  private buildTableName (tableName: string) {
+    return this.aliasToTableName[tableName] ?? tableName
+  }
+
+  private getModelRegistryKey (json: any, keyPath: string) {
+    return keyPath + json.id
+  }
+}

server/core/models/shared/model-cache.ts

@@ -0,0 +1,94 @@
+import { Model } from 'sequelize-typescript'
+import { logger } from '@server/helpers/logger.js'
+
+type ModelCacheType =
+  'server-account'
+  | 'local-actor-name'
+  | 'local-actor-url'
+  | 'load-video-immutable-id'
+  | 'load-video-immutable-url'
+
+type DeleteKey =
+  'video'
+
+class ModelCache {
+
+  private static instance: ModelCache
+
+  private readonly localCache: { [id in ModelCacheType]: Map<string, any> } = {
+    'server-account': new Map(),
+    'local-actor-name': new Map(),
+    'local-actor-url': new Map(),
+    'load-video-immutable-id': new Map(),
+    'load-video-immutable-url': new Map()
+  }
+
+  private readonly deleteIds: {
+    [deleteKey in DeleteKey]: Map<number, { cacheType: ModelCacheType, key: string }[]>
+  } = {
+    video: new Map()
+  }
+
+  private constructor () {
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  doCache<T extends Model> (options: {
+    cacheType: ModelCacheType
+    key: string
+    fun: () => Promise<T>
+    whitelist?: () => boolean
+    deleteKey?: DeleteKey
+  }) {
+    const { cacheType, key, fun, whitelist, deleteKey } = options
+
+    if (whitelist && whitelist() !== true) return fun()
+
+    const cache = this.localCache[cacheType]
+
+    if (cache.has(key)) {
+      logger.debug('Model cache hit for %s -> %s.', cacheType, key)
+      return Promise.resolve<T>(cache.get(key))
+    }
+
+    return fun().then(m => {
+      if (!m) return m
+
+      if (!whitelist || whitelist()) cache.set(key, m)
+
+      if (deleteKey) {
+        const map = this.deleteIds[deleteKey]
+        if (!map.has(m.id)) map.set(m.id, [])
+
+        const a = map.get(m.id)
+        a.push({ cacheType, key })
+      }
+
+      return m
+    })
+  }
+
+  invalidateCache (deleteKey: DeleteKey, modelId: number) {
+    const map = this.deleteIds[deleteKey]
+
+    if (!map.has(modelId)) return
+
+    for (const toDelete of map.get(modelId)) {
+      logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key)
+      this.localCache[toDelete.cacheType].delete(toDelete.key)
+    }
+
+    map.delete(modelId)
+  }
+
+  clearCache (cacheType: ModelCacheType) {
+    this.localCache[cacheType] = new Map()
+  }
+}
+
+export {
+  ModelCache
+}

server/core/models/shared/position.ts

@@ -0,0 +1,78 @@
+import { forceNumber } from '@peertube/peertube-core-utils'
+import { AggregateOptions, Attributes, Model, ModelStatic, Op, Sequelize, Transaction, WhereOptions } from 'sequelize'
+
+export async function getNextPositionOf<T extends Model> (options: {
+  model: ModelStatic<T>
+  columnName: keyof Attributes<T>
+  where: WhereOptions<Attributes<T>>
+  transaction: Transaction
+}) {
+  const { columnName, model, where, transaction } = options
+
+  const query: AggregateOptions<number> = {
+    where,
+    transaction
+  }
+
+  const position = await model.max(columnName, query)
+
+  return position
+    ? position + 1
+    : 1
+}
+
+export function reassignPositionOf<T extends Model> (options: {
+  model: ModelStatic<T>
+  columnName: keyof Attributes<T>
+  where: WhereOptions<Attributes<T>>
+  transaction: Transaction
+
+  firstPosition: number
+  endPosition: number
+  newPosition: number
+}) {
+  const { firstPosition, endPosition, newPosition, model, where, columnName, transaction } = options
+
+  const query = {
+    where: {
+      ...where,
+
+      [columnName]: {
+        [Op.gte]: firstPosition,
+        [Op.lte]: endPosition
+      }
+    },
+    transaction,
+    validate: false // We use a literal to update the position
+  }
+
+  const escapedColumnName = model.sequelize.escape(columnName as string).replace(/'/g, '')
+  const positionQuery = Sequelize.literal(`${forceNumber(newPosition)} + "${escapedColumnName}" - ${forceNumber(firstPosition)}`)
+
+  return model.update({ [columnName]: positionQuery } as Record<typeof columnName, typeof positionQuery>, query)
+}
+
+export function increasePositionOf<T extends Model> (options: {
+  model: ModelStatic<T>
+  columnName: keyof Attributes<T>
+  where: WhereOptions<Attributes<T>>
+  transaction: Transaction
+
+  fromPosition: number
+  by: number
+}) {
+  const { model, where, transaction, fromPosition, by, columnName } = options
+
+  const query = {
+    where: {
+      ...where,
+
+      [columnName]: {
+        [Op.gte]: fromPosition
+      }
+    },
+    transaction
+  }
+
+  return model.increment({ [columnName]: by } as Record<typeof columnName, number>, query)
+}

server/core/models/shared/query.ts

@@ -0,0 +1,91 @@
+import { forceNumber } from '@peertube/peertube-core-utils'
+import { BindOrReplacements, Op, QueryOptionsWithType, QueryTypes, Sequelize, Transaction } from 'sequelize'
+import { Fn } from 'sequelize/types/utils'
+import validator from 'validator'
+
+async function doesExist (options: {
+  sequelize: Sequelize
+  query: string
+  bind?: BindOrReplacements
+  transaction?: Transaction
+}) {
+  const { sequelize, query, bind, transaction } = options
+
+  const queryOptions: QueryOptionsWithType<QueryTypes.SELECT> = {
+    type: QueryTypes.SELECT,
+    bind,
+    raw: true,
+    transaction
+  }
+
+  const results = await sequelize.query(query, queryOptions)
+
+  return results.length === 1
+}
+
+// FIXME: have to specify the result type to not break peertube typings generation
+function createSimilarityAttribute (col: string, value: string): Fn {
+  return Sequelize.fn(
+    'similarity',
+    searchTrigramNormalizeCol(col),
+    searchTrigramNormalizeValue(value)
+  )
+}
+
+function buildWhereIdOrUUID (id: number | string) {
+  return validator.default.isInt('' + id) ? { id } : { uuid: id }
+}
+
+function parseAggregateResult (result: any) {
+  if (!result) return 0
+
+  const total = forceNumber(result)
+  if (isNaN(total)) return 0
+
+  return total
+}
+
+function parseRowCountResult (result: any): number {
+  if (result.length !== 0) return result[0].total
+
+  return 0
+}
+
+function createSafeIn (sequelize: Sequelize, toEscape: (string | number)[], additionalUnescaped: string[] = []) {
+  return toEscape.map(t => {
+    return t === null
+      ? null
+      : sequelize.escape('' + t)
+  }).concat(additionalUnescaped).join(', ')
+}
+
+function searchAttribute (sourceField?: string, targetField?: string) {
+  if (!sourceField) return {}
+
+  return {
+    [targetField]: {
+      // FIXME: ts error
+      [Op.iLike as any]: `%${sourceField}%`
+    }
+  }
+}
+
+export {
+  buildWhereIdOrUUID,
+  createSafeIn,
+  createSimilarityAttribute,
+  doesExist,
+  parseAggregateResult,
+  parseRowCountResult,
+  searchAttribute
+}
+
+// ---------------------------------------------------------------------------
+
+function searchTrigramNormalizeValue (value: string) {
+  return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', value))
+}
+
+function searchTrigramNormalizeCol (col: string) {
+  return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col)))
+}

server/core/models/shared/sequelize-helpers.ts

@@ -0,0 +1,39 @@
+import { Sequelize } from 'sequelize'
+
+function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
+  if (!model.createdAt || !model.updatedAt) {
+    throw new Error('Miss createdAt & updatedAt attributes to model')
+  }
+
+  const now = Date.now()
+  const createdAtTime = model.createdAt.getTime()
+  const updatedAtTime = model.updatedAt.getTime()
+
+  return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
+}
+
+function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value', nullable = false) {
+  if (nullable && (value === null || value === undefined)) return
+
+  if (validator(value) === false) {
+    throw new Error(`"${value}" is not a valid ${fieldName}.`)
+  }
+}
+
+function buildTrigramSearchIndex (indexName: string, attribute: string) {
+  return {
+    name: indexName,
+    // FIXME: gin_trgm_ops is not taken into account in Sequelize 6, so adding it ourselves in the literal function
+    fields: [ Sequelize.literal('lower(immutable_unaccent(' + attribute + ')) gin_trgm_ops') as any ],
+    using: 'gin',
+    operator: 'gin_trgm_ops'
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  throwIfNotValid,
+  buildTrigramSearchIndex,
+  isOutdated
+}

server/core/models/shared/sequelize-type.ts

@@ -0,0 +1,6 @@
+import { AttributesOnly } from '@peertube/peertube-typescript-utils'
+import { Model } from 'sequelize-typescript'
+
+export abstract class SequelizeModel<T> extends Model<Partial<AttributesOnly<T>>> {
+  declare id: number
+}

server/core/models/shared/sort.ts

@@ -0,0 +1,146 @@
+import { literal, OrderItem, Sequelize } from 'sequelize'
+
+// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
+export function getSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  let finalField: string | ReturnType<typeof Sequelize.col>
+
+  if (field.toLowerCase() === 'match') { // Search
+    finalField = Sequelize.col('similarity')
+  } else {
+    finalField = field
+  }
+
+  return [ [ finalField, direction ], lastSort ]
+}
+
+export function getAdminUsersSort (value: string): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  let finalField: string | ReturnType<typeof Sequelize.col>
+
+  if (field === 'videoQuotaUsed') { // Users list
+    finalField = Sequelize.col('videoQuotaUsed')
+  } else {
+    finalField = field
+  }
+
+  const nullPolicy = direction === 'ASC'
+    ? 'NULLS FIRST'
+    : 'NULLS LAST'
+
+  // FIXME: typings
+  return [ [ finalField as any, direction, nullPolicy ], [ 'id', 'ASC' ] ]
+}
+
+export function getPlaylistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field.toLowerCase() === 'name') {
+    return [ [ 'name', direction ], lastSort ]
+  }
+
+  return getSort(value, lastSort)
+}
+
+export function getVideoSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field.toLowerCase() === 'trending') { // Sort by aggregation
+    return [
+      [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoViews.views')), '0'), direction ],
+
+      [ Sequelize.col('VideoModel.views'), direction ],
+
+      lastSort
+    ]
+  } else if (field === 'publishedAt') {
+    return [
+      [ 'ScheduleVideoUpdate', 'updateAt', direction + ' NULLS LAST' ],
+
+      [ Sequelize.col('VideoModel.publishedAt'), direction ],
+
+      lastSort
+    ]
+  }
+
+  let finalField: string | ReturnType<typeof Sequelize.col>
+
+  // Alias
+  if (field.toLowerCase() === 'match') { // Search
+    finalField = Sequelize.col('similarity')
+  } else {
+    finalField = field
+  }
+
+  const firstSort: OrderItem = typeof finalField === 'string'
+    ? finalField.split('.').concat([ direction ]) as OrderItem
+    : [ finalField, direction ]
+
+  return [ firstSort, lastSort ]
+}
+
+export function getBlacklistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  const videoFields = new Set([ 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid' ])
+
+  if (videoFields.has(field)) {
+    return [
+      [ literal(`"Video.${field}" ${direction}`) ],
+      lastSort
+    ] as OrderItem[]
+  }
+
+  return getSort(value, lastSort)
+}
+
+export function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field === 'redundancyAllowed') {
+    return [
+      [ 'ActorFollowing.Server.redundancyAllowed', direction ],
+      lastSort
+    ]
+  }
+
+  return getSort(value, lastSort)
+}
+
+export function getChannelSyncSort (value: string): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+  if (field.toLowerCase() === 'videochannel') {
+    return [
+      [ literal('"VideoChannel.name"'), direction ]
+    ]
+  }
+  return [ [ field, direction ] ]
+}
+
+export function getSubscriptionSort (value: string): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field === 'channelUpdatedAt') {
+    return [
+      [ literal('"ActorFollowing.VideoChannel.updatedAt"'), direction ]
+    ]
+  }
+  return [ [ field, direction ] ]
+}
+
+export function buildSortDirectionAndField (value: string) {
+  let field: string
+  let direction: 'ASC' | 'DESC'
+
+  if (value.startsWith('-')) {
+    direction = 'DESC'
+    field = value.substring(1)
+  } else {
+    direction = 'ASC'
+    field = value
+  }
+
+  return { direction, field }
+}

server/core/models/shared/sql/actor-helpers.ts

@@ -0,0 +1,101 @@
+import { ActorImageType } from '@peertube/peertube-models'
+
+export function getActorJoin (options: {
+  base?: string
+  on: string
+  required: boolean
+  type: 'account' | 'channel'
+  includeAvatars?: boolean // default false
+}) {
+  const { base = '', on, includeAvatars = false, type, required } = options
+
+  const avatarsJoin = includeAvatars
+    ? getAvatarsJoin({ base: `${base}Actor`, on: `"${base}Actor"."id"` })
+    : ''
+
+  const join = required
+    ? 'INNER JOIN'
+    : 'LEFT JOIN'
+
+  const column = type === 'account'
+    ? 'accountId'
+    : 'videoChannelId'
+
+  return ` ${join} "actor" "${base}Actor" ON "${base}Actor"."${column}" = ${on} ` +
+    `LEFT JOIN "server" "${base}Actor->Server" ` +
+    `  ON "${base}Actor->Server"."id" = "${base}Actor"."serverId" ` +
+    avatarsJoin
+}
+
+export function getChannelJoin (options: {
+  base?: string
+  on: string
+  includeAccount: boolean
+  includeAvatars: boolean
+  includeActors: boolean
+  required: boolean
+}) {
+  const { base = '', on, includeAccount, includeAvatars, includeActors, required } = options
+
+  const accountJoin = includeAccount
+    ? getAccountJoin({
+      base: `${base}VideoChannel->`,
+      on: `"${base}VideoChannel"."accountId"`,
+      includeAvatars,
+      includeActor: includeActors,
+      required
+    })
+    : ''
+
+  const actorJoin = includeActors
+    ? getActorJoin({
+      base: `${base}VideoChannel->`,
+      on: `"${base}VideoChannel"."id"`,
+      type: 'channel',
+      includeAvatars,
+      required
+    })
+    : ''
+
+  const join = required
+    ? 'INNER JOIN'
+    : 'LEFT JOIN'
+
+  return ` ${join} "videoChannel" "${base}VideoChannel" ON "${base}VideoChannel"."id" = ${on} ` +
+    actorJoin +
+    accountJoin
+}
+
+export function getAccountJoin (options: {
+  base?: string
+  on: string
+  includeAvatars: boolean
+  includeActor: boolean
+  required: boolean
+}) {
+  const { base = '', on, includeAvatars, includeActor, required } = options
+
+  const actorJoin = includeActor
+    ? getActorJoin({
+      base: `${base}Account->`,
+      on: `"${base}Account"."id"`,
+      type: 'account',
+      includeAvatars,
+      required
+    })
+    : ''
+
+  return ` LEFT JOIN "account" "${base}Account" ON "${base}Account"."id" = ${on} ` +
+    actorJoin
+}
+
+export function getAvatarsJoin (options: {
+  base?: string
+  on: string
+}) {
+  const { base = '', on } = options
+
+  return ` LEFT JOIN "actorImage" "${base}Avatars" ` +
+    `ON "${base}Avatars"."actorId" = ${on} ` +
+    `AND "${base}Avatars"."type" = ${ActorImageType.AVATAR} `
+}

server/core/models/shared/sql/common-helpers.ts

@@ -0,0 +1,39 @@
+import { forceNumber } from '@peertube/peertube-core-utils'
+import { FollowState } from '@peertube/peertube-models'
+import { literal } from 'sequelize'
+import { Literal } from 'sequelize/types/utils'
+
+// FIXME: have to specify the result type to not break peertube typings generation
+export function buildLocalAccountIdsIn (): Literal {
+  return literal(
+    '(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."accountId" = "account"."id" AND "actor"."serverId" IS NULL)'
+  )
+}
+
+// FIXME: have to specify the result type to not break peertube typings generation
+export function buildLocalActorIdsIn (): Literal {
+  return literal(
+    '(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)'
+  )
+}
+
+export function buildBlockedAccountSQL (blockerIds: number[]) {
+  const blockerIdsString = blockerIds.join(', ')
+
+  return 'SELECT "targetAccountId" AS "id" FROM "accountBlocklist" WHERE "accountId" IN (' + blockerIdsString + ')' +
+    ' UNION ' +
+    'SELECT "account"."id" AS "id" FROM account INNER JOIN "actor" ON account."id" = actor."accountId" ' +
+    'INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ' +
+    'WHERE "serverBlocklist"."accountId" IN (' + blockerIdsString + ')'
+}
+
+export function buildServerIdsFollowedBy (actorId: any) {
+  const actorIdNumber = forceNumber(actorId)
+  const followState: FollowState = 'accepted'
+
+  return '(' +
+    'SELECT "actor"."serverId" FROM "actorFollow" ' +
+    'INNER JOIN "actor" ON actor.id = "actorFollow"."targetActorId" ' +
+    `WHERE "actorFollow"."actorId" = ${actorIdNumber} AND "actorFollow"."state" = '${followState}'` +
+    ')'
+}

server/core/models/shared/table.ts

@@ -0,0 +1,44 @@
+import { AttributesOnly } from '@peertube/peertube-typescript-utils'
+import { Model, ModelStatic } from 'sequelize'
+
+export function buildSQLAttributes<M extends Model> (options: {
+  model: ModelStatic<M>
+  tableName: string
+
+  excludeAttributes?: readonly Exclude<keyof AttributesOnly<M>, symbol>[]
+  includeAttributes?: readonly Exclude<keyof AttributesOnly<M>, symbol>[]
+
+  aliasPrefix?: string
+
+  idBuilder?: string[]
+}) {
+  const { model, tableName, aliasPrefix = '', excludeAttributes, includeAttributes, idBuilder } = options
+
+  const attributes = Object.keys(model.getAttributes()) as Exclude<keyof AttributesOnly<M>, symbol>[]
+
+  const builtAttributes = attributes
+    .filter(a => {
+      if (!excludeAttributes) return true
+      if (excludeAttributes.includes(a)) return false
+
+      return true
+    })
+    .filter(a => {
+      if (!includeAttributes) return true
+      if (includeAttributes.includes(a)) return true
+
+      return false
+    })
+    .map(a => {
+      return `"${tableName}"."${a}" AS "${aliasPrefix}${a}"`
+    })
+
+  if (idBuilder) {
+    const idSelect = idBuilder.map(a => `"${tableName}"."${a}"`)
+      .join(` || '-' || `)
+
+    builtAttributes.push(`${idSelect} AS "${aliasPrefix}id"`)
+  }
+
+  return builtAttributes
+}

server/core/models/shared/update.ts

@@ -0,0 +1,30 @@
+import { QueryTypes, Sequelize, Transaction } from 'sequelize'
+
+const updating = new Set<string>()
+
+// Sequelize always skip the update if we only update updatedAt field
+export async function setAsUpdated (options: {
+  sequelize: Sequelize
+  table: string
+  id: number
+  transaction?: Transaction
+}) {
+  const { sequelize, table, id, transaction } = options
+  const key = table + '-' + id
+
+  if (updating.has(key)) return
+  updating.add(key)
+
+  try {
+    await sequelize.query(
+      `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`,
+      {
+        replacements: { table, id, updatedAt: new Date() },
+        type: QueryTypes.UPDATE,
+        transaction
+      }
+    )
+  } finally {
+    updating.delete(key)
+  }
+}