Init commit

This commit is contained in:
ShreejitPanchal
2026-05-19 19:56:02 +08:00
commit 6601501eff
4047 changed files with 2185372 additions and 0 deletions

View File

@@ -0,0 +1,51 @@
import { Activity, ActivityPubActor, ActivityPubOrderedCollection } from '@peertube/peertube-models'
import { Awaitable } from '@peertube/peertube-typescript-utils'
import { MUserDefault } from '@server/types/models/user/user.js'
import { Readable } from 'stream'
export type ExportResult <T> = {
json: T[] | T
staticFiles: {
archivePath: string
readStreamFactory: () => Promise<Readable>
}[]
activityPub?: ActivityPubActor | ActivityPubOrderedCollection<string>
activityPubOutbox?: Omit<Activity, '@context'>[]
}
type ActivityPubFilenames = {
likes: string
dislikes: string
outbox: string
following: string
account: string
}
export abstract class AbstractUserExporter <T> {
protected user: MUserDefault
protected activityPubFilenames: ActivityPubFilenames
protected relativeStaticDirPath: string
constructor (options: {
user: MUserDefault
activityPubFilenames: ActivityPubFilenames
relativeStaticDirPath?: string
}) {
this.user = options.user
this.activityPubFilenames = options.activityPubFilenames
this.relativeStaticDirPath = options.relativeStaticDirPath
}
getActivityPubFilename () {
return null
}
abstract export (): Awaitable<ExportResult<T>>
}

View File

@@ -0,0 +1,68 @@
import { AccountExportJSON, ActivityPubActor, ActorImageType } from '@peertube/peertube-models'
import { MAccountDefault, MActorDefaultBanner } from '@server/types/models/index.js'
import { ActorExporter } from './actor-exporter.js'
import { AccountModel } from '@server/models/account/account.js'
import { getContextFilter } from '@server/lib/activitypub/context.js'
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
import { join } from 'path'
export class AccountExporter extends ActorExporter <AccountExportJSON> {
async export () {
const account = await AccountModel.loadLocalByName(this.user.username)
const { staticFiles, relativePathsFromJSON } = this.exportActorFiles(account.Actor as MActorDefaultBanner)
return {
json: this.exportAccountJSON(account, relativePathsFromJSON),
staticFiles,
activityPub: await this.exportAccountAP(account)
}
}
getActivityPubFilename () {
return this.activityPubFilenames.account
}
// ---------------------------------------------------------------------------
private exportAccountJSON (account: MAccountDefault, archiveFiles: { avatar: string }): AccountExportJSON {
return {
...this.exportActorJSON(account.Actor as MActorDefaultBanner),
displayName: account.getDisplayName(),
description: account.description,
updatedAt: account.updatedAt.toISOString(),
createdAt: account.createdAt.toISOString(),
archiveFiles
}
}
private async exportAccountAP (account: MAccountDefault): Promise<ActivityPubActor> {
const avatar = account.Actor.getMaxQualityImage(ActorImageType.AVATAR)
return activityPubContextify(
{
...await account.toActivityPubObject(),
likes: this.activityPubFilenames.likes,
dislikes: this.activityPubFilenames.dislikes,
outbox: this.activityPubFilenames.outbox,
following: this.activityPubFilenames.following,
icon: avatar
? [
{
...avatar.toActivityPubObject(),
url: join(this.relativeStaticDirPath, this.getAvatarPath(account.Actor, avatar.filename))
}
]
: []
},
'Actor',
getContextFilter())
}
}

View File

@@ -0,0 +1,81 @@
import { ActorImageModel } from '@server/models/actor/actor-image.js'
import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js'
import { ActorImageType } from '@peertube/peertube-models'
import { MActor, MActorDefaultBanner, MActorImage } from '@server/types/models/index.js'
import { extname, join } from 'path'
import { createReadStream } from 'fs'
export abstract class ActorExporter <T> extends AbstractUserExporter<T> {
protected exportActorJSON (actor: MActorDefaultBanner) {
return {
url: actor.url,
name: actor.preferredUsername,
avatars: actor.hasImage(ActorImageType.AVATAR)
? this.exportActorImageJSON(actor.Avatars)
: [],
banners: actor.hasImage(ActorImageType.BANNER)
? this.exportActorImageJSON(actor.Banners)
: []
}
}
protected exportActorImageJSON (images: MActorImage[]) {
return images.map(i => ({
width: i.width,
url: ActorImageModel.getImageUrl(i),
createdAt: i.createdAt.toISOString(),
updatedAt: i.updatedAt.toISOString()
}))
}
// ---------------------------------------------------------------------------
protected exportActorFiles (actor: MActorDefaultBanner) {
const staticFiles: ExportResult<any>['staticFiles'] = []
const relativePathsFromJSON = {
avatar: null as string,
banner: null as string
}
const toProcess = [
{
archivePathBuilder: (filename: string) => this.getBannerPath(actor, filename),
type: ActorImageType.BANNER
},
{
archivePathBuilder: (filename: string) => this.getAvatarPath(actor, filename),
type: ActorImageType.AVATAR
}
]
for (const { archivePathBuilder, type } of toProcess) {
if (!actor.hasImage(type)) continue
const image = actor.getMaxQualityImage(type)
staticFiles.push({
archivePath: archivePathBuilder(image.filename),
readStreamFactory: () => Promise.resolve(createReadStream(image.getPath()))
})
const relativePath = join(this.relativeStaticDirPath, archivePathBuilder(image.filename))
if (type === ActorImageType.AVATAR) relativePathsFromJSON.avatar = relativePath
else if (type === ActorImageType.BANNER) relativePathsFromJSON.banner = relativePath
}
return { staticFiles, relativePathsFromJSON }
}
protected getAvatarPath (actor: MActor, filename: string) {
return join('avatars', actor.preferredUsername + extname(filename))
}
protected getBannerPath (actor: MActor, filename: string) {
return join('banners', actor.preferredUsername + extname(filename))
}
}

View File

@@ -0,0 +1,18 @@
import { AutoTagPoliciesJSON } from '@peertube/peertube-models'
import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
import { AbstractUserExporter } from './abstract-user-exporter.js'
export class AutoTagPoliciesExporter extends AbstractUserExporter <AutoTagPoliciesJSON> {
async export () {
const data = await AutomaticTagger.getAutomaticTagPolicies(this.user.Account)
return {
json: {
reviewComments: data.review.map(name => ({ name }))
} as AutoTagPoliciesJSON,
staticFiles: []
}
}
}

View File

@@ -0,0 +1,24 @@
import { AbstractUserExporter } from './abstract-user-exporter.js'
import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js'
import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js'
import { BlocklistExportJSON } from '@peertube/peertube-models'
export class BlocklistExporter extends AbstractUserExporter <BlocklistExportJSON> {
async export () {
const [ instancesBlocklist, accountsBlocklist ] = await Promise.all([
ServerBlocklistModel.listHostsBlockedBy([ this.user.Account.id ]),
AccountBlocklistModel.listHandlesBlockedBy([ this.user.Account.id ])
])
return {
json: {
instances: instancesBlocklist.map(b => ({ host: b })),
actors: accountsBlocklist.map(h => ({ handle: h }))
} as BlocklistExportJSON,
staticFiles: []
}
}
}

View File

@@ -0,0 +1,78 @@
import { ChannelExportJSON, PlayerChannelSettings } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { MChannelBannerAccountDefault } from '@server/types/models/index.js'
import { MPlayerSetting } from '@server/types/models/video/player-setting.js'
import { ExportResult } from './abstract-user-exporter.js'
import { ActorExporter } from './actor-exporter.js'
export class ChannelsExporter extends ActorExporter<ChannelExportJSON> {
async export () {
const channelsJSON: ChannelExportJSON['channels'] = []
let staticFiles: ExportResult<ChannelExportJSON>['staticFiles'] = []
const channels = await VideoChannelModel.listAllOwnedByAccount(this.user.Account.id)
for (const channel of channels) {
try {
const exported = await this.exportChannel(channel.id)
channelsJSON.push(exported.json)
staticFiles = staticFiles.concat(exported.staticFiles)
} catch (err) {
logger.warn('Cannot export channel %s.', channel.name, { err })
}
}
return {
json: { channels: channelsJSON },
staticFiles
}
}
private async exportChannel (channelId: number) {
const [ channel, playerSettings ] = await Promise.all([
VideoChannelModel.loadAndPopulateAccount(channelId),
PlayerSettingModel.loadByChannelId(channelId)
])
const { relativePathsFromJSON, staticFiles } = this.exportActorFiles(channel.Actor)
return {
json: this.exportChannelJSON(channel, playerSettings, relativePathsFromJSON),
staticFiles
}
}
// ---------------------------------------------------------------------------
private exportChannelJSON (
channel: MChannelBannerAccountDefault,
playerSettings: MPlayerSetting,
archiveFiles: { avatar: string, banner: string }
): ChannelExportJSON['channels'][0] {
return {
...this.exportActorJSON(channel.Actor),
displayName: channel.getDisplayName(),
description: channel.description,
support: channel.support,
playerSettings: this.exportPlayerSettingsJSON(playerSettings),
updatedAt: channel.updatedAt.toISOString(),
createdAt: channel.createdAt.toISOString(),
archiveFiles
}
}
private exportPlayerSettingsJSON (playerSettings: MPlayerSetting) {
if (!playerSettings) return null
return {
theme: playerSettings.theme as PlayerChannelSettings['theme']
}
}
}

View File

@@ -0,0 +1,48 @@
import { CommentsExportJSON, VideoCommentObject } from '@peertube/peertube-models'
import { audiencify, getPublicAudience } from '@server/lib/activitypub/audience.js'
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
import { VideoCommentModel } from '@server/models/video/video-comment.js'
import { MCommentExport } from '@server/types/models/index.js'
import Bluebird from 'bluebird'
import { AbstractUserExporter } from './abstract-user-exporter.js'
export class CommentsExporter extends AbstractUserExporter<CommentsExportJSON> {
async export () {
const comments = await VideoCommentModel.listForExport(this.user.Account.id)
return {
json: {
comments: this.formatCommentsJSON(comments)
},
activityPubOutbox: await this.formatCommentsAP(comments),
staticFiles: []
}
}
private formatCommentsJSON (comments: MCommentExport[]) {
return comments.map(c => ({
url: c.url,
text: c.text,
createdAt: c.createdAt.toISOString(),
videoUrl: c.Video.url,
inReplyToCommentUrl: c?.InReplyToVideoComment?.url
}))
}
private formatCommentsAP (comments: MCommentExport[]) {
return Bluebird.mapSeries(comments, async ({ url }) => {
const comment = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoImmutableAndAccount(url)
const threadParentComments = await VideoCommentModel.listThreadParentComments({ comment })
let commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject
const audience = getPublicAudience(comment.Account.Actor)
commentObject = audiencify(commentObject, audience)
return buildCreateActivity(comment.url, comment.Account.Actor, commentObject, audience)
})
}
}

View File

@@ -0,0 +1,41 @@
import { AbstractUserExporter } from './abstract-user-exporter.js'
import { MAccountVideoRateVideoUrl } from '@server/types/models/index.js'
import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js'
import { ActivityPubOrderedCollection, DislikesExportJSON } from '@peertube/peertube-models'
import { activityPubCollection } from '@server/lib/activitypub/collection.js'
import { getContextFilter } from '@server/lib/activitypub/context.js'
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
export class DislikesExporter extends AbstractUserExporter <DislikesExportJSON> {
async export () {
const dislikes = await AccountVideoRateModel.listRatesOfAccountIdForExport(this.user.Account.id, 'dislike')
return {
json: {
dislikes: this.formatDislikesJSON(dislikes)
} as DislikesExportJSON,
activityPub: await this.formatDislikesAP(dislikes),
staticFiles: []
}
}
getActivityPubFilename () {
return this.activityPubFilenames.dislikes
}
private formatDislikesJSON (dislikes: MAccountVideoRateVideoUrl[]) {
return dislikes.map(o => ({ videoUrl: o.Video.url, createdAt: o.createdAt.toISOString() }))
}
private formatDislikesAP (dislikes: MAccountVideoRateVideoUrl[]): Promise<ActivityPubOrderedCollection<string>> {
return activityPubContextify(
activityPubCollection(this.getActivityPubFilename(), dislikes.map(l => l.Video.url)),
'Rate',
getContextFilter()
)
}
}

View File

@@ -0,0 +1,44 @@
import { AbstractUserExporter } from './abstract-user-exporter.js'
import { FollowersExportJSON } from '@peertube/peertube-models'
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
export class FollowersExporter extends AbstractUserExporter<FollowersExportJSON> {
async export () {
let followersJSON = this.formatFollowersJSON(
await ActorFollowModel.listAcceptedFollowersForExport(this.user.Account.Actor.id),
this.user.Account.Actor.getFullIdentifier()
)
const channels = await VideoChannelModel.listAllOwnedByAccount(this.user.Account.id)
for (const channel of channels) {
followersJSON = followersJSON.concat(
this.formatFollowersJSON(
await ActorFollowModel.listAcceptedFollowersForExport(channel.Actor.id),
channel.Actor.getFullIdentifier()
)
)
}
return {
json: { followers: followersJSON } as FollowersExportJSON,
staticFiles: []
}
}
private formatFollowersJSON (
follows: {
createdAt: Date
followerHandle: string
}[],
targetHandle: string
): FollowersExportJSON['followers'] {
return follows.map(f => ({
targetHandle,
handle: f.followerHandle,
createdAt: f.createdAt.toISOString()
}))
}
}

View File

@@ -0,0 +1,47 @@
import { ActivityPubOrderedCollection, FollowingExportJSON } from '@peertube/peertube-models'
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
import { activityPubCollection } from '@server/lib/activitypub/collection.js'
import { getContextFilter } from '@server/lib/activitypub/context.js'
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
import { AbstractUserExporter } from './abstract-user-exporter.js'
export class FollowingExporter extends AbstractUserExporter<FollowingExportJSON> {
async export () {
const following = await ActorFollowModel.listAcceptedFollowingForExport(this.user.Account.Actor.id)
const followingJSON = this.formatFollowingJSON(following, this.user.Account.Actor.getFullIdentifier())
return {
json: { following: followingJSON } as FollowingExportJSON,
staticFiles: [],
activityPub: await this.formatFollowingAP(following)
}
}
getActivityPubFilename () {
return this.activityPubFilenames.following
}
private formatFollowingJSON (
follows: {
createdAt: Date
followingHandle: string
}[],
handle: string
): FollowingExportJSON['following'] {
return follows.map(f => ({
handle,
targetHandle: f.followingHandle,
createdAt: f.createdAt.toISOString()
}))
}
private formatFollowingAP (follows: { followingUrl: string }[]): Promise<ActivityPubOrderedCollection<string>> {
return activityPubContextify(
activityPubCollection(this.getActivityPubFilename(), follows.map(f => f.followingUrl)),
'Collection',
getContextFilter()
)
}
}

View File

@@ -0,0 +1,15 @@
export * from './abstract-user-exporter.js'
export * from './account-exporter.js'
export * from './auto-tag-policies.js'
export * from './blocklist-exporter.js'
export * from './channels-exporter.js'
export * from './comments-exporter.js'
export * from './dislikes-exporter.js'
export * from './followers-exporter.js'
export * from './following-exporter.js'
export * from './likes-exporter.js'
export * from './user-settings-exporter.js'
export * from './user-video-history-exporter.js'
export * from './video-playlists-exporter.js'
export * from './videos-exporter.js'
export * from './watched-words-lists-exporter.js'

View File

@@ -0,0 +1,40 @@
import { AbstractUserExporter } from './abstract-user-exporter.js'
import { MAccountVideoRateVideoUrl } from '@server/types/models/index.js'
import { ActivityPubOrderedCollection, LikesExportJSON } from '@peertube/peertube-models'
import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js'
import { activityPubCollection } from '@server/lib/activitypub/collection.js'
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
import { getContextFilter } from '@server/lib/activitypub/context.js'
export class LikesExporter extends AbstractUserExporter <LikesExportJSON> {
async export () {
const likes = await AccountVideoRateModel.listRatesOfAccountIdForExport(this.user.Account.id, 'like')
return {
json: {
likes: this.formatLikesJSON(likes)
} as LikesExportJSON,
activityPub: await this.formatLikesAP(likes),
staticFiles: []
}
}
getActivityPubFilename () {
return this.activityPubFilenames.likes
}
private formatLikesJSON (likes: MAccountVideoRateVideoUrl[]) {
return likes.map(o => ({ videoUrl: o.Video.url, createdAt: o.createdAt.toISOString() }))
}
private formatLikesAP (likes: MAccountVideoRateVideoUrl[]): Promise<ActivityPubOrderedCollection<string>> {
return activityPubContextify(
activityPubCollection(this.getActivityPubFilename(), likes.map(l => l.Video.url)),
'Collection',
getContextFilter()
)
}
}

View File

@@ -0,0 +1,33 @@
import { AbstractUserExporter } from './abstract-user-exporter.js'
import { UserSettingsExportJSON } from '@peertube/peertube-models'
export class UserSettingsExporter extends AbstractUserExporter<UserSettingsExportJSON> {
export () {
return {
json: {
email: this.user.email,
emailPublic: this.user.emailPublic,
nsfwPolicy: this.user.nsfwPolicy,
autoPlayVideo: this.user.autoPlayVideo,
autoPlayNextVideo: this.user.autoPlayNextVideo,
autoPlayNextVideoPlaylist: this.user.autoPlayNextVideoPlaylist,
p2pEnabled: this.user.p2pEnabled,
videosHistoryEnabled: this.user.videosHistoryEnabled,
videoLanguages: this.user.videoLanguages,
language: this.user.language,
theme: this.user.theme,
createdAt: this.user.createdAt,
notificationSettings: this.user.NotificationSetting.toFormattedJSON()
},
staticFiles: []
}
}
}

View File

@@ -0,0 +1,23 @@
import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
import { AbstractUserExporter } from './abstract-user-exporter.js'
import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
export class UserVideoHistoryExporter extends AbstractUserExporter <UserVideoHistoryExportJSON> {
async export () {
const videos = await UserVideoHistoryModel.listForExport(this.user)
return {
json: {
watchedVideos: videos.map(v => ({
videoUrl: v.videoUrl,
lastTimecode: v.currentTime,
createdAt: v.createdAt.toISOString(),
updatedAt: v.updatedAt.toISOString()
}))
} as UserVideoHistoryExportJSON,
staticFiles: []
}
}
}

View File

@@ -0,0 +1,75 @@
import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js'
import { VideoPlaylistsExportJSON } from '@peertube/peertube-models'
import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
import { extname, join } from 'path'
import { createReadStream } from 'fs'
import { MThumbnail, MVideoPlaylist } from '@server/types/models/index.js'
export class VideoPlaylistsExporter extends AbstractUserExporter <VideoPlaylistsExportJSON> {
async export () {
const playlistsJSON: VideoPlaylistsExportJSON['videoPlaylists'] = []
const staticFiles: ExportResult<VideoPlaylistsExportJSON>['staticFiles'] = []
const playlists = await VideoPlaylistModel.listPlaylistForExport(this.user.Account.id)
for (const playlist of playlists) {
const elements = await VideoPlaylistElementModel.listElementsForExport(playlist.id)
const archiveFiles = {
thumbnail: null as string
}
if (playlist.hasThumbnail()) {
const thumbnail = playlist.Thumbnail
staticFiles.push({
archivePath: this.getArchiveThumbnailPath(playlist, thumbnail),
readStreamFactory: () => Promise.resolve(createReadStream(thumbnail.getPath()))
})
archiveFiles.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailPath(playlist, thumbnail))
}
playlistsJSON.push({
displayName: playlist.name,
description: playlist.description,
privacy: playlist.privacy,
url: playlist.url,
uuid: playlist.uuid,
type: playlist.type,
channel: {
name: playlist.VideoChannel?.Actor?.preferredUsername
},
createdAt: playlist.createdAt.toISOString(),
updatedAt: playlist.updatedAt.toISOString(),
thumbnailUrl: playlist.Thumbnail?.getOriginFileUrl(playlist),
elements: elements.map(e => ({
videoUrl: e.Video.url,
startTimestamp: e.startTimestamp,
stopTimestamp: e.stopTimestamp
})),
archiveFiles
})
}
return {
json: {
videoPlaylists: playlistsJSON
},
staticFiles
}
}
private getArchiveThumbnailPath (playlist: MVideoPlaylist, thumbnail: MThumbnail) {
return join('thumbnails', playlist.uuid + extname(thumbnail.filename))
}
}

View File

@@ -0,0 +1,483 @@
import { pick, sortBy } from '@peertube/peertube-core-utils'
import { ActivityCreate, FileStorage, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { audiencify, getVideoAudience } from '@server/lib/activitypub/audience.js'
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
import {
getCaptionReadStream,
getHLSFileReadStream,
getOriginalFileReadStream,
getWebVideoFileReadStream
} from '@server/lib/object-storage/videos.js'
import { VideoDownload } from '@server/lib/video-download.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
import {
MStreamingPlaylistFiles,
MThumbnail,
MVideo,
MVideoAP,
MVideoCaption,
MVideoCaptionLanguageUrl,
MVideoChapter,
MVideoFile,
MVideoFullLight,
MVideoLiveWithSettingSchedules,
MVideoPassword
} from '@server/types/models/index.js'
import { MPlayerSetting } from '@server/types/models/video/player-setting.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import Bluebird from 'bluebird'
import { createReadStream } from 'fs'
import { extname, join } from 'path'
import { PassThrough, Readable } from 'stream'
import { AbstractUserExporter, ExportResult } from './abstract-user-exporter.js'
export class VideosExporter extends AbstractUserExporter<VideoExportJSON> {
constructor (
private readonly options: ConstructorParameters<typeof AbstractUserExporter<VideoExportJSON>>[0] & {
withVideoFiles: boolean
}
) {
super(options)
}
async export () {
const videosJSON: VideoExportJSON['videos'] = []
const activityPubOutbox: ActivityCreate<VideoObject>[] = []
let staticFiles: ExportResult<VideoExportJSON>['staticFiles'] = []
let videoIds: number[] = []
let start = 0
const chunkSize = 100
do {
videoIds = await VideoModel.getAllIdsByAccount({ account: this.user.Account, start, count: chunkSize })
start += videoIds.length
await Bluebird.map(videoIds, async id => {
try {
const exported = await this.exportVideo(id)
videosJSON.push(exported.json)
staticFiles = staticFiles.concat(exported.staticFiles)
activityPubOutbox.push(exported.activityPubOutbox)
} catch (err) {
logger.warn('Cannot export video %d.', id, { err })
}
}, { concurrency: 10 })
} while (videoIds.length === chunkSize)
return {
json: { videos: sortBy(videosJSON, 'publishedAt') },
activityPubOutbox,
staticFiles
}
}
private async exportVideo (videoId: number) {
const [ video, captions, source, chapters, playerSettings ] = await Promise.all([
VideoModel.loadFull(videoId),
VideoCaptionModel.listVideoCaptions(videoId),
VideoSourceModel.loadLatest(videoId),
VideoChapterModel.listChaptersOfVideo(videoId),
PlayerSettingModel.loadByVideoId(videoId)
])
const passwords = video.privacy === VideoPrivacy.PASSWORD_PROTECTED
? (await VideoPasswordModel.listPasswords({ videoId, start: 0, count: undefined, sort: 'createdAt' })).data
: []
const live = video.isLive
? await VideoLiveModel.loadByVideoIdFull(videoId)
: undefined // We already have captions, so we can set it to the video object
;(video as any).VideoCaptions = captions
// Then fetch more attributes for AP serialization
const videoAP = await video.lightAPToFullAP(undefined)
const { relativePathsFromJSON, staticFiles, exportedVideoFileOrSource } = await this.exportVideoFiles({ video, captions })
return {
json: this.exportVideoJSON({
video,
captions,
live,
passwords,
source,
chapters,
playerSettings,
archiveFiles: relativePathsFromJSON
}),
staticFiles,
relativePathsFromJSON,
activityPubOutbox: await this.exportVideoAP(videoAP, chapters, exportedVideoFileOrSource)
}
}
// ---------------------------------------------------------------------------
private exportVideoJSON (options: {
video: MVideoFullLight
captions: MVideoCaption[]
live: MVideoLiveWithSettingSchedules
passwords: MVideoPassword[]
source: MVideoSource
playerSettings: MPlayerSetting
chapters: MVideoChapter[]
archiveFiles: VideoExportJSON['videos'][0]['archiveFiles']
}): VideoExportJSON['videos'][0] {
const { video, captions, live, passwords, source, chapters, playerSettings, archiveFiles } = options
return {
uuid: video.uuid,
createdAt: video.createdAt.toISOString(),
updatedAt: video.updatedAt.toISOString(),
publishedAt: video.publishedAt.toISOString(),
originallyPublishedAt: video.originallyPublishedAt
? video.originallyPublishedAt.toISOString()
: undefined,
name: video.name,
category: video.category,
licence: video.licence,
language: video.language,
tags: video.Tags.map(t => t.name),
privacy: video.privacy,
passwords: passwords.map(p => p.password),
duration: video.duration,
description: video.description,
support: video.support,
isLive: video.isLive,
live: this.exportLiveJSON(video, live),
url: video.url,
thumbnailUrl: video.getMiniature()?.getOriginFileUrl(video) || null,
previewUrl: video.getPreview()?.getOriginFileUrl(video) || null,
views: video.views,
likes: video.likes,
dislikes: video.dislikes,
nsfw: video.nsfw,
commentsPolicy: video.commentsPolicy,
downloadEnabled: video.downloadEnabled,
waitTranscoding: video.waitTranscoding,
state: video.state,
channel: {
name: video.VideoChannel.Actor.preferredUsername
},
captions: this.exportCaptionsJSON(video, captions),
chapters: this.exportChaptersJSON(chapters),
files: this.exportFilesJSON(video, video.VideoFiles),
streamingPlaylists: this.exportStreamingPlaylistsJSON(video, video.VideoStreamingPlaylists),
source: this.exportVideoSourceJSON(source),
playerSettings: this.exportPlayerSettingsJSON(playerSettings),
archiveFiles
}
}
private exportLiveJSON (video: MVideo, live: MVideoLiveWithSettingSchedules) {
if (!video.isLive) return undefined
return {
saveReplay: live.saveReplay,
permanentLive: live.permanentLive,
latencyMode: live.latencyMode,
streamKey: live.streamKey,
replaySettings: live.ReplaySetting
? { privacy: live.ReplaySetting.privacy }
: undefined,
schedules: live.LiveSchedules?.map(s => ({
startAt: s.startAt.toISOString()
}))
}
}
private exportCaptionsJSON (video: MVideo, captions: MVideoCaption[]) {
return captions.map(c => ({
createdAt: c.createdAt.toISOString(),
updatedAt: c.updatedAt.toISOString(),
language: c.language,
filename: c.filename,
automaticallyGenerated: c.automaticallyGenerated,
fileUrl: c.getFileUrl(video)
}))
}
private exportChaptersJSON (chapters: MVideoChapter[]) {
return chapters.map(c => ({
timecode: c.timecode,
title: c.title
}))
}
private exportFilesJSON (video: MVideo, files: MVideoFile[]) {
return files.map(f => ({
resolution: f.resolution,
size: f.size,
fps: f.fps,
torrentUrl: f.getTorrentUrl(),
fileUrl: f.getFileUrl(video)
}))
}
private exportStreamingPlaylistsJSON (video: MVideo, streamingPlaylists: MStreamingPlaylistFiles[]) {
return streamingPlaylists.map(p => ({
type: p.type,
playlistUrl: p.getMasterPlaylistUrl(video),
segmentsSha256Url: p.getSha256SegmentsUrl(video),
files: this.exportFilesJSON(video, p.VideoFiles)
}))
}
private exportVideoSourceJSON (source: MVideoSource) {
if (!source) return null
return {
inputFilename: source.inputFilename,
resolution: source.resolution,
size: source.size,
width: source.width,
height: source.height,
fps: source.fps,
metadata: source.metadata
}
}
private exportPlayerSettingsJSON (playerSettings: MPlayerSetting) {
if (!playerSettings) return null
return {
theme: playerSettings.theme
}
}
// ---------------------------------------------------------------------------
private async exportVideoAP (
video: MVideoAP,
chapters: MVideoChapter[],
exportedVideoFileOrSource: MVideoFile | MVideoSource
): Promise<ActivityCreate<VideoObject>> {
const icon = video.getPreview()
const audience = getVideoAudience({
account: video.VideoChannel.Account,
channel: video.VideoChannel,
privacy: video.privacy,
skipPrivacyCheck: true
})
const videoObject: VideoObject = {
...audiencify(await video.toActivityPubObject(), audience),
icon: [
{
...icon.toActivityPubObject(video),
url: join(this.options.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, icon))
}
],
subtitleLanguage: video.VideoCaptions.map(c => ({
...c.toActivityPubObject(video),
url: [
{
mediaType: 'text/vtt',
type: 'Link',
href: join(this.options.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, c))
}
]
})),
hasParts: buildChaptersAPHasPart(video, chapters),
attachment: this.options.withVideoFiles && exportedVideoFileOrSource
? [
{
type: 'Video' as 'Video',
url: join(this.options.relativeStaticDirPath, this.getArchiveVideoFilePath(video, exportedVideoFileOrSource)),
// FIXME: typings
...pick((exportedVideoFileOrSource as MVideoFile & MVideoSource).toActivityPubObject(video), [
'mediaType',
'height',
'size',
'fps'
])
}
]
: undefined
}
return buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
}
// ---------------------------------------------------------------------------
private async exportVideoFiles (options: {
video: MVideoFullLight
captions: MVideoCaption[]
}) {
const { video, captions } = options
const staticFiles: ExportResult<VideoExportJSON>['staticFiles'] = []
let exportedVideoFileOrSource: MVideoFile | MVideoSource
const relativePathsFromJSON = {
videoFile: null as string,
thumbnail: null as string,
captions: {} as { [lang: string]: string }
}
if (this.options.withVideoFiles) {
const { source, videoFile, separatedAudioFile } = await this.getArchiveVideo(video)
if (source || videoFile || separatedAudioFile) {
const videoPath = this.getArchiveVideoFilePath(video, source || videoFile || separatedAudioFile)
staticFiles.push({
archivePath: videoPath,
// Prefer using original file if possible
readStreamFactory: () =>
source?.keptOriginalFilename
? this.generateVideoSourceReadStream(source)
: this.generateVideoFileReadStream({ video, videoFile, separatedAudioFile })
})
relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, videoPath)
exportedVideoFileOrSource = source?.keptOriginalFilename
? source
: videoFile || separatedAudioFile
}
}
for (const caption of captions) {
staticFiles.push({
archivePath: this.getArchiveCaptionFilePath(video, caption),
readStreamFactory: () => this.generateCaptionReadStream(caption)
})
relativePathsFromJSON.captions[caption.language] = join(this.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, caption))
}
const thumbnail = video.getPreview() || video.getMiniature()
if (thumbnail) {
staticFiles.push({
archivePath: this.getArchiveThumbnailFilePath(video, thumbnail),
readStreamFactory: () => Promise.resolve(createReadStream(thumbnail.getPath()))
})
relativePathsFromJSON.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, thumbnail))
}
return { staticFiles, relativePathsFromJSON, exportedVideoFileOrSource }
}
private async generateVideoSourceReadStream (source: MVideoSource): Promise<Readable> {
if (source.storage === FileStorage.FILE_SYSTEM) {
return createReadStream(VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename))
}
const { stream } = await getOriginalFileReadStream({ keptOriginalFilename: source.keptOriginalFilename, rangeHeader: undefined })
return stream
}
private async generateVideoFileReadStream (options: {
videoFile: MVideoFile
separatedAudioFile: MVideoFile
video: MVideoFullLight
}): Promise<Readable> {
const { video, videoFile, separatedAudioFile } = options
if (separatedAudioFile) {
const stream = new PassThrough()
new VideoDownload({ video, videoFiles: [ videoFile, separatedAudioFile ] })
.muxToMergeVideoFiles(stream)
.catch(err => logger.error('Cannot mux video files', { err }))
return Promise.resolve(stream)
}
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
return createReadStream(VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile))
}
const { stream } = videoFile.isHLS()
? await getHLSFileReadStream({ playlist: video.getHLSPlaylist(), filename: videoFile.filename, rangeHeader: undefined })
: await getWebVideoFileReadStream({ filename: videoFile.filename, rangeHeader: undefined })
return stream
}
private async generateCaptionReadStream (caption: MVideoCaption): Promise<Readable> {
if (caption.storage === FileStorage.FILE_SYSTEM) {
return createReadStream(caption.getFSFilePath())
}
const { stream } = await getCaptionReadStream({ filename: caption.filename, rangeHeader: undefined })
return stream
}
// ---------------------------------------------------------------------------
private async getArchiveVideo (video: MVideoFullLight) {
const source = await VideoSourceModel.loadLatest(video.id)
const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
if (source?.keptOriginalFilename) return { source }
return { videoFile, separatedAudioFile }
}
private getArchiveVideoFilePath (video: MVideo, file: { keptOriginalFilename?: string, filename?: string }) {
return join('video-files', video.uuid + extname(file.keptOriginalFilename || file.filename))
}
private getArchiveCaptionFilePath (video: MVideo, caption: MVideoCaptionLanguageUrl) {
return join('captions', video.uuid + '-' + caption.language + extname(caption.filename))
}
private getArchiveThumbnailFilePath (video: MVideo, thumbnail: MThumbnail) {
return join('thumbnails', video.uuid + extname(thumbnail.filename))
}
}

View File

@@ -0,0 +1,23 @@
import { WatchedWordsListsJSON } from '@peertube/peertube-models'
import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
import { AbstractUserExporter } from './abstract-user-exporter.js'
export class WatchedWordsListsExporter extends AbstractUserExporter <WatchedWordsListsJSON> {
async export () {
const data = await WatchedWordsListModel.listForExport({ accountId: this.user.Account.id })
return {
json: {
watchedWordLists: data.map(list => ({
listName: list.listName,
words: list.words,
createdAt: list.createdAt.toISOString(),
updatedAt: list.updatedAt.toISOString()
}))
} as WatchedWordsListsJSON,
staticFiles: []
}
}
}

View File

@@ -0,0 +1,33 @@
import { AbstractUserImporter } from './abstract-user-importer.js'
import { VideoRateType } from '@peertube/peertube-models'
import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
import { userRateVideo } from '@server/lib/rate.js'
import { VideoModel } from '@server/models/video/video.js'
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
import { pick } from '@peertube/peertube-core-utils'
export type SanitizedRateObject = { videoUrl: string }
export abstract class AbstractRatesImporter <ROOT_OBJECT, OBJECT> extends AbstractUserImporter <ROOT_OBJECT, OBJECT, SanitizedRateObject> {
protected sanitizeRate <O extends { videoUrl: string }> (data: O) {
if (!isUrlValid(data.videoUrl)) return undefined
return pick(data, [ 'videoUrl' ])
}
protected async importRate (data: SanitizedRateObject, rateType: VideoRateType) {
const videoUrl = data.videoUrl
const videoImmutable = await loadOrCreateVideoIfAllowedForUser(videoUrl)
if (!videoImmutable) {
throw new Error(`Cannot get or create video ${videoUrl} to import user ${rateType}`)
}
const video = await VideoModel.loadFull(videoImmutable.id)
await userRateVideo({ account: this.user.Account, rateType, video })
return { duplicate: false }
}
}

View File

@@ -0,0 +1,123 @@
import { getFileSize } from '@peertube/peertube-node-utils'
import { Awaitable } from '@peertube/peertube-typescript-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { MUserDefault } from '@server/types/models/user/user.js'
import { pathExists, readJSON, remove } from 'fs-extra/esm'
import { dirname, resolve } from 'path'
const lTags = loggerTagsFactory('user-import')
export abstract class AbstractUserImporter <
ROOT_OBJECT,
OBJECT extends { archiveFiles?: Record<string, string | Record<string, string>> },
SANITIZED_OBJECT
> {
protected user: MUserDefault
protected extractedDirectory: string
protected jsonFilePath: string
constructor (options: {
user: MUserDefault
extractedDirectory: string
jsonFilePath: string
}) {
this.user = options.user
this.extractedDirectory = options.extractedDirectory
this.jsonFilePath = options.jsonFilePath
}
getJSONFilePath () {
return this.jsonFilePath
}
protected getSafeArchivePathOrThrow (path: string) {
if (!path) return undefined
const resolved = resolve(dirname(this.jsonFilePath), path)
if (resolved.startsWith(this.extractedDirectory) !== true) {
throw new Error(`Static file path ${resolved} is outside the archive directory ${this.extractedDirectory}`)
}
return resolved
}
protected async cleanupImportedStaticFilePaths (archiveFiles: Record<string, string | Record<string, string>>) {
if (!archiveFiles || typeof archiveFiles !== 'object') return
for (const file of Object.values(archiveFiles)) {
if (!file) continue
try {
if (typeof file === 'string') {
await remove(this.getSafeArchivePathOrThrow(file))
} else { // Avoid recursion to prevent security issue
for (const subFile of Object.values(file)) {
await remove(this.getSafeArchivePathOrThrow(subFile))
}
}
} catch (err) {
logger.error(`Cannot remove file ${file} after successful import`, { err, ...lTags() })
}
}
}
protected async isFileValidOrLog (filePath: string, maxSize: number) {
if (!await pathExists(filePath)) {
logger.warn(`Do not import file ${filePath} that do not exist in zip`, lTags())
return false
}
const size = await getFileSize(filePath)
if (size > maxSize) {
logger.warn(
`Do not import too big file ${filePath} (${size} > ${maxSize})`,
lTags()
)
return false
}
return true
}
async import () {
const importData: ROOT_OBJECT = await readJSON(this.jsonFilePath)
const summary = {
duplicates: 0,
success: 0,
errors: 0
}
for (const importObject of this.getImportObjects(importData)) {
try {
const sanitized = this.sanitize(importObject)
if (!sanitized) {
logger.warn('Do not import object after invalid sanitization', { importObject, ...lTags() })
summary.errors++
continue
}
const result = await this.importObject(sanitized)
await this.cleanupImportedStaticFilePaths(importObject.archiveFiles)
if (result.duplicate === true) summary.duplicates++
else summary.success++
} catch (err) {
logger.error('Cannot import object from ' + this.jsonFilePath, { err, importObject, ...lTags() })
summary.errors++
}
}
return summary
}
protected abstract getImportObjects (object: ROOT_OBJECT): OBJECT[]
protected abstract sanitize (object: OBJECT): SANITIZED_OBJECT | undefined
protected abstract importObject (object: SANITIZED_OBJECT): Awaitable<{ duplicate: boolean }>
}

View File

@@ -0,0 +1,66 @@
import { BlocklistExportJSON } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
import { addAccountInBlocklist, addServerInBlocklist } from '@server/lib/blocklist.js'
import { ServerModel } from '@server/models/server/server.js'
import { AccountModel } from '@server/models/account/account.js'
import { isValidActorHandle } from '@server/helpers/custom-validators/activitypub/actor.js'
import { isHostValid } from '@server/helpers/custom-validators/servers.js'
import { pick } from '@peertube/peertube-core-utils'
const lTags = loggerTagsFactory('user-import')
type ImportObject = { handle: string | null, host: string | null, archiveFiles?: never }
export class BlocklistImporter extends AbstractUserImporter<BlocklistExportJSON, ImportObject, ImportObject> {
protected getImportObjects (json: BlocklistExportJSON) {
return [
...json.actors.map(o => ({ handle: o.handle, host: null })),
...json.instances.map(o => ({ handle: null, host: o.host }))
]
}
protected sanitize (blocklistImportData: ImportObject) {
if (!isValidActorHandle(blocklistImportData.handle) && !isHostValid(blocklistImportData.host)) return undefined
return pick(blocklistImportData, [ 'handle', 'host' ])
}
protected async importObject (blocklistImportData: ImportObject) {
if (blocklistImportData.handle) {
await this.importAccountBlock(blocklistImportData.handle)
} else {
await this.importServerBlock(blocklistImportData.host)
}
return { duplicate: false }
}
private async importAccountBlock (handle: string) {
const accountToBlock = await AccountModel.loadByHandle(handle)
if (!accountToBlock) {
logger.info('Account %s was not blocked on user import because it cannot be found in the database.', handle, lTags())
return
}
await addAccountInBlocklist({
byAccountId: this.user.Account.id,
targetAccountId: accountToBlock.id,
removeNotificationOfUserId: this.user.id
})
logger.info('Account %s blocked on user import.', handle, lTags())
}
private async importServerBlock (hostToBlock: string) {
const serverToBlock = await ServerModel.loadOrCreateByHost(hostToBlock)
await addServerInBlocklist({
byAccountId: this.user.Account.id,
targetServerId: serverToBlock.id,
removeNotificationOfUserId: this.user.id
})
logger.info('Server %s blocked on user import.', hostToBlock, lTags())
}
}

View File

@@ -0,0 +1,57 @@
import { AccountExportJSON, ActorImageType } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
import { updateLocalActorImageFiles } from '@server/lib/local-actor.js'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { MAccountDefault } from '@server/types/models/index.js'
import { isUserDescriptionValid, isUserDisplayNameValid } from '@server/helpers/custom-validators/users.js'
import { pick } from '@peertube/peertube-core-utils'
const lTags = loggerTagsFactory('user-import')
type SanitizedObject = Pick<AccountExportJSON, 'description' | 'displayName' | 'archiveFiles'>
export class AccountImporter extends AbstractUserImporter <AccountExportJSON, AccountExportJSON, SanitizedObject> {
protected getImportObjects (json: AccountExportJSON) {
return [ json ]
}
protected sanitize (blocklistImportData: AccountExportJSON) {
if (!isUserDisplayNameValid(blocklistImportData.displayName)) return undefined
if (!isUserDescriptionValid(blocklistImportData.description)) blocklistImportData.description = null
return pick(blocklistImportData, [ 'displayName', 'description', 'archiveFiles' ])
}
protected async importObject (accountImportData: SanitizedObject) {
const account = this.user.Account
account.name = accountImportData.displayName
account.description = accountImportData.description
await saveInTransactionWithRetries(account)
await this.importAvatar(account, accountImportData)
logger.info('Account %s imported.', account.name, lTags())
return { duplicate: false }
}
private async importAvatar (account: MAccountDefault, accountImportData: SanitizedObject) {
const avatarPath = this.getSafeArchivePathOrThrow(accountImportData.archiveFiles.avatar)
if (!avatarPath) return undefined
if (!await this.isFileValidOrLog(avatarPath, CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max)) return undefined
await updateLocalActorImageFiles({
accountOrChannel: account,
imagePhysicalFile: { path: avatarPath },
type: ActorImageType.AVATAR,
sendActorUpdate: false
})
}
}

View File

@@ -0,0 +1,97 @@
import { pick } from '@peertube/peertube-core-utils'
import { ActorImageType, ChannelExportJSON } from '@peertube/peertube-models'
import { isPlayerChannelThemeSettingValid } from '@server/helpers/custom-validators/player-settings.js'
import {
isVideoChannelDescriptionValid,
isVideoChannelDisplayNameValid,
isVideoChannelSupportValid,
isVideoChannelUsernameValid
} from '@server/helpers/custom-validators/video-channels.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { JobQueue } from '@server/lib/job-queue/job-queue.js'
import { updateLocalActorImageFiles } from '@server/lib/local-actor.js'
import { createLocalVideoChannelWithoutKeys } from '@server/lib/video-channel.js'
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { MChannelId } from '@server/types/models/index.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
const lTags = loggerTagsFactory('user-import')
type SanitizedObject = Pick<
ChannelExportJSON['channels'][0],
'name' | 'displayName' | 'description' | 'support' | 'playerSettings' | 'archiveFiles'
>
export class ChannelsImporter extends AbstractUserImporter<ChannelExportJSON, ChannelExportJSON['channels'][0], SanitizedObject> {
protected getImportObjects (json: ChannelExportJSON) {
return json.channels
}
protected sanitize (channelImportData: ChannelExportJSON['channels'][0]) {
if (!isVideoChannelUsernameValid(channelImportData.name)) return undefined
if (!isVideoChannelDisplayNameValid(channelImportData.displayName)) return undefined
if (!isVideoChannelDescriptionValid(channelImportData.description)) channelImportData.description = null
if (!isVideoChannelSupportValid(channelImportData.support)) channelImportData.support = null
if (channelImportData.playerSettings) {
if (!isPlayerChannelThemeSettingValid(channelImportData.playerSettings.theme)) channelImportData.playerSettings.theme = undefined
}
return pick(channelImportData, [ 'name', 'displayName', 'description', 'support', 'playerSettings', 'archiveFiles' ])
}
protected async importObject (channelImportData: SanitizedObject) {
const account = this.user.Account
const existingChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(channelImportData.name)
if (existingChannel) {
logger.info(`Do not import channel ${existingChannel.name} that already exists on this PeerTube instance`, lTags())
} else {
const videoChannelCreated = await sequelizeTypescript.transaction(async t => {
return createLocalVideoChannelWithoutKeys(pick(channelImportData, [ 'displayName', 'name', 'description', 'support' ]), account, t)
})
await this.importPlayerSettings(videoChannelCreated, channelImportData)
await JobQueue.Instance.createJob({ type: 'actor-keys', payload: { actorId: videoChannelCreated.Actor.id } })
for (const type of [ ActorImageType.AVATAR, ActorImageType.BANNER ]) {
const relativePath = type === ActorImageType.AVATAR
? channelImportData.archiveFiles.avatar
: channelImportData.archiveFiles.banner
if (!relativePath) continue
const absolutePath = this.getSafeArchivePathOrThrow(relativePath)
if (!await this.isFileValidOrLog(absolutePath, CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max)) continue
await updateLocalActorImageFiles({
accountOrChannel: videoChannelCreated,
imagePhysicalFile: { path: absolutePath },
type,
sendActorUpdate: false
})
}
logger.info('Video channel %s imported.', channelImportData.name, lTags())
}
return {
duplicate: !!existingChannel
}
}
private async importPlayerSettings (channel: MChannelId, channelImportData: SanitizedObject) {
const playerSettings = channelImportData.playerSettings
if (!playerSettings?.theme) return
await PlayerSettingModel.create({
theme: playerSettings.theme,
channelId: channel.id
})
}
}

View File

@@ -0,0 +1,17 @@
import { DislikesExportJSON } from '@peertube/peertube-models'
import { AbstractRatesImporter, SanitizedRateObject } from './abstract-rates-importer.js'
export class DislikesImporter extends AbstractRatesImporter <DislikesExportJSON, DislikesExportJSON['dislikes'][0]> {
protected getImportObjects (json: DislikesExportJSON) {
return json.dislikes
}
protected sanitize (o: DislikesExportJSON['dislikes'][0]) {
return this.sanitizeRate(o)
}
protected async importObject (dislikesImportData: SanitizedRateObject) {
return this.importRate(dislikesImportData, 'dislike')
}
}

View File

@@ -0,0 +1,40 @@
import { FollowingExportJSON } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
import { JobQueue } from '@server/lib/job-queue/job-queue.js'
import { isValidActorHandle } from '@server/helpers/custom-validators/activitypub/actor.js'
import { pick } from '@peertube/peertube-core-utils'
const lTags = loggerTagsFactory('user-import')
type SanitizedObject = Pick<FollowingExportJSON['following'][0], 'targetHandle'>
export class FollowingImporter extends AbstractUserImporter <FollowingExportJSON, FollowingExportJSON['following'][0], SanitizedObject> {
protected getImportObjects (json: FollowingExportJSON) {
return json.following
}
protected sanitize (followingImportData: FollowingExportJSON['following'][0]) {
if (!isValidActorHandle(followingImportData.targetHandle)) return undefined
return pick(followingImportData, [ 'targetHandle' ])
}
protected async importObject (followingImportData: SanitizedObject) {
const [ name, host ] = followingImportData.targetHandle.split('@')
const payload = {
name,
host,
assertIsChannel: true,
followerActorId: this.user.Account.Actor.id
}
await JobQueue.Instance.createJob({ type: 'activitypub-follow', payload })
logger.info('Subscription job of %s created on user import.', followingImportData.targetHandle, lTags())
return { duplicate: false }
}
}

View File

@@ -0,0 +1,10 @@
export * from './account-blocklist-importer.js'
export * from './account-importer.js'
export * from './channels-importer.js'
export * from './dislikes-importer.js'
export * from './following-importer.js'
export * from './likes-importer.js'
export * from './user-settings-importer.js'
export * from './user-video-history-importer.js'
export * from './video-playlists-importer.js'
export * from './videos-importer.js'

View File

@@ -0,0 +1,17 @@
import { LikesExportJSON } from '@peertube/peertube-models'
import { AbstractRatesImporter, SanitizedRateObject } from './abstract-rates-importer.js'
export class LikesImporter extends AbstractRatesImporter <LikesExportJSON, LikesExportJSON['likes'][0]> {
protected getImportObjects (json: LikesExportJSON) {
return json.likes
}
protected sanitize (o: LikesExportJSON['likes'][0]) {
return this.sanitizeRate(o)
}
protected async importObject (likesImportData: SanitizedRateObject) {
return this.importRate(likesImportData, 'like')
}
}

View File

@@ -0,0 +1,30 @@
import { AutoTagPoliciesJSON, AutomaticTagPolicy } from '@peertube/peertube-models'
import { isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
import { setAccountAutomaticTagsPolicy } from '@server/lib/automatic-tags/automatic-tags.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
type SanitizedObject = AutoTagPoliciesJSON['reviewComments']
export class ReviewCommentsTagPoliciesImporter
extends AbstractUserImporter<AutoTagPoliciesJSON, AutoTagPoliciesJSON['reviewComments'] & { archiveFiles?: never }, SanitizedObject>
{
protected getImportObjects (json: AutoTagPoliciesJSON) {
if (!json.reviewComments) return []
return [ json.reviewComments ]
}
protected sanitize (data: AutoTagPoliciesJSON['reviewComments']) {
return data.filter(d => isWatchedWordListNameValid(d.name))
}
protected async importObject (data: SanitizedObject) {
await setAccountAutomaticTagsPolicy({
account: this.user.Account,
policy: AutomaticTagPolicy.REVIEW_COMMENT,
tags: data.map(v => v.name)
})
return { duplicate: false }
}
}

View File

@@ -0,0 +1,116 @@
import { pick } from '@peertube/peertube-core-utils'
import { UserNotificationSetting, UserSettingsExportJSON } from '@peertube/peertube-models'
import { exists } from '@server/helpers/custom-validators/misc.js'
import { isThemeNameValid } from '@server/helpers/custom-validators/plugins.js'
import { isUserNotificationSettingValid } from '@server/helpers/custom-validators/user-notifications.js'
import {
isUserAutoPlayNextVideoPlaylistValid,
isUserAutoPlayNextVideoValid,
isUserAutoPlayVideoValid,
isUserLanguage,
isUserNSFWPolicyValid,
isUserP2PEnabledValid,
isUserVideoLanguages,
isUserVideosHistoryEnabledValid
} from '@server/helpers/custom-validators/users.js'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { isThemeRegistered } from '@server/lib/plugins/theme-utils.js'
import { UserNotificationSettingModel } from '@server/models/user/user-notification-setting.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
const lTags = loggerTagsFactory('user-import')
type SanitizedObject = Pick<
UserSettingsExportJSON,
| 'nsfwPolicy'
| 'autoPlayVideo'
| 'autoPlayNextVideo'
| 'autoPlayNextVideoPlaylist'
| 'p2pEnabled'
| 'videosHistoryEnabled'
| 'videoLanguages'
| 'language'
| 'theme'
| 'notificationSettings'
>
export class UserSettingsImporter extends AbstractUserImporter<UserSettingsExportJSON, UserSettingsExportJSON, SanitizedObject> {
protected getImportObjects (json: UserSettingsExportJSON) {
return [ json ]
}
protected sanitize (o: UserSettingsExportJSON) {
if (!isUserNSFWPolicyValid(o.nsfwPolicy)) o.nsfwPolicy = undefined
if (!isUserAutoPlayVideoValid(o.autoPlayVideo)) o.autoPlayVideo = undefined
if (!isUserAutoPlayNextVideoValid(o.autoPlayNextVideo)) o.autoPlayNextVideo = undefined
if (!isUserAutoPlayNextVideoPlaylistValid(o.autoPlayNextVideoPlaylist)) o.autoPlayNextVideoPlaylist = undefined
if (!isUserP2PEnabledValid(o.p2pEnabled)) o.p2pEnabled = undefined
if (!isUserVideosHistoryEnabledValid(o.videosHistoryEnabled)) o.videosHistoryEnabled = undefined
if (!isUserVideoLanguages(o.videoLanguages)) o.videoLanguages = undefined
if (!isUserLanguage(o.language)) o.language = undefined
if (!isThemeNameValid(o.theme) || !isThemeRegistered(o.theme)) o.theme = undefined
for (const key of Object.keys(o.notificationSettings || {})) {
if (!isUserNotificationSettingValid(o.notificationSettings[key])) (o.notificationSettings[key] as any) = undefined
}
return pick(o, [
'nsfwPolicy',
'autoPlayVideo',
'autoPlayNextVideo',
'autoPlayNextVideoPlaylist',
'p2pEnabled',
'videosHistoryEnabled',
'videoLanguages',
'language',
'theme',
'notificationSettings'
])
}
protected async importObject (userImportData: SanitizedObject) {
if (exists(userImportData.nsfwPolicy)) this.user.nsfwPolicy = userImportData.nsfwPolicy
if (exists(userImportData.autoPlayVideo)) this.user.autoPlayVideo = userImportData.autoPlayVideo
if (exists(userImportData.autoPlayNextVideo)) this.user.autoPlayNextVideo = userImportData.autoPlayNextVideo
if (exists(userImportData.autoPlayNextVideoPlaylist)) this.user.autoPlayNextVideoPlaylist = userImportData.autoPlayNextVideoPlaylist
if (exists(userImportData.p2pEnabled)) this.user.p2pEnabled = userImportData.p2pEnabled
if (exists(userImportData.videosHistoryEnabled)) this.user.videosHistoryEnabled = userImportData.videosHistoryEnabled
if (exists(userImportData.videoLanguages)) this.user.videoLanguages = userImportData.videoLanguages
if (exists(userImportData.language)) this.user.language = userImportData.language
if (exists(userImportData.theme)) this.user.theme = userImportData.theme
await saveInTransactionWithRetries(this.user)
await this.updateSettings(userImportData.notificationSettings)
logger.info('Settings of user %s imported.', this.user.username, lTags())
return { duplicate: false }
}
private async updateSettings (settingsImportData: UserSettingsExportJSON['notificationSettings']) {
const values: UserNotificationSetting = {
newVideoFromSubscription: settingsImportData.newVideoFromSubscription,
newCommentOnMyVideo: settingsImportData.newCommentOnMyVideo,
myVideoImportFinished: settingsImportData.myVideoImportFinished,
myVideoPublished: settingsImportData.myVideoPublished,
abuseAsModerator: settingsImportData.abuseAsModerator,
videoAutoBlacklistAsModerator: settingsImportData.videoAutoBlacklistAsModerator,
blacklistOnMyVideo: settingsImportData.blacklistOnMyVideo,
newUserRegistration: settingsImportData.newUserRegistration,
commentMention: settingsImportData.commentMention,
newFollow: settingsImportData.newFollow,
newInstanceFollower: settingsImportData.newInstanceFollower,
abuseNewMessage: settingsImportData.abuseNewMessage,
abuseStateChange: settingsImportData.abuseStateChange,
autoInstanceFollowing: settingsImportData.autoInstanceFollowing,
newPeerTubeVersion: settingsImportData.newPeerTubeVersion,
newPluginVersion: settingsImportData.newPluginVersion,
myVideoStudioEditionFinished: settingsImportData.myVideoStudioEditionFinished,
myVideoTranscriptionGenerated: settingsImportData.myVideoTranscriptionGenerated
}
await UserNotificationSettingModel.updateUserSettings(values, this.user.id)
}
}

View File

@@ -0,0 +1,41 @@
import { pick } from '@peertube/peertube-core-utils'
import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
type SanitizedObject = Pick<UserVideoHistoryExportJSON['watchedVideos'][0], 'videoUrl' | 'lastTimecode' | 'archiveFiles'>
// eslint-disable-next-line max-len
export class UserVideoHistoryImporter extends AbstractUserImporter <UserVideoHistoryExportJSON, UserVideoHistoryExportJSON['watchedVideos'][0], SanitizedObject> {
protected getImportObjects (json: UserVideoHistoryExportJSON) {
return json.watchedVideos
}
protected sanitize (data: UserVideoHistoryExportJSON['watchedVideos'][0]) {
if (!isUrlValid(data.videoUrl)) return undefined
return pick(data, [ 'videoUrl', 'lastTimecode' ])
}
protected async importObject (data: SanitizedObject) {
if (!this.user.videosHistoryEnabled) return { duplicate: false }
const videoUrl = data.videoUrl
const videoImmutable = await loadOrCreateVideoIfAllowedForUser(videoUrl)
if (!videoImmutable) {
throw new Error(`Cannot get or create video ${videoUrl} to import user history`)
}
await UserVideoHistoryModel.upsert({
videoId: videoImmutable.id,
userId: this.user.id,
currentTime: data.lastTimecode
})
return { duplicate: false }
}
}

View File

@@ -0,0 +1,179 @@
import { VideoPlaylistPrivacy, VideoPlaylistType, VideoPlaylistsExportJSON } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { buildUUID } from '@peertube/peertube-node-utils'
import { MChannelBannerAccountDefault, MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'
import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { updateLocalPlaylistMiniatureFromExisting } from '@server/lib/thumbnail.js'
import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js'
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
import {
isVideoPlaylistDescriptionValid,
isVideoPlaylistNameValid,
isVideoPlaylistPrivacyValid,
isVideoPlaylistTimestampValid,
isVideoPlaylistTypeValid
} from '@server/helpers/custom-validators/video-playlists.js'
import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
import { isArray } from '@server/helpers/custom-validators/misc.js'
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
import { pick } from '@peertube/peertube-core-utils'
import { VideoModel } from '@server/models/video/video.js'
import { generateThumbnailForPlaylist } from '@server/lib/video-playlist.js'
const lTags = loggerTagsFactory('user-import')
type ImportObject = VideoPlaylistsExportJSON['videoPlaylists'][0]
type SanitizedObject = Pick<ImportObject, 'type' | 'displayName' | 'privacy' | 'elements' | 'description' | 'channel' | 'archiveFiles'>
export class VideoPlaylistsImporter extends AbstractUserImporter<VideoPlaylistsExportJSON, ImportObject, SanitizedObject> {
protected getImportObjects (json: VideoPlaylistsExportJSON) {
return json.videoPlaylists
}
protected sanitize (o: ImportObject) {
if (!isVideoPlaylistTypeValid(o.type)) return undefined
if (!isVideoPlaylistNameValid(o.displayName)) return undefined
if (!isVideoPlaylistPrivacyValid(o.privacy)) return undefined
if (!isArray(o.elements)) return undefined
if (o.channel?.name && !isActorPreferredUsernameValid(o.channel.name)) o.channel = undefined
if (!isVideoPlaylistDescriptionValid(o.description)) o.description = undefined
o.elements = o.elements.filter(e => {
if (!isUrlValid(e.videoUrl)) return false
if (e.startTimestamp && !isVideoPlaylistTimestampValid(e.startTimestamp)) return false
if (e.stopTimestamp && !isVideoPlaylistTimestampValid(e.stopTimestamp)) return false
return true
})
return pick(o, [ 'type', 'displayName', 'privacy', 'elements', 'channel', 'description', 'archiveFiles' ])
}
protected async importObject (playlistImportData: SanitizedObject) {
const existingPlaylist = await VideoPlaylistModel.loadRegularByAccountAndName(this.user.Account, playlistImportData.displayName)
if (existingPlaylist) {
logger.info(`Do not import playlist ${playlistImportData.displayName} that already exists in the account`, lTags())
return { duplicate: true }
}
const videoPlaylist = playlistImportData.type === VideoPlaylistType.WATCH_LATER
? await this.getWatchLaterPlaylist()
: await this.createPlaylist(playlistImportData)
await this.createElements(videoPlaylist, playlistImportData)
await sendCreateVideoPlaylist(videoPlaylist, undefined)
logger.info('Video playlist %s imported.', videoPlaylist.name, lTags(videoPlaylist.uuid))
return { duplicate: false }
}
private async createPlaylist (playlistImportData: SanitizedObject) {
let videoChannel: MChannelBannerAccountDefault
if (playlistImportData.channel.name) {
videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(playlistImportData.channel.name)
if (!videoChannel) throw new Error(`Channel ${playlistImportData} not found`)
if (videoChannel.accountId !== this.user.Account.id) {
throw new Error(`Channel ${videoChannel.name} is not owned by user ${this.user.username}`)
}
} else if (playlistImportData.privacy !== VideoPlaylistPrivacy.PRIVATE) {
throw new Error('Cannot create a non private playlist without channel')
}
const playlist: MVideoPlaylistFull = new VideoPlaylistModel({
name: playlistImportData.displayName,
description: playlistImportData.description,
privacy: playlistImportData.privacy,
uuid: buildUUID(),
videoChannelId: videoChannel?.id,
ownerAccountId: this.user.Account.id
})
playlist.url = getLocalVideoPlaylistActivityPubUrl(playlist)
playlist.VideoChannel = videoChannel
playlist.OwnerAccount = this.user.Account
await saveInTransactionWithRetries(playlist)
await this.createThumbnail(playlist, playlistImportData)
return playlist
}
private async getWatchLaterPlaylist () {
return VideoPlaylistModel.loadWatchLaterOf(this.user.Account)
}
private async createThumbnail (playlist: MVideoPlaylistThumbnail, playlistImportData: SanitizedObject) {
const thumbnailPath = this.getSafeArchivePathOrThrow(playlistImportData.archiveFiles.thumbnail)
if (!thumbnailPath) return undefined
if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.FILE_SIZE.max)) return undefined
const thumbnail = await updateLocalPlaylistMiniatureFromExisting({
inputPath: thumbnailPath,
playlist,
automaticallyGenerated: false
})
await playlist.setAndSaveThumbnail(thumbnail, undefined)
}
private async createElements (playlist: MVideoPlaylistThumbnail, playlistImportData: SanitizedObject) {
const elementsToCreate: { videoId: number, startTimestamp: number, stopTimestamp: number }[] = []
for (const element of playlistImportData.elements.slice(0, USER_IMPORT.MAX_PLAYLIST_ELEMENTS)) {
const video = await loadOrCreateVideoIfAllowedForUser(element.videoUrl)
if (!video) {
logger.debug(`Cannot get or create video ${element.videoUrl} to create playlist element in user import`, lTags())
continue
}
elementsToCreate.push({
videoId: video.id,
startTimestamp: element.startTimestamp,
stopTimestamp: element.stopTimestamp
})
}
await sequelizeTypescript.transaction(async t => {
let position = await VideoPlaylistElementModel.getNextPositionOf(playlist.id, t)
for (const elementToCreate of elementsToCreate) {
const playlistElement = new VideoPlaylistElementModel({
position,
startTimestamp: elementToCreate.startTimestamp,
stopTimestamp: elementToCreate.stopTimestamp,
videoPlaylistId: playlist.id,
videoId: elementToCreate.videoId
})
await playlistElement.save({ transaction: t })
playlistElement.url = getLocalVideoPlaylistElementActivityPubUrl(playlist, playlistElement)
await playlistElement.save({ transaction: t })
if (playlist.shouldGenerateThumbnailWithNewElement(playlistElement)) {
const video = await VideoModel.loadFull(elementToCreate.videoId)
generateThumbnailForPlaylist(playlist, video)
.catch(err => logger.error('Cannot generate thumbnail from playlist', { err }))
}
position++
}
})
}
}

View File

@@ -0,0 +1,354 @@
import { pick } from '@peertube/peertube-core-utils'
import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models'
import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js'
import { isPlayerVideoThemeSettingValid } from '@server/helpers/custom-validators/player-settings.js'
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js'
import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js'
import { isLiveLatencyModeValid, isLiveScheduleValid } from '@server/helpers/custom-validators/video-lives.js'
import {
isPasswordValid,
isVideoCategoryValid,
isVideoCommentsPolicyValid,
isVideoDescriptionValid,
isVideoDurationValid,
isVideoLanguageValid,
isVideoLicenceValid,
isVideoNameValid,
isVideoOriginallyPublishedAtValid,
isVideoPrivacyValid,
isVideoReplayPrivacyValid,
isVideoSourceFilenameValid,
isVideoSupportValid,
isVideoTagValid
} from '@server/helpers/custom-validators/videos.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { LocalVideoCreator, ThumbnailOptions } from '@server/lib/local-video-creator.js'
import { isLocalVideoFileAccepted } from '@server/lib/moderation.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { createLocalCaption, updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoModel } from '@server/models/video/video.js'
import { MChannelId, MVideoFullLight } from '@server/types/models/index.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { parse } from 'path'
import { AbstractUserImporter } from './abstract-user-importer.js'
const lTags = loggerTagsFactory('user-import')
type ImportObject = VideoExportJSON['videos'][0]
type SanitizedObject = Pick<
ImportObject,
| 'name'
| 'duration'
| 'channel'
| 'privacy'
| 'archiveFiles'
| 'captions'
| 'category'
| 'licence'
| 'language'
| 'description'
| 'support'
| 'nsfw'
| 'isLive'
| 'commentsPolicy'
| 'downloadEnabled'
| 'waitTranscoding'
| 'originallyPublishedAt'
| 'tags'
| 'live'
| 'passwords'
| 'source'
| 'chapters'
| 'playerSettings'
>
export class VideosImporter extends AbstractUserImporter<VideoExportJSON, ImportObject, SanitizedObject> {
protected getImportObjects (json: VideoExportJSON) {
return json.videos
}
protected sanitize (o: ImportObject) {
if (!isVideoNameValid(o.name)) return undefined
if (!isVideoDurationValid(o.duration + '')) return undefined
if (!isVideoChannelUsernameValid(o.channel?.name)) return undefined
if (!isVideoPrivacyValid(o.privacy)) return undefined
if (o.isLive !== true && !o.archiveFiles?.videoFile) return undefined
if (!isVideoCategoryValid(o.category)) o.category = null
if (!o.licence || !isVideoLicenceValid(o.licence)) o.licence = CONFIG.DEFAULTS.PUBLISH.LICENCE
if (!isVideoLanguageValid(o.language)) o.language = null
if (!isVideoDescriptionValid(o.description)) o.description = null
if (!isVideoSupportValid(o.support)) o.support = null
if (!isBooleanValid(o.nsfw)) o.nsfw = false
if (!isBooleanValid(o.isLive)) o.isLive = false
if (!isBooleanValid(o.downloadEnabled)) o.downloadEnabled = CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED
if (!isBooleanValid(o.waitTranscoding)) o.waitTranscoding = true
if (!o.commentsPolicy || !isVideoCommentsPolicyValid(o.commentsPolicy)) {
o.commentsPolicy = CONFIG.DEFAULTS.PUBLISH.COMMENTS_POLICY
}
if (!isVideoSourceFilenameValid(o.source?.inputFilename)) o.source = undefined
if (!isVideoOriginallyPublishedAtValid(o.originallyPublishedAt)) o.originallyPublishedAt = null
if (!isArray(o.tags)) o.tags = []
if (!isArray(o.captions)) o.captions = []
if (!isArray(o.chapters)) o.chapters = []
o.tags = o.tags.filter(t => isVideoTagValid(t))
o.captions = o.captions.filter(c => isVideoCaptionLanguageValid(c.language))
o.chapters = o.chapters.filter(c => isVideoChapterTimecodeValid(c.timecode) && isVideoChapterTitleValid(c.title))
if (o.isLive) {
if (!o.live) return undefined
if (!isBooleanValid(o.live.permanentLive)) return undefined
if (!isBooleanValid(o.live.saveReplay)) o.live.saveReplay = false
if (o.live.saveReplay && !isVideoReplayPrivacyValid(o.live.replaySettings.privacy)) return undefined
if (!o.live.latencyMode || !isLiveLatencyModeValid(o.live.latencyMode)) o.live.latencyMode = LiveVideoLatencyMode.DEFAULT
if (!o.live.streamKey) o.live.streamKey = buildUUID()
else if (!isUUIDValid(o.live.streamKey)) return undefined
if (!isArray(o.live.schedules)) o.live.schedules = []
o.live.schedules = o.live.schedules.filter(s => isLiveScheduleValid(s))
}
if (o.playerSettings) {
if (!isPlayerVideoThemeSettingValid(o.playerSettings.theme)) o.playerSettings.theme = undefined
}
if (o.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
if (!isArray(o.passwords)) return undefined
// Refuse the import rather than handle only a portion of the passwords, which can be difficult for video owners to debug
if (o.passwords.some(p => !isPasswordValid(p))) return undefined
}
return pick(o, [
'name',
'duration',
'channel',
'privacy',
'archiveFiles',
'category',
'licence',
'language',
'description',
'support',
'nsfw',
'isLive',
'commentsPolicy',
'downloadEnabled',
'waitTranscoding',
'originallyPublishedAt',
'tags',
'captions',
'live',
'passwords',
'source',
'chapters',
'playerSettings'
])
}
protected async importObject (videoImportData: SanitizedObject) {
const videoFilePath = !videoImportData.isLive
? this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.videoFile)
: null
const videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(videoImportData.channel.name)
if (!videoChannel) throw new Error(`Channel ${videoImportData} not found`)
if (videoChannel.accountId !== this.user.Account.id) {
throw new Error(`Channel ${videoChannel.name} is not owned by user ${this.user.username}`)
}
const existingVideo = await VideoModel.loadByNameAndChannel(videoChannel, videoImportData.name)
if (existingVideo && Math.abs(existingVideo.duration - videoImportData.duration) <= 1) {
logger.info(`Do not import video ${videoImportData.name} that already exists in the account`, lTags())
return { duplicate: true }
}
const videoSize = videoFilePath
? await getFileSize(videoFilePath)
: undefined
let duration = 0
let ffprobe: FfprobeData
if (videoFilePath) {
if (await isUserQuotaValid({ userId: this.user.id, uploadSize: videoSize, checkDaily: false }) === false) {
throw new Error(`Cannot import video ${videoImportData.name} for user ${this.user.username} because of exceeded quota`)
}
await this.checkVideoFileIsAcceptedOrThrow({ videoFilePath, size: videoSize, channel: videoChannel, videoImportData })
ffprobe = await ffprobePromise(videoFilePath)
duration = await getVideoStreamDuration(videoFilePath, ffprobe)
}
const thumbnailPath = this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.thumbnail)
const thumbnails: ThumbnailOptions = []
for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) {
if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max)) continue
thumbnails.push({
path: thumbnailPath,
automaticallyGenerated: false,
keepOriginal: true,
type
})
}
const localVideoCreator = new LocalVideoCreator({
lTags,
videoFile: videoFilePath
? { path: videoFilePath, probe: ffprobe }
: undefined,
user: this.user,
channel: videoChannel,
chapters: videoImportData.chapters,
fallbackChapters: {
fromDescription: false,
finalFallback: undefined
},
videoAttributes: {
...pick(videoImportData, [
'name',
'category',
'licence',
'language',
'privacy',
'description',
'support',
'isLive',
'nsfw',
'tags',
'commentsPolicy',
'downloadEnabled',
'waitTranscoding',
'originallyPublishedAt'
]),
videoPasswords: videoImportData.passwords,
duration,
inputFilename: videoImportData.source?.inputFilename,
state: videoImportData.isLive
? VideoState.WAITING_FOR_LIVE
: buildNextVideoState()
},
liveAttributes: videoImportData.live,
videoAttributeResultHook: 'filter:api.video.user-import.video-attribute.result',
thumbnails
})
const { video } = await localVideoCreator.create()
await this.importCaptions(video, videoImportData)
await this.importPlayerSettings(video, videoImportData)
logger.info('Video %s imported.', video.name, lTags(video.uuid))
return { duplicate: false }
}
private async importCaptions (video: MVideoFullLight, videoImportData: SanitizedObject) {
const captionPaths: string[] = []
let updateHLS = false
for (const captionImport of videoImportData.captions) {
const relativeFilePath = videoImportData.archiveFiles?.captions?.[captionImport.language]
if (!relativeFilePath) {
logger.warn('Cannot import caption ' + captionImport.language + ': file does not exist in the archive', lTags(video.uuid))
continue
}
const absoluteFilePath = this.getSafeArchivePathOrThrow(relativeFilePath)
if (!await this.isFileValidOrLog(absoluteFilePath, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)) continue
const caption = await createLocalCaption({
video,
language: captionImport.language,
path: absoluteFilePath,
automaticallyGenerated: captionImport.automaticallyGenerated === true
})
captionPaths.push(absoluteFilePath)
if (caption.m3u8Filename) updateHLS = true
}
if (updateHLS && video.getHLSPlaylist()) {
await updateHLSMasterOnCaptionChange(video, video.getHLSPlaylist())
}
return captionPaths
}
private async importPlayerSettings (video: MVideoFullLight, videoImportData: SanitizedObject) {
const playerSettings = videoImportData.playerSettings
if (!playerSettings?.theme) return
await PlayerSettingModel.create({
theme: playerSettings.theme,
videoId: video.id
})
}
private async checkVideoFileIsAcceptedOrThrow (options: {
videoFilePath: string
size: number
channel: MChannelId
videoImportData: SanitizedObject
}) {
const { videoFilePath, size, videoImportData, channel } = options
// Check we accept this video
const acceptParameters = {
videoBody: {
...videoImportData,
channelId: channel.id
},
videoFile: {
path: videoFilePath,
filename: parse(videoFilePath).name,
size,
originalname: null
},
user: this.user
}
const acceptedResult = await Hooks.wrapFun(isLocalVideoFileAccepted, acceptParameters, 'filter:api.video.user-import.accept.result')
if (acceptedResult?.accepted !== true) {
logger.info('Refused local video file to import.', { acceptedResult, acceptParameters, ...lTags() })
throw new Error('Video file is not accepted: ' + acceptedResult.errorMessage || 'unknown reason')
}
}
}

View File

@@ -0,0 +1,35 @@
import { pick } from '@peertube/peertube-core-utils'
import { WatchedWordsListsJSON } from '@peertube/peertube-models'
import { areWatchedWordsValid, isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
type SanitizedObject = Pick<WatchedWordsListsJSON['watchedWordLists'][0], 'listName' | 'words'>
// eslint-disable-next-line max-len
export class WatchedWordsListsImporter extends AbstractUserImporter <WatchedWordsListsJSON, WatchedWordsListsJSON['watchedWordLists'][0], SanitizedObject> {
protected getImportObjects (json: WatchedWordsListsJSON) {
return json.watchedWordLists
}
protected sanitize (data: WatchedWordsListsJSON['watchedWordLists'][0]) {
if (!isWatchedWordListNameValid(data.listName)) return undefined
if (!areWatchedWordsValid(data.words)) return undefined
return pick(data, [ 'listName', 'words' ])
}
protected async importObject (data: SanitizedObject) {
const accountId = this.user.Account.id
const existing = await WatchedWordsListModel.loadByListName({ listName: data.listName, accountId })
if (existing) {
await existing.updateList({ listName: data.listName, words: data.words })
} else {
await WatchedWordsListModel.createList({ accountId, listName: data.listName, words: data.words })
}
return { duplicate: false }
}
}

View File

@@ -0,0 +1,302 @@
import { FileStorage, UserExportState } from '@peertube/peertube-models'
import { getFileSize } from '@peertube/peertube-node-utils'
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { UserModel } from '@server/models/user/user.js'
import { MUserDefault, MUserExport } from '@server/types/models/index.js'
import archiver, { Archiver } from 'archiver'
import { createWriteStream } from 'fs'
import { remove } from 'fs-extra/esm'
import { join, parse } from 'path'
import { PassThrough, Readable, Writable } from 'stream'
import { activityPubCollection } from '../activitypub/collection.js'
import { getContextFilter } from '../activitypub/context.js'
import { getUserExportFileObjectStorageSize, removeUserExportObjectStorage, storeUserExportFile } from '../object-storage/user-export.js'
import { getFSUserExportFilePath } from '../paths.js'
import {
AbstractUserExporter,
AccountExporter,
AutoTagPoliciesExporter,
BlocklistExporter,
ChannelsExporter,
CommentsExporter,
DislikesExporter,
ExportResult,
FollowersExporter,
FollowingExporter,
LikesExporter,
UserSettingsExporter,
UserVideoHistoryExporter,
VideoPlaylistsExporter,
VideosExporter,
WatchedWordsListsExporter
} from './exporters/index.js'
const lTags = loggerTagsFactory('user-export')
export class UserExporter {
private archive: Archiver
async export (exportModel: MUserExport) {
try {
exportModel.state = UserExportState.PROCESSING
await saveInTransactionWithRetries(exportModel)
const user = await UserModel.loadByIdFull(exportModel.userId)
let endPromise: Promise<any>
let output: Writable
if (exportModel.storage === FileStorage.FILE_SYSTEM) {
output = createWriteStream(getFSUserExportFilePath(exportModel))
endPromise = new Promise<string>(res => output.on('close', () => res('')))
} else {
output = new PassThrough()
endPromise = storeUserExportFile(output as PassThrough, exportModel)
}
await this.createZip({ exportModel, user, output })
const fileUrl = await endPromise
if (exportModel.storage === FileStorage.OBJECT_STORAGE) {
exportModel.fileUrl = fileUrl
exportModel.size = await getUserExportFileObjectStorageSize(exportModel)
} else if (exportModel.storage === FileStorage.FILE_SYSTEM) {
exportModel.size = await getFileSize(getFSUserExportFilePath(exportModel))
}
exportModel.state = UserExportState.COMPLETED
await saveInTransactionWithRetries(exportModel)
} catch (err) {
logger.error('Cannot generate an export', { err, ...lTags() })
try {
exportModel.state = UserExportState.ERRORED
exportModel.error = err.message
await saveInTransactionWithRetries(exportModel)
} catch (innerErr) {
logger.error('Cannot set export error state', { err: innerErr, ...lTags() })
}
try {
if (exportModel.storage === FileStorage.FILE_SYSTEM) {
await remove(getFSUserExportFilePath(exportModel))
} else {
await removeUserExportObjectStorage(exportModel)
}
} catch (innerErr) {
logger.error('Cannot remove archive path after failure', { err: innerErr, ...lTags() })
}
throw err
}
}
private createZip (options: {
exportModel: MUserExport
user: MUserDefault
output: Writable
}) {
const { output, exportModel, user } = options
let activityPubOutboxStore: ExportResult<any>['activityPubOutbox'] = []
this.archive = archiver('zip', {
zlib: {
level: 9
}
})
return new Promise<void>(async (res, rej) => {
this.archive.on('warning', err => {
logger.warn('Warning to archive a file in ' + exportModel.filename, { ...lTags(), err })
})
this.archive.on('error', err => {
rej(err)
})
this.archive.pipe(output)
try {
for (const { exporter, jsonFilename } of this.buildExporters(exportModel, user)) {
const { json, staticFiles, activityPub, activityPubOutbox } = await exporter.export()
logger.debug(`Adding JSON file ${jsonFilename} in archive ${exportModel.filename}`, lTags())
this.appendJSON(json, join('peertube', jsonFilename))
if (activityPub) {
const activityPubFilename = exporter.getActivityPubFilename()
if (!activityPubFilename) throw new Error('ActivityPub filename is required for exporter that export activity pub data')
this.appendJSON(activityPub, join('activity-pub', activityPubFilename))
}
if (activityPubOutbox) {
activityPubOutboxStore = activityPubOutboxStore.concat(activityPubOutbox)
}
for (const file of staticFiles) {
const archivePath = join('files', parse(jsonFilename).name, file.archivePath)
logger.debug(`Adding static file ${archivePath} in archive`, lTags())
try {
await this.addToArchiveAndWait(await file.readStreamFactory(), archivePath)
} catch (err) {
logger.error(`Cannot add ${archivePath} in archive`, { err, ...lTags() })
}
}
}
this.appendJSON(
await activityPubContextify(activityPubCollection('outbox.json', activityPubOutboxStore), 'Video', getContextFilter()),
join('activity-pub', 'outbox.json')
)
await this.archive.finalize()
res()
} catch (err) {
this.archive.abort()
rej(err)
}
})
}
private buildExporters (exportModel: MUserExport, user: MUserDefault) {
const options = {
user,
activityPubFilenames: {
dislikes: 'dislikes.json',
likes: 'likes.json',
outbox: 'outbox.json',
following: 'following.json',
account: 'actor.json'
}
}
return [
{
jsonFilename: 'videos.json',
exporter: new VideosExporter({
...options,
relativeStaticDirPath: '../files/videos',
withVideoFiles: exportModel.withVideoFiles
})
},
{
jsonFilename: 'channels.json',
exporter: new ChannelsExporter({
...options,
relativeStaticDirPath: '../files/channels'
})
},
{
jsonFilename: 'account.json',
exporter: new AccountExporter({
...options,
relativeStaticDirPath: '../files/account'
})
},
{
jsonFilename: 'blocklist.json',
exporter: new BlocklistExporter(options)
},
{
jsonFilename: 'likes.json',
exporter: new LikesExporter(options)
},
{
jsonFilename: 'dislikes.json',
exporter: new DislikesExporter(options)
},
{
jsonFilename: 'follower.json',
exporter: new FollowersExporter(options)
},
{
jsonFilename: 'following.json',
exporter: new FollowingExporter(options)
},
{
jsonFilename: 'user-settings.json',
exporter: new UserSettingsExporter(options)
},
{
jsonFilename: 'comments.json',
exporter: new CommentsExporter(options)
},
{
jsonFilename: 'video-playlists.json',
exporter: new VideoPlaylistsExporter({
...options,
relativeStaticDirPath: '../files/video-playlists'
})
},
{
jsonFilename: 'video-history.json',
exporter: new UserVideoHistoryExporter(options)
},
{
jsonFilename: 'watched-words-lists.json',
exporter: new WatchedWordsListsExporter(options)
},
{
jsonFilename: 'automatic-tag-policies.json',
exporter: new AutoTagPoliciesExporter(options)
}
] as { jsonFilename: string, exporter: AbstractUserExporter<any> }[]
}
private addToArchiveAndWait (stream: Readable, archivePath: string) {
let errored = false
return new Promise<void>((res, rej) => {
const self = this
function cleanup () {
self.archive.off('entry', entryListener)
}
function entryListener ({ name }) {
if (name !== archivePath) return
cleanup()
return res()
}
stream.once('error', err => {
cleanup()
errored = true
return rej(err)
})
this.archive.on('entry', entryListener)
// Prevent sending a stream that has an error on open resulting in a stucked archiving process
stream.once('readable', () => {
if (errored) return
this.archive.append(stream, { name: archivePath })
})
})
}
private appendJSON (json: any, name: string) {
this.archive.append(JSON.stringify(json, undefined, 2), { name })
}
}

View File

@@ -0,0 +1,171 @@
import { UserImportResultSummary, UserImportState } from '@peertube/peertube-models'
import { getFilenameWithoutExt, getFileSize } from '@peertube/peertube-node-utils'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { unzip } from '@server/helpers/unzip.js'
import { UserModel } from '@server/models/user/user.js'
import { MUserDefault, MUserImport } from '@server/types/models/index.js'
import { remove } from 'fs-extra/esm'
import { dirname, join } from 'path'
import { getFSUserImportFilePath } from '../paths.js'
import { BlocklistImporter } from './importers/account-blocklist-importer.js'
import { AccountImporter } from './importers/account-importer.js'
import { ChannelsImporter } from './importers/channels-importer.js'
import { DislikesImporter } from './importers/dislikes-importer.js'
import { FollowingImporter } from './importers/following-importer.js'
import { LikesImporter } from './importers/likes-importer.js'
import { ReviewCommentsTagPoliciesImporter } from './importers/review-comments-tag-policies-importer.js'
import { UserSettingsImporter } from './importers/user-settings-importer.js'
import { UserVideoHistoryImporter } from './importers/user-video-history-importer.js'
import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js'
import { VideosImporter } from './importers/videos-importer.js'
import { WatchedWordsListsImporter } from './importers/watched-words-lists-importer.js'
import { parseBytes } from '@server/helpers/core-utils.js'
const lTags = loggerTagsFactory('user-import')
export class UserImporter {
private extractedDirectory: string
async import (importModel: MUserImport) {
const resultSummary: UserImportResultSummary = {
stats: {
blocklist: this.buildSummary(),
channels: this.buildSummary(),
likes: this.buildSummary(),
dislikes: this.buildSummary(),
following: this.buildSummary(),
videoPlaylists: this.buildSummary(),
videos: this.buildSummary(),
account: this.buildSummary(),
userSettings: this.buildSummary(),
userVideoHistory: this.buildSummary(),
watchedWordsLists: this.buildSummary(),
commentAutoTagPolicies: this.buildSummary()
}
}
try {
importModel.state = UserImportState.PROCESSING
await saveInTransactionWithRetries(importModel)
const inputZip = getFSUserImportFilePath(importModel)
this.extractedDirectory = join(dirname(inputZip), getFilenameWithoutExt(inputZip))
await unzip({
source: inputZip,
destination: this.extractedDirectory,
// Videos that take a lot of space don't have a good compression ratio
// Keep a minimum of 1GB if the archive doesn't contain video files
maxSize: Math.max(await getFileSize(inputZip) * 2, parseBytes('1GB')),
maxFiles: 10000
})
const user = await UserModel.loadByIdFull(importModel.userId)
for (const { name, importer } of this.buildImporters(user)) {
try {
const { duplicates, errors, success } = await importer.import()
resultSummary.stats[name].duplicates += duplicates
resultSummary.stats[name].errors += errors
resultSummary.stats[name].success += success
} catch (err) {
logger.error(`Cannot import ${importer.getJSONFilePath()} from ${inputZip}`, { err, ...lTags() })
resultSummary.stats[name].errors++
}
}
importModel.state = UserImportState.COMPLETED
importModel.resultSummary = resultSummary
await saveInTransactionWithRetries(importModel)
} catch (err) {
logger.error('Cannot import user archive', { err, ...lTags() })
try {
importModel.state = UserImportState.ERRORED
importModel.error = err.message
await saveInTransactionWithRetries(importModel)
} catch (innerErr) {
logger.error('Cannot set import error state', { err: innerErr, ...lTags() })
}
throw err
} finally {
try {
await remove(getFSUserImportFilePath(importModel))
await remove(this.extractedDirectory)
} catch (innerErr) {
logger.error('Cannot remove import archive and directory after failure', { err: innerErr, ...lTags() })
}
}
}
private buildImporters (user: MUserDefault) {
// Keep consistency in import order (don't import videos before channels for example)
return [
{
name: 'account' as const,
importer: new AccountImporter(this.buildImporterOptions(user, 'account.json'))
},
{
name: 'userSettings' as const,
importer: new UserSettingsImporter(this.buildImporterOptions(user, 'user-settings.json'))
},
{
name: 'channels' as const,
importer: new ChannelsImporter(this.buildImporterOptions(user, 'channels.json'))
},
{
name: 'blocklist' as const,
importer: new BlocklistImporter(this.buildImporterOptions(user, 'blocklist.json'))
},
{
name: 'following' as const,
importer: new FollowingImporter(this.buildImporterOptions(user, 'following.json'))
},
{
name: 'videos' as const,
importer: new VideosImporter(this.buildImporterOptions(user, 'videos.json'))
},
{
name: 'likes' as const,
importer: new LikesImporter(this.buildImporterOptions(user, 'likes.json'))
},
{
name: 'dislikes' as const,
importer: new DislikesImporter(this.buildImporterOptions(user, 'dislikes.json'))
},
{
name: 'videoPlaylists' as const,
importer: new VideoPlaylistsImporter(this.buildImporterOptions(user, 'video-playlists.json'))
},
{
name: 'userVideoHistory' as const,
importer: new UserVideoHistoryImporter(this.buildImporterOptions(user, 'video-history.json'))
},
{
name: 'watchedWordsLists' as const,
importer: new WatchedWordsListsImporter(this.buildImporterOptions(user, 'watched-words-lists.json'))
},
{
name: 'commentAutoTagPolicies' as const,
importer: new ReviewCommentsTagPoliciesImporter(this.buildImporterOptions(user, 'automatic-tag-policies.json'))
}
]
}
private buildImporterOptions (user: MUserDefault, jsonFilename: string) {
return {
extractedDirectory: this.extractedDirectory,
user,
jsonFilePath: join(this.extractedDirectory, 'peertube', jsonFilename)
}
}
private buildSummary () {
return { success: 0, duplicates: 0, errors: 0 }
}
}