Init commit
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import { Activity, ActivityPubActor, ActivityPubOrderedCollection } from '@peertube/peertube-models'
|
||||
import { Awaitable } from '@peertube/peertube-typescript-utils'
|
||||
import { MUserDefault } from '@server/types/models/user/user.js'
|
||||
import { Readable } from 'stream'
|
||||
|
||||
export type ExportResult <T> = {
|
||||
json: T[] | T
|
||||
|
||||
staticFiles: {
|
||||
archivePath: string
|
||||
readStreamFactory: () => Promise<Readable>
|
||||
}[]
|
||||
|
||||
activityPub?: ActivityPubActor | ActivityPubOrderedCollection<string>
|
||||
|
||||
activityPubOutbox?: Omit<Activity, '@context'>[]
|
||||
}
|
||||
|
||||
type ActivityPubFilenames = {
|
||||
likes: string
|
||||
dislikes: string
|
||||
outbox: string
|
||||
following: string
|
||||
account: string
|
||||
}
|
||||
|
||||
export abstract class AbstractUserExporter <T> {
|
||||
protected user: MUserDefault
|
||||
|
||||
protected activityPubFilenames: ActivityPubFilenames
|
||||
|
||||
protected relativeStaticDirPath: string
|
||||
|
||||
constructor (options: {
|
||||
user: MUserDefault
|
||||
|
||||
activityPubFilenames: ActivityPubFilenames
|
||||
|
||||
relativeStaticDirPath?: string
|
||||
}) {
|
||||
this.user = options.user
|
||||
this.activityPubFilenames = options.activityPubFilenames
|
||||
this.relativeStaticDirPath = options.relativeStaticDirPath
|
||||
}
|
||||
|
||||
getActivityPubFilename () {
|
||||
return null
|
||||
}
|
||||
|
||||
abstract export (): Awaitable<ExportResult<T>>
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { AccountExportJSON, ActivityPubActor, ActorImageType } from '@peertube/peertube-models'
|
||||
import { MAccountDefault, MActorDefaultBanner } from '@server/types/models/index.js'
|
||||
import { ActorExporter } from './actor-exporter.js'
|
||||
import { AccountModel } from '@server/models/account/account.js'
|
||||
import { getContextFilter } from '@server/lib/activitypub/context.js'
|
||||
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
|
||||
import { join } from 'path'
|
||||
|
||||
export class AccountExporter extends ActorExporter <AccountExportJSON> {
|
||||
|
||||
async export () {
|
||||
const account = await AccountModel.loadLocalByName(this.user.username)
|
||||
|
||||
const { staticFiles, relativePathsFromJSON } = this.exportActorFiles(account.Actor as MActorDefaultBanner)
|
||||
|
||||
return {
|
||||
json: this.exportAccountJSON(account, relativePathsFromJSON),
|
||||
staticFiles,
|
||||
activityPub: await this.exportAccountAP(account)
|
||||
}
|
||||
}
|
||||
|
||||
getActivityPubFilename () {
|
||||
return this.activityPubFilenames.account
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private exportAccountJSON (account: MAccountDefault, archiveFiles: { avatar: string }): AccountExportJSON {
|
||||
return {
|
||||
...this.exportActorJSON(account.Actor as MActorDefaultBanner),
|
||||
|
||||
displayName: account.getDisplayName(),
|
||||
description: account.description,
|
||||
|
||||
updatedAt: account.updatedAt.toISOString(),
|
||||
createdAt: account.createdAt.toISOString(),
|
||||
|
||||
archiveFiles
|
||||
}
|
||||
}
|
||||
|
||||
private async exportAccountAP (account: MAccountDefault): Promise<ActivityPubActor> {
|
||||
const avatar = account.Actor.getMaxQualityImage(ActorImageType.AVATAR)
|
||||
|
||||
return activityPubContextify(
|
||||
{
|
||||
...await account.toActivityPubObject(),
|
||||
|
||||
likes: this.activityPubFilenames.likes,
|
||||
dislikes: this.activityPubFilenames.dislikes,
|
||||
outbox: this.activityPubFilenames.outbox,
|
||||
following: this.activityPubFilenames.following,
|
||||
|
||||
icon: avatar
|
||||
? [
|
||||
{
|
||||
...avatar.toActivityPubObject(),
|
||||
|
||||
url: join(this.relativeStaticDirPath, this.getAvatarPath(account.Actor, avatar.filename))
|
||||
}
|
||||
]
|
||||
: []
|
||||
},
|
||||
'Actor',
|
||||
getContextFilter())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { ActorImageModel } from '@server/models/actor/actor-image.js'
|
||||
import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { ActorImageType } from '@peertube/peertube-models'
|
||||
import { MActor, MActorDefaultBanner, MActorImage } from '@server/types/models/index.js'
|
||||
import { extname, join } from 'path'
|
||||
import { createReadStream } from 'fs'
|
||||
|
||||
export abstract class ActorExporter <T> extends AbstractUserExporter<T> {
|
||||
|
||||
protected exportActorJSON (actor: MActorDefaultBanner) {
|
||||
return {
|
||||
url: actor.url,
|
||||
|
||||
name: actor.preferredUsername,
|
||||
|
||||
avatars: actor.hasImage(ActorImageType.AVATAR)
|
||||
? this.exportActorImageJSON(actor.Avatars)
|
||||
: [],
|
||||
|
||||
banners: actor.hasImage(ActorImageType.BANNER)
|
||||
? this.exportActorImageJSON(actor.Banners)
|
||||
: []
|
||||
}
|
||||
}
|
||||
|
||||
protected exportActorImageJSON (images: MActorImage[]) {
|
||||
return images.map(i => ({
|
||||
width: i.width,
|
||||
url: ActorImageModel.getImageUrl(i),
|
||||
createdAt: i.createdAt.toISOString(),
|
||||
updatedAt: i.updatedAt.toISOString()
|
||||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
protected exportActorFiles (actor: MActorDefaultBanner) {
|
||||
const staticFiles: ExportResult<any>['staticFiles'] = []
|
||||
const relativePathsFromJSON = {
|
||||
avatar: null as string,
|
||||
banner: null as string
|
||||
}
|
||||
|
||||
const toProcess = [
|
||||
{
|
||||
archivePathBuilder: (filename: string) => this.getBannerPath(actor, filename),
|
||||
type: ActorImageType.BANNER
|
||||
},
|
||||
{
|
||||
archivePathBuilder: (filename: string) => this.getAvatarPath(actor, filename),
|
||||
type: ActorImageType.AVATAR
|
||||
}
|
||||
]
|
||||
|
||||
for (const { archivePathBuilder, type } of toProcess) {
|
||||
if (!actor.hasImage(type)) continue
|
||||
|
||||
const image = actor.getMaxQualityImage(type)
|
||||
|
||||
staticFiles.push({
|
||||
archivePath: archivePathBuilder(image.filename),
|
||||
readStreamFactory: () => Promise.resolve(createReadStream(image.getPath()))
|
||||
})
|
||||
|
||||
const relativePath = join(this.relativeStaticDirPath, archivePathBuilder(image.filename))
|
||||
|
||||
if (type === ActorImageType.AVATAR) relativePathsFromJSON.avatar = relativePath
|
||||
else if (type === ActorImageType.BANNER) relativePathsFromJSON.banner = relativePath
|
||||
}
|
||||
|
||||
return { staticFiles, relativePathsFromJSON }
|
||||
}
|
||||
|
||||
protected getAvatarPath (actor: MActor, filename: string) {
|
||||
return join('avatars', actor.preferredUsername + extname(filename))
|
||||
}
|
||||
|
||||
protected getBannerPath (actor: MActor, filename: string) {
|
||||
return join('banners', actor.preferredUsername + extname(filename))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { AutoTagPoliciesJSON } from '@peertube/peertube-models'
|
||||
import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
|
||||
export class AutoTagPoliciesExporter extends AbstractUserExporter <AutoTagPoliciesJSON> {
|
||||
|
||||
async export () {
|
||||
const data = await AutomaticTagger.getAutomaticTagPolicies(this.user.Account)
|
||||
|
||||
return {
|
||||
json: {
|
||||
reviewComments: data.review.map(name => ({ name }))
|
||||
} as AutoTagPoliciesJSON,
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js'
|
||||
import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js'
|
||||
import { BlocklistExportJSON } from '@peertube/peertube-models'
|
||||
|
||||
export class BlocklistExporter extends AbstractUserExporter <BlocklistExportJSON> {
|
||||
|
||||
async export () {
|
||||
const [ instancesBlocklist, accountsBlocklist ] = await Promise.all([
|
||||
ServerBlocklistModel.listHostsBlockedBy([ this.user.Account.id ]),
|
||||
AccountBlocklistModel.listHandlesBlockedBy([ this.user.Account.id ])
|
||||
])
|
||||
|
||||
return {
|
||||
json: {
|
||||
instances: instancesBlocklist.map(b => ({ host: b })),
|
||||
actors: accountsBlocklist.map(h => ({ handle: h }))
|
||||
} as BlocklistExportJSON,
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { ChannelExportJSON, PlayerChannelSettings } from '@peertube/peertube-models'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
import { MChannelBannerAccountDefault } from '@server/types/models/index.js'
|
||||
import { MPlayerSetting } from '@server/types/models/video/player-setting.js'
|
||||
import { ExportResult } from './abstract-user-exporter.js'
|
||||
import { ActorExporter } from './actor-exporter.js'
|
||||
|
||||
export class ChannelsExporter extends ActorExporter<ChannelExportJSON> {
|
||||
async export () {
|
||||
const channelsJSON: ChannelExportJSON['channels'] = []
|
||||
let staticFiles: ExportResult<ChannelExportJSON>['staticFiles'] = []
|
||||
|
||||
const channels = await VideoChannelModel.listAllOwnedByAccount(this.user.Account.id)
|
||||
|
||||
for (const channel of channels) {
|
||||
try {
|
||||
const exported = await this.exportChannel(channel.id)
|
||||
|
||||
channelsJSON.push(exported.json)
|
||||
staticFiles = staticFiles.concat(exported.staticFiles)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot export channel %s.', channel.name, { err })
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
json: { channels: channelsJSON },
|
||||
staticFiles
|
||||
}
|
||||
}
|
||||
|
||||
private async exportChannel (channelId: number) {
|
||||
const [ channel, playerSettings ] = await Promise.all([
|
||||
VideoChannelModel.loadAndPopulateAccount(channelId),
|
||||
PlayerSettingModel.loadByChannelId(channelId)
|
||||
])
|
||||
|
||||
const { relativePathsFromJSON, staticFiles } = this.exportActorFiles(channel.Actor)
|
||||
|
||||
return {
|
||||
json: this.exportChannelJSON(channel, playerSettings, relativePathsFromJSON),
|
||||
staticFiles
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private exportChannelJSON (
|
||||
channel: MChannelBannerAccountDefault,
|
||||
playerSettings: MPlayerSetting,
|
||||
archiveFiles: { avatar: string, banner: string }
|
||||
): ChannelExportJSON['channels'][0] {
|
||||
return {
|
||||
...this.exportActorJSON(channel.Actor),
|
||||
|
||||
displayName: channel.getDisplayName(),
|
||||
description: channel.description,
|
||||
support: channel.support,
|
||||
|
||||
playerSettings: this.exportPlayerSettingsJSON(playerSettings),
|
||||
|
||||
updatedAt: channel.updatedAt.toISOString(),
|
||||
createdAt: channel.createdAt.toISOString(),
|
||||
|
||||
archiveFiles
|
||||
}
|
||||
}
|
||||
|
||||
private exportPlayerSettingsJSON (playerSettings: MPlayerSetting) {
|
||||
if (!playerSettings) return null
|
||||
|
||||
return {
|
||||
theme: playerSettings.theme as PlayerChannelSettings['theme']
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { CommentsExportJSON, VideoCommentObject } from '@peertube/peertube-models'
|
||||
import { audiencify, getPublicAudience } from '@server/lib/activitypub/audience.js'
|
||||
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
|
||||
import { VideoCommentModel } from '@server/models/video/video-comment.js'
|
||||
import { MCommentExport } from '@server/types/models/index.js'
|
||||
import Bluebird from 'bluebird'
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
|
||||
export class CommentsExporter extends AbstractUserExporter<CommentsExportJSON> {
|
||||
async export () {
|
||||
const comments = await VideoCommentModel.listForExport(this.user.Account.id)
|
||||
|
||||
return {
|
||||
json: {
|
||||
comments: this.formatCommentsJSON(comments)
|
||||
},
|
||||
|
||||
activityPubOutbox: await this.formatCommentsAP(comments),
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
|
||||
private formatCommentsJSON (comments: MCommentExport[]) {
|
||||
return comments.map(c => ({
|
||||
url: c.url,
|
||||
text: c.text,
|
||||
createdAt: c.createdAt.toISOString(),
|
||||
videoUrl: c.Video.url,
|
||||
inReplyToCommentUrl: c?.InReplyToVideoComment?.url
|
||||
}))
|
||||
}
|
||||
|
||||
private formatCommentsAP (comments: MCommentExport[]) {
|
||||
return Bluebird.mapSeries(comments, async ({ url }) => {
|
||||
const comment = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoImmutableAndAccount(url)
|
||||
|
||||
const threadParentComments = await VideoCommentModel.listThreadParentComments({ comment })
|
||||
let commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject
|
||||
|
||||
const audience = getPublicAudience(comment.Account.Actor)
|
||||
|
||||
commentObject = audiencify(commentObject, audience)
|
||||
|
||||
return buildCreateActivity(comment.url, comment.Account.Actor, commentObject, audience)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { MAccountVideoRateVideoUrl } from '@server/types/models/index.js'
|
||||
import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js'
|
||||
import { ActivityPubOrderedCollection, DislikesExportJSON } from '@peertube/peertube-models'
|
||||
import { activityPubCollection } from '@server/lib/activitypub/collection.js'
|
||||
import { getContextFilter } from '@server/lib/activitypub/context.js'
|
||||
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
|
||||
|
||||
export class DislikesExporter extends AbstractUserExporter <DislikesExportJSON> {
|
||||
|
||||
async export () {
|
||||
const dislikes = await AccountVideoRateModel.listRatesOfAccountIdForExport(this.user.Account.id, 'dislike')
|
||||
|
||||
return {
|
||||
json: {
|
||||
dislikes: this.formatDislikesJSON(dislikes)
|
||||
} as DislikesExportJSON,
|
||||
|
||||
activityPub: await this.formatDislikesAP(dislikes),
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
|
||||
getActivityPubFilename () {
|
||||
return this.activityPubFilenames.dislikes
|
||||
}
|
||||
|
||||
private formatDislikesJSON (dislikes: MAccountVideoRateVideoUrl[]) {
|
||||
return dislikes.map(o => ({ videoUrl: o.Video.url, createdAt: o.createdAt.toISOString() }))
|
||||
}
|
||||
|
||||
private formatDislikesAP (dislikes: MAccountVideoRateVideoUrl[]): Promise<ActivityPubOrderedCollection<string>> {
|
||||
return activityPubContextify(
|
||||
activityPubCollection(this.getActivityPubFilename(), dislikes.map(l => l.Video.url)),
|
||||
'Rate',
|
||||
getContextFilter()
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { FollowersExportJSON } from '@peertube/peertube-models'
|
||||
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
|
||||
export class FollowersExporter extends AbstractUserExporter<FollowersExportJSON> {
|
||||
async export () {
|
||||
let followersJSON = this.formatFollowersJSON(
|
||||
await ActorFollowModel.listAcceptedFollowersForExport(this.user.Account.Actor.id),
|
||||
this.user.Account.Actor.getFullIdentifier()
|
||||
)
|
||||
|
||||
const channels = await VideoChannelModel.listAllOwnedByAccount(this.user.Account.id)
|
||||
|
||||
for (const channel of channels) {
|
||||
followersJSON = followersJSON.concat(
|
||||
this.formatFollowersJSON(
|
||||
await ActorFollowModel.listAcceptedFollowersForExport(channel.Actor.id),
|
||||
channel.Actor.getFullIdentifier()
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
json: { followers: followersJSON } as FollowersExportJSON,
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
|
||||
private formatFollowersJSON (
|
||||
follows: {
|
||||
createdAt: Date
|
||||
followerHandle: string
|
||||
}[],
|
||||
targetHandle: string
|
||||
): FollowersExportJSON['followers'] {
|
||||
return follows.map(f => ({
|
||||
targetHandle,
|
||||
handle: f.followerHandle,
|
||||
createdAt: f.createdAt.toISOString()
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { ActivityPubOrderedCollection, FollowingExportJSON } from '@peertube/peertube-models'
|
||||
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
|
||||
import { activityPubCollection } from '@server/lib/activitypub/collection.js'
|
||||
import { getContextFilter } from '@server/lib/activitypub/context.js'
|
||||
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
|
||||
export class FollowingExporter extends AbstractUserExporter<FollowingExportJSON> {
|
||||
async export () {
|
||||
const following = await ActorFollowModel.listAcceptedFollowingForExport(this.user.Account.Actor.id)
|
||||
const followingJSON = this.formatFollowingJSON(following, this.user.Account.Actor.getFullIdentifier())
|
||||
|
||||
return {
|
||||
json: { following: followingJSON } as FollowingExportJSON,
|
||||
|
||||
staticFiles: [],
|
||||
|
||||
activityPub: await this.formatFollowingAP(following)
|
||||
}
|
||||
}
|
||||
|
||||
getActivityPubFilename () {
|
||||
return this.activityPubFilenames.following
|
||||
}
|
||||
|
||||
private formatFollowingJSON (
|
||||
follows: {
|
||||
createdAt: Date
|
||||
followingHandle: string
|
||||
}[],
|
||||
handle: string
|
||||
): FollowingExportJSON['following'] {
|
||||
return follows.map(f => ({
|
||||
handle,
|
||||
targetHandle: f.followingHandle,
|
||||
createdAt: f.createdAt.toISOString()
|
||||
}))
|
||||
}
|
||||
|
||||
private formatFollowingAP (follows: { followingUrl: string }[]): Promise<ActivityPubOrderedCollection<string>> {
|
||||
return activityPubContextify(
|
||||
activityPubCollection(this.getActivityPubFilename(), follows.map(f => f.followingUrl)),
|
||||
'Collection',
|
||||
getContextFilter()
|
||||
)
|
||||
}
|
||||
}
|
||||
15
server/core/lib/user-import-export/exporters/index.ts
Normal file
15
server/core/lib/user-import-export/exporters/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
export * from './abstract-user-exporter.js'
|
||||
export * from './account-exporter.js'
|
||||
export * from './auto-tag-policies.js'
|
||||
export * from './blocklist-exporter.js'
|
||||
export * from './channels-exporter.js'
|
||||
export * from './comments-exporter.js'
|
||||
export * from './dislikes-exporter.js'
|
||||
export * from './followers-exporter.js'
|
||||
export * from './following-exporter.js'
|
||||
export * from './likes-exporter.js'
|
||||
export * from './user-settings-exporter.js'
|
||||
export * from './user-video-history-exporter.js'
|
||||
export * from './video-playlists-exporter.js'
|
||||
export * from './videos-exporter.js'
|
||||
export * from './watched-words-lists-exporter.js'
|
||||
@@ -0,0 +1,40 @@
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { MAccountVideoRateVideoUrl } from '@server/types/models/index.js'
|
||||
import { ActivityPubOrderedCollection, LikesExportJSON } from '@peertube/peertube-models'
|
||||
import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js'
|
||||
import { activityPubCollection } from '@server/lib/activitypub/collection.js'
|
||||
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
|
||||
import { getContextFilter } from '@server/lib/activitypub/context.js'
|
||||
|
||||
export class LikesExporter extends AbstractUserExporter <LikesExportJSON> {
|
||||
|
||||
async export () {
|
||||
const likes = await AccountVideoRateModel.listRatesOfAccountIdForExport(this.user.Account.id, 'like')
|
||||
|
||||
return {
|
||||
json: {
|
||||
likes: this.formatLikesJSON(likes)
|
||||
} as LikesExportJSON,
|
||||
|
||||
activityPub: await this.formatLikesAP(likes),
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
|
||||
getActivityPubFilename () {
|
||||
return this.activityPubFilenames.likes
|
||||
}
|
||||
|
||||
private formatLikesJSON (likes: MAccountVideoRateVideoUrl[]) {
|
||||
return likes.map(o => ({ videoUrl: o.Video.url, createdAt: o.createdAt.toISOString() }))
|
||||
}
|
||||
|
||||
private formatLikesAP (likes: MAccountVideoRateVideoUrl[]): Promise<ActivityPubOrderedCollection<string>> {
|
||||
return activityPubContextify(
|
||||
activityPubCollection(this.getActivityPubFilename(), likes.map(l => l.Video.url)),
|
||||
'Collection',
|
||||
getContextFilter()
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { UserSettingsExportJSON } from '@peertube/peertube-models'
|
||||
|
||||
export class UserSettingsExporter extends AbstractUserExporter<UserSettingsExportJSON> {
|
||||
export () {
|
||||
return {
|
||||
json: {
|
||||
email: this.user.email,
|
||||
|
||||
emailPublic: this.user.emailPublic,
|
||||
nsfwPolicy: this.user.nsfwPolicy,
|
||||
|
||||
autoPlayVideo: this.user.autoPlayVideo,
|
||||
autoPlayNextVideo: this.user.autoPlayNextVideo,
|
||||
autoPlayNextVideoPlaylist: this.user.autoPlayNextVideoPlaylist,
|
||||
|
||||
p2pEnabled: this.user.p2pEnabled,
|
||||
|
||||
videosHistoryEnabled: this.user.videosHistoryEnabled,
|
||||
videoLanguages: this.user.videoLanguages,
|
||||
language: this.user.language,
|
||||
|
||||
theme: this.user.theme,
|
||||
|
||||
createdAt: this.user.createdAt,
|
||||
|
||||
notificationSettings: this.user.NotificationSetting.toFormattedJSON()
|
||||
},
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
|
||||
|
||||
export class UserVideoHistoryExporter extends AbstractUserExporter <UserVideoHistoryExportJSON> {
|
||||
|
||||
async export () {
|
||||
const videos = await UserVideoHistoryModel.listForExport(this.user)
|
||||
|
||||
return {
|
||||
json: {
|
||||
watchedVideos: videos.map(v => ({
|
||||
videoUrl: v.videoUrl,
|
||||
lastTimecode: v.currentTime,
|
||||
createdAt: v.createdAt.toISOString(),
|
||||
updatedAt: v.updatedAt.toISOString()
|
||||
}))
|
||||
} as UserVideoHistoryExportJSON,
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
import { VideoPlaylistsExportJSON } from '@peertube/peertube-models'
|
||||
import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
|
||||
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
|
||||
import { extname, join } from 'path'
|
||||
import { createReadStream } from 'fs'
|
||||
import { MThumbnail, MVideoPlaylist } from '@server/types/models/index.js'
|
||||
|
||||
export class VideoPlaylistsExporter extends AbstractUserExporter <VideoPlaylistsExportJSON> {
|
||||
|
||||
async export () {
|
||||
const playlistsJSON: VideoPlaylistsExportJSON['videoPlaylists'] = []
|
||||
const staticFiles: ExportResult<VideoPlaylistsExportJSON>['staticFiles'] = []
|
||||
|
||||
const playlists = await VideoPlaylistModel.listPlaylistForExport(this.user.Account.id)
|
||||
|
||||
for (const playlist of playlists) {
|
||||
const elements = await VideoPlaylistElementModel.listElementsForExport(playlist.id)
|
||||
|
||||
const archiveFiles = {
|
||||
thumbnail: null as string
|
||||
}
|
||||
|
||||
if (playlist.hasThumbnail()) {
|
||||
const thumbnail = playlist.Thumbnail
|
||||
|
||||
staticFiles.push({
|
||||
archivePath: this.getArchiveThumbnailPath(playlist, thumbnail),
|
||||
readStreamFactory: () => Promise.resolve(createReadStream(thumbnail.getPath()))
|
||||
})
|
||||
|
||||
archiveFiles.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailPath(playlist, thumbnail))
|
||||
}
|
||||
|
||||
playlistsJSON.push({
|
||||
displayName: playlist.name,
|
||||
description: playlist.description,
|
||||
privacy: playlist.privacy,
|
||||
url: playlist.url,
|
||||
uuid: playlist.uuid,
|
||||
|
||||
type: playlist.type,
|
||||
|
||||
channel: {
|
||||
name: playlist.VideoChannel?.Actor?.preferredUsername
|
||||
},
|
||||
|
||||
createdAt: playlist.createdAt.toISOString(),
|
||||
updatedAt: playlist.updatedAt.toISOString(),
|
||||
|
||||
thumbnailUrl: playlist.Thumbnail?.getOriginFileUrl(playlist),
|
||||
|
||||
elements: elements.map(e => ({
|
||||
videoUrl: e.Video.url,
|
||||
startTimestamp: e.startTimestamp,
|
||||
stopTimestamp: e.stopTimestamp
|
||||
})),
|
||||
|
||||
archiveFiles
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
json: {
|
||||
videoPlaylists: playlistsJSON
|
||||
},
|
||||
|
||||
staticFiles
|
||||
}
|
||||
}
|
||||
|
||||
private getArchiveThumbnailPath (playlist: MVideoPlaylist, thumbnail: MThumbnail) {
|
||||
return join('thumbnails', playlist.uuid + extname(thumbnail.filename))
|
||||
}
|
||||
}
|
||||
483
server/core/lib/user-import-export/exporters/videos-exporter.ts
Normal file
483
server/core/lib/user-import-export/exporters/videos-exporter.ts
Normal file
@@ -0,0 +1,483 @@
|
||||
import { pick, sortBy } from '@peertube/peertube-core-utils'
|
||||
import { ActivityCreate, FileStorage, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { audiencify, getVideoAudience } from '@server/lib/activitypub/audience.js'
|
||||
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
|
||||
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
|
||||
import {
|
||||
getCaptionReadStream,
|
||||
getHLSFileReadStream,
|
||||
getOriginalFileReadStream,
|
||||
getWebVideoFileReadStream
|
||||
} from '@server/lib/object-storage/videos.js'
|
||||
import { VideoDownload } from '@server/lib/video-download.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
|
||||
import { VideoLiveModel } from '@server/models/video/video-live.js'
|
||||
import { VideoPasswordModel } from '@server/models/video/video-password.js'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import {
|
||||
MStreamingPlaylistFiles,
|
||||
MThumbnail,
|
||||
MVideo,
|
||||
MVideoAP,
|
||||
MVideoCaption,
|
||||
MVideoCaptionLanguageUrl,
|
||||
MVideoChapter,
|
||||
MVideoFile,
|
||||
MVideoFullLight,
|
||||
MVideoLiveWithSettingSchedules,
|
||||
MVideoPassword
|
||||
} from '@server/types/models/index.js'
|
||||
import { MPlayerSetting } from '@server/types/models/video/player-setting.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import Bluebird from 'bluebird'
|
||||
import { createReadStream } from 'fs'
|
||||
import { extname, join } from 'path'
|
||||
import { PassThrough, Readable } from 'stream'
|
||||
import { AbstractUserExporter, ExportResult } from './abstract-user-exporter.js'
|
||||
|
||||
export class VideosExporter extends AbstractUserExporter<VideoExportJSON> {
|
||||
constructor (
|
||||
private readonly options: ConstructorParameters<typeof AbstractUserExporter<VideoExportJSON>>[0] & {
|
||||
withVideoFiles: boolean
|
||||
}
|
||||
) {
|
||||
super(options)
|
||||
}
|
||||
|
||||
async export () {
|
||||
const videosJSON: VideoExportJSON['videos'] = []
|
||||
const activityPubOutbox: ActivityCreate<VideoObject>[] = []
|
||||
let staticFiles: ExportResult<VideoExportJSON>['staticFiles'] = []
|
||||
|
||||
let videoIds: number[] = []
|
||||
let start = 0
|
||||
const chunkSize = 100
|
||||
|
||||
do {
|
||||
videoIds = await VideoModel.getAllIdsByAccount({ account: this.user.Account, start, count: chunkSize })
|
||||
start += videoIds.length
|
||||
|
||||
await Bluebird.map(videoIds, async id => {
|
||||
try {
|
||||
const exported = await this.exportVideo(id)
|
||||
|
||||
videosJSON.push(exported.json)
|
||||
staticFiles = staticFiles.concat(exported.staticFiles)
|
||||
activityPubOutbox.push(exported.activityPubOutbox)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot export video %d.', id, { err })
|
||||
}
|
||||
}, { concurrency: 10 })
|
||||
} while (videoIds.length === chunkSize)
|
||||
|
||||
return {
|
||||
json: { videos: sortBy(videosJSON, 'publishedAt') },
|
||||
activityPubOutbox,
|
||||
staticFiles
|
||||
}
|
||||
}
|
||||
|
||||
private async exportVideo (videoId: number) {
|
||||
const [ video, captions, source, chapters, playerSettings ] = await Promise.all([
|
||||
VideoModel.loadFull(videoId),
|
||||
VideoCaptionModel.listVideoCaptions(videoId),
|
||||
VideoSourceModel.loadLatest(videoId),
|
||||
VideoChapterModel.listChaptersOfVideo(videoId),
|
||||
PlayerSettingModel.loadByVideoId(videoId)
|
||||
])
|
||||
|
||||
const passwords = video.privacy === VideoPrivacy.PASSWORD_PROTECTED
|
||||
? (await VideoPasswordModel.listPasswords({ videoId, start: 0, count: undefined, sort: 'createdAt' })).data
|
||||
: []
|
||||
|
||||
const live = video.isLive
|
||||
? await VideoLiveModel.loadByVideoIdFull(videoId)
|
||||
: undefined // We already have captions, so we can set it to the video object
|
||||
;(video as any).VideoCaptions = captions
|
||||
// Then fetch more attributes for AP serialization
|
||||
const videoAP = await video.lightAPToFullAP(undefined)
|
||||
|
||||
const { relativePathsFromJSON, staticFiles, exportedVideoFileOrSource } = await this.exportVideoFiles({ video, captions })
|
||||
|
||||
return {
|
||||
json: this.exportVideoJSON({
|
||||
video,
|
||||
captions,
|
||||
live,
|
||||
passwords,
|
||||
source,
|
||||
chapters,
|
||||
playerSettings,
|
||||
archiveFiles: relativePathsFromJSON
|
||||
}),
|
||||
staticFiles,
|
||||
relativePathsFromJSON,
|
||||
activityPubOutbox: await this.exportVideoAP(videoAP, chapters, exportedVideoFileOrSource)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private exportVideoJSON (options: {
|
||||
video: MVideoFullLight
|
||||
captions: MVideoCaption[]
|
||||
live: MVideoLiveWithSettingSchedules
|
||||
passwords: MVideoPassword[]
|
||||
source: MVideoSource
|
||||
playerSettings: MPlayerSetting
|
||||
chapters: MVideoChapter[]
|
||||
archiveFiles: VideoExportJSON['videos'][0]['archiveFiles']
|
||||
}): VideoExportJSON['videos'][0] {
|
||||
const { video, captions, live, passwords, source, chapters, playerSettings, archiveFiles } = options
|
||||
|
||||
return {
|
||||
uuid: video.uuid,
|
||||
|
||||
createdAt: video.createdAt.toISOString(),
|
||||
updatedAt: video.updatedAt.toISOString(),
|
||||
publishedAt: video.publishedAt.toISOString(),
|
||||
originallyPublishedAt: video.originallyPublishedAt
|
||||
? video.originallyPublishedAt.toISOString()
|
||||
: undefined,
|
||||
|
||||
name: video.name,
|
||||
category: video.category,
|
||||
licence: video.licence,
|
||||
language: video.language,
|
||||
tags: video.Tags.map(t => t.name),
|
||||
|
||||
privacy: video.privacy,
|
||||
passwords: passwords.map(p => p.password),
|
||||
|
||||
duration: video.duration,
|
||||
|
||||
description: video.description,
|
||||
support: video.support,
|
||||
|
||||
isLive: video.isLive,
|
||||
live: this.exportLiveJSON(video, live),
|
||||
|
||||
url: video.url,
|
||||
|
||||
thumbnailUrl: video.getMiniature()?.getOriginFileUrl(video) || null,
|
||||
previewUrl: video.getPreview()?.getOriginFileUrl(video) || null,
|
||||
|
||||
views: video.views,
|
||||
|
||||
likes: video.likes,
|
||||
dislikes: video.dislikes,
|
||||
|
||||
nsfw: video.nsfw,
|
||||
|
||||
commentsPolicy: video.commentsPolicy,
|
||||
|
||||
downloadEnabled: video.downloadEnabled,
|
||||
|
||||
waitTranscoding: video.waitTranscoding,
|
||||
state: video.state,
|
||||
|
||||
channel: {
|
||||
name: video.VideoChannel.Actor.preferredUsername
|
||||
},
|
||||
|
||||
captions: this.exportCaptionsJSON(video, captions),
|
||||
chapters: this.exportChaptersJSON(chapters),
|
||||
|
||||
files: this.exportFilesJSON(video, video.VideoFiles),
|
||||
|
||||
streamingPlaylists: this.exportStreamingPlaylistsJSON(video, video.VideoStreamingPlaylists),
|
||||
|
||||
source: this.exportVideoSourceJSON(source),
|
||||
|
||||
playerSettings: this.exportPlayerSettingsJSON(playerSettings),
|
||||
|
||||
archiveFiles
|
||||
}
|
||||
}
|
||||
|
||||
private exportLiveJSON (video: MVideo, live: MVideoLiveWithSettingSchedules) {
|
||||
if (!video.isLive) return undefined
|
||||
|
||||
return {
|
||||
saveReplay: live.saveReplay,
|
||||
permanentLive: live.permanentLive,
|
||||
latencyMode: live.latencyMode,
|
||||
streamKey: live.streamKey,
|
||||
|
||||
replaySettings: live.ReplaySetting
|
||||
? { privacy: live.ReplaySetting.privacy }
|
||||
: undefined,
|
||||
|
||||
schedules: live.LiveSchedules?.map(s => ({
|
||||
startAt: s.startAt.toISOString()
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
private exportCaptionsJSON (video: MVideo, captions: MVideoCaption[]) {
|
||||
return captions.map(c => ({
|
||||
createdAt: c.createdAt.toISOString(),
|
||||
updatedAt: c.updatedAt.toISOString(),
|
||||
language: c.language,
|
||||
filename: c.filename,
|
||||
automaticallyGenerated: c.automaticallyGenerated,
|
||||
fileUrl: c.getFileUrl(video)
|
||||
}))
|
||||
}
|
||||
|
||||
private exportChaptersJSON (chapters: MVideoChapter[]) {
|
||||
return chapters.map(c => ({
|
||||
timecode: c.timecode,
|
||||
title: c.title
|
||||
}))
|
||||
}
|
||||
|
||||
private exportFilesJSON (video: MVideo, files: MVideoFile[]) {
|
||||
return files.map(f => ({
|
||||
resolution: f.resolution,
|
||||
size: f.size,
|
||||
fps: f.fps,
|
||||
|
||||
torrentUrl: f.getTorrentUrl(),
|
||||
fileUrl: f.getFileUrl(video)
|
||||
}))
|
||||
}
|
||||
|
||||
private exportStreamingPlaylistsJSON (video: MVideo, streamingPlaylists: MStreamingPlaylistFiles[]) {
|
||||
return streamingPlaylists.map(p => ({
|
||||
type: p.type,
|
||||
playlistUrl: p.getMasterPlaylistUrl(video),
|
||||
segmentsSha256Url: p.getSha256SegmentsUrl(video),
|
||||
files: this.exportFilesJSON(video, p.VideoFiles)
|
||||
}))
|
||||
}
|
||||
|
||||
private exportVideoSourceJSON (source: MVideoSource) {
|
||||
if (!source) return null
|
||||
|
||||
return {
|
||||
inputFilename: source.inputFilename,
|
||||
|
||||
resolution: source.resolution,
|
||||
size: source.size,
|
||||
|
||||
width: source.width,
|
||||
height: source.height,
|
||||
|
||||
fps: source.fps,
|
||||
|
||||
metadata: source.metadata
|
||||
}
|
||||
}
|
||||
|
||||
private exportPlayerSettingsJSON (playerSettings: MPlayerSetting) {
|
||||
if (!playerSettings) return null
|
||||
|
||||
return {
|
||||
theme: playerSettings.theme
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private async exportVideoAP (
|
||||
video: MVideoAP,
|
||||
chapters: MVideoChapter[],
|
||||
exportedVideoFileOrSource: MVideoFile | MVideoSource
|
||||
): Promise<ActivityCreate<VideoObject>> {
|
||||
const icon = video.getPreview()
|
||||
|
||||
const audience = getVideoAudience({
|
||||
account: video.VideoChannel.Account,
|
||||
channel: video.VideoChannel,
|
||||
privacy: video.privacy,
|
||||
skipPrivacyCheck: true
|
||||
})
|
||||
|
||||
const videoObject: VideoObject = {
|
||||
...audiencify(await video.toActivityPubObject(), audience),
|
||||
|
||||
icon: [
|
||||
{
|
||||
...icon.toActivityPubObject(video),
|
||||
|
||||
url: join(this.options.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, icon))
|
||||
}
|
||||
],
|
||||
|
||||
subtitleLanguage: video.VideoCaptions.map(c => ({
|
||||
...c.toActivityPubObject(video),
|
||||
|
||||
url: [
|
||||
{
|
||||
mediaType: 'text/vtt',
|
||||
type: 'Link',
|
||||
href: join(this.options.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, c))
|
||||
}
|
||||
]
|
||||
})),
|
||||
|
||||
hasParts: buildChaptersAPHasPart(video, chapters),
|
||||
|
||||
attachment: this.options.withVideoFiles && exportedVideoFileOrSource
|
||||
? [
|
||||
{
|
||||
type: 'Video' as 'Video',
|
||||
url: join(this.options.relativeStaticDirPath, this.getArchiveVideoFilePath(video, exportedVideoFileOrSource)),
|
||||
|
||||
// FIXME: typings
|
||||
...pick((exportedVideoFileOrSource as MVideoFile & MVideoSource).toActivityPubObject(video), [
|
||||
'mediaType',
|
||||
'height',
|
||||
'size',
|
||||
'fps'
|
||||
])
|
||||
}
|
||||
]
|
||||
: undefined
|
||||
}
|
||||
|
||||
return buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private async exportVideoFiles (options: {
|
||||
video: MVideoFullLight
|
||||
captions: MVideoCaption[]
|
||||
}) {
|
||||
const { video, captions } = options
|
||||
|
||||
const staticFiles: ExportResult<VideoExportJSON>['staticFiles'] = []
|
||||
|
||||
let exportedVideoFileOrSource: MVideoFile | MVideoSource
|
||||
|
||||
const relativePathsFromJSON = {
|
||||
videoFile: null as string,
|
||||
thumbnail: null as string,
|
||||
captions: {} as { [lang: string]: string }
|
||||
}
|
||||
|
||||
if (this.options.withVideoFiles) {
|
||||
const { source, videoFile, separatedAudioFile } = await this.getArchiveVideo(video)
|
||||
|
||||
if (source || videoFile || separatedAudioFile) {
|
||||
const videoPath = this.getArchiveVideoFilePath(video, source || videoFile || separatedAudioFile)
|
||||
|
||||
staticFiles.push({
|
||||
archivePath: videoPath,
|
||||
|
||||
// Prefer using original file if possible
|
||||
readStreamFactory: () =>
|
||||
source?.keptOriginalFilename
|
||||
? this.generateVideoSourceReadStream(source)
|
||||
: this.generateVideoFileReadStream({ video, videoFile, separatedAudioFile })
|
||||
})
|
||||
|
||||
relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, videoPath)
|
||||
|
||||
exportedVideoFileOrSource = source?.keptOriginalFilename
|
||||
? source
|
||||
: videoFile || separatedAudioFile
|
||||
}
|
||||
}
|
||||
|
||||
for (const caption of captions) {
|
||||
staticFiles.push({
|
||||
archivePath: this.getArchiveCaptionFilePath(video, caption),
|
||||
readStreamFactory: () => this.generateCaptionReadStream(caption)
|
||||
})
|
||||
|
||||
relativePathsFromJSON.captions[caption.language] = join(this.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, caption))
|
||||
}
|
||||
|
||||
const thumbnail = video.getPreview() || video.getMiniature()
|
||||
if (thumbnail) {
|
||||
staticFiles.push({
|
||||
archivePath: this.getArchiveThumbnailFilePath(video, thumbnail),
|
||||
readStreamFactory: () => Promise.resolve(createReadStream(thumbnail.getPath()))
|
||||
})
|
||||
|
||||
relativePathsFromJSON.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, thumbnail))
|
||||
}
|
||||
|
||||
return { staticFiles, relativePathsFromJSON, exportedVideoFileOrSource }
|
||||
}
|
||||
|
||||
private async generateVideoSourceReadStream (source: MVideoSource): Promise<Readable> {
|
||||
if (source.storage === FileStorage.FILE_SYSTEM) {
|
||||
return createReadStream(VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename))
|
||||
}
|
||||
|
||||
const { stream } = await getOriginalFileReadStream({ keptOriginalFilename: source.keptOriginalFilename, rangeHeader: undefined })
|
||||
|
||||
return stream
|
||||
}
|
||||
|
||||
private async generateVideoFileReadStream (options: {
|
||||
videoFile: MVideoFile
|
||||
separatedAudioFile: MVideoFile
|
||||
video: MVideoFullLight
|
||||
}): Promise<Readable> {
|
||||
const { video, videoFile, separatedAudioFile } = options
|
||||
|
||||
if (separatedAudioFile) {
|
||||
const stream = new PassThrough()
|
||||
|
||||
new VideoDownload({ video, videoFiles: [ videoFile, separatedAudioFile ] })
|
||||
.muxToMergeVideoFiles(stream)
|
||||
.catch(err => logger.error('Cannot mux video files', { err }))
|
||||
|
||||
return Promise.resolve(stream)
|
||||
}
|
||||
|
||||
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
|
||||
return createReadStream(VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile))
|
||||
}
|
||||
|
||||
const { stream } = videoFile.isHLS()
|
||||
? await getHLSFileReadStream({ playlist: video.getHLSPlaylist(), filename: videoFile.filename, rangeHeader: undefined })
|
||||
: await getWebVideoFileReadStream({ filename: videoFile.filename, rangeHeader: undefined })
|
||||
|
||||
return stream
|
||||
}
|
||||
|
||||
private async generateCaptionReadStream (caption: MVideoCaption): Promise<Readable> {
|
||||
if (caption.storage === FileStorage.FILE_SYSTEM) {
|
||||
return createReadStream(caption.getFSFilePath())
|
||||
}
|
||||
|
||||
const { stream } = await getCaptionReadStream({ filename: caption.filename, rangeHeader: undefined })
|
||||
|
||||
return stream
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private async getArchiveVideo (video: MVideoFullLight) {
|
||||
const source = await VideoSourceModel.loadLatest(video.id)
|
||||
|
||||
const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
|
||||
|
||||
if (source?.keptOriginalFilename) return { source }
|
||||
|
||||
return { videoFile, separatedAudioFile }
|
||||
}
|
||||
|
||||
private getArchiveVideoFilePath (video: MVideo, file: { keptOriginalFilename?: string, filename?: string }) {
|
||||
return join('video-files', video.uuid + extname(file.keptOriginalFilename || file.filename))
|
||||
}
|
||||
|
||||
private getArchiveCaptionFilePath (video: MVideo, caption: MVideoCaptionLanguageUrl) {
|
||||
return join('captions', video.uuid + '-' + caption.language + extname(caption.filename))
|
||||
}
|
||||
|
||||
private getArchiveThumbnailFilePath (video: MVideo, thumbnail: MThumbnail) {
|
||||
return join('thumbnails', video.uuid + extname(thumbnail.filename))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { WatchedWordsListsJSON } from '@peertube/peertube-models'
|
||||
import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
|
||||
import { AbstractUserExporter } from './abstract-user-exporter.js'
|
||||
|
||||
export class WatchedWordsListsExporter extends AbstractUserExporter <WatchedWordsListsJSON> {
|
||||
|
||||
async export () {
|
||||
const data = await WatchedWordsListModel.listForExport({ accountId: this.user.Account.id })
|
||||
|
||||
return {
|
||||
json: {
|
||||
watchedWordLists: data.map(list => ({
|
||||
listName: list.listName,
|
||||
words: list.words,
|
||||
createdAt: list.createdAt.toISOString(),
|
||||
updatedAt: list.updatedAt.toISOString()
|
||||
}))
|
||||
} as WatchedWordsListsJSON,
|
||||
|
||||
staticFiles: []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
import { VideoRateType } from '@peertube/peertube-models'
|
||||
import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
|
||||
import { userRateVideo } from '@server/lib/rate.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
|
||||
export type SanitizedRateObject = { videoUrl: string }
|
||||
|
||||
export abstract class AbstractRatesImporter <ROOT_OBJECT, OBJECT> extends AbstractUserImporter <ROOT_OBJECT, OBJECT, SanitizedRateObject> {
|
||||
|
||||
protected sanitizeRate <O extends { videoUrl: string }> (data: O) {
|
||||
if (!isUrlValid(data.videoUrl)) return undefined
|
||||
|
||||
return pick(data, [ 'videoUrl' ])
|
||||
}
|
||||
|
||||
protected async importRate (data: SanitizedRateObject, rateType: VideoRateType) {
|
||||
const videoUrl = data.videoUrl
|
||||
const videoImmutable = await loadOrCreateVideoIfAllowedForUser(videoUrl)
|
||||
|
||||
if (!videoImmutable) {
|
||||
throw new Error(`Cannot get or create video ${videoUrl} to import user ${rateType}`)
|
||||
}
|
||||
|
||||
const video = await VideoModel.loadFull(videoImmutable.id)
|
||||
|
||||
await userRateVideo({ account: this.user.Account, rateType, video })
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { getFileSize } from '@peertube/peertube-node-utils'
|
||||
import { Awaitable } from '@peertube/peertube-typescript-utils'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { MUserDefault } from '@server/types/models/user/user.js'
|
||||
import { pathExists, readJSON, remove } from 'fs-extra/esm'
|
||||
import { dirname, resolve } from 'path'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
export abstract class AbstractUserImporter <
|
||||
ROOT_OBJECT,
|
||||
OBJECT extends { archiveFiles?: Record<string, string | Record<string, string>> },
|
||||
SANITIZED_OBJECT
|
||||
> {
|
||||
protected user: MUserDefault
|
||||
protected extractedDirectory: string
|
||||
protected jsonFilePath: string
|
||||
|
||||
constructor (options: {
|
||||
user: MUserDefault
|
||||
|
||||
extractedDirectory: string
|
||||
|
||||
jsonFilePath: string
|
||||
}) {
|
||||
this.user = options.user
|
||||
this.extractedDirectory = options.extractedDirectory
|
||||
this.jsonFilePath = options.jsonFilePath
|
||||
}
|
||||
|
||||
getJSONFilePath () {
|
||||
return this.jsonFilePath
|
||||
}
|
||||
|
||||
protected getSafeArchivePathOrThrow (path: string) {
|
||||
if (!path) return undefined
|
||||
|
||||
const resolved = resolve(dirname(this.jsonFilePath), path)
|
||||
if (resolved.startsWith(this.extractedDirectory) !== true) {
|
||||
throw new Error(`Static file path ${resolved} is outside the archive directory ${this.extractedDirectory}`)
|
||||
}
|
||||
|
||||
return resolved
|
||||
}
|
||||
|
||||
protected async cleanupImportedStaticFilePaths (archiveFiles: Record<string, string | Record<string, string>>) {
|
||||
if (!archiveFiles || typeof archiveFiles !== 'object') return
|
||||
|
||||
for (const file of Object.values(archiveFiles)) {
|
||||
if (!file) continue
|
||||
|
||||
try {
|
||||
if (typeof file === 'string') {
|
||||
await remove(this.getSafeArchivePathOrThrow(file))
|
||||
} else { // Avoid recursion to prevent security issue
|
||||
for (const subFile of Object.values(file)) {
|
||||
await remove(this.getSafeArchivePathOrThrow(subFile))
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(`Cannot remove file ${file} after successful import`, { err, ...lTags() })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected async isFileValidOrLog (filePath: string, maxSize: number) {
|
||||
if (!await pathExists(filePath)) {
|
||||
logger.warn(`Do not import file ${filePath} that do not exist in zip`, lTags())
|
||||
return false
|
||||
}
|
||||
|
||||
const size = await getFileSize(filePath)
|
||||
if (size > maxSize) {
|
||||
logger.warn(
|
||||
`Do not import too big file ${filePath} (${size} > ${maxSize})`,
|
||||
lTags()
|
||||
)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async import () {
|
||||
const importData: ROOT_OBJECT = await readJSON(this.jsonFilePath)
|
||||
const summary = {
|
||||
duplicates: 0,
|
||||
success: 0,
|
||||
errors: 0
|
||||
}
|
||||
|
||||
for (const importObject of this.getImportObjects(importData)) {
|
||||
try {
|
||||
const sanitized = this.sanitize(importObject)
|
||||
|
||||
if (!sanitized) {
|
||||
logger.warn('Do not import object after invalid sanitization', { importObject, ...lTags() })
|
||||
summary.errors++
|
||||
continue
|
||||
}
|
||||
|
||||
const result = await this.importObject(sanitized)
|
||||
|
||||
await this.cleanupImportedStaticFilePaths(importObject.archiveFiles)
|
||||
|
||||
if (result.duplicate === true) summary.duplicates++
|
||||
else summary.success++
|
||||
} catch (err) {
|
||||
logger.error('Cannot import object from ' + this.jsonFilePath, { err, importObject, ...lTags() })
|
||||
|
||||
summary.errors++
|
||||
}
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
protected abstract getImportObjects (object: ROOT_OBJECT): OBJECT[]
|
||||
|
||||
protected abstract sanitize (object: OBJECT): SANITIZED_OBJECT | undefined
|
||||
|
||||
protected abstract importObject (object: SANITIZED_OBJECT): Awaitable<{ duplicate: boolean }>
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { BlocklistExportJSON } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
import { addAccountInBlocklist, addServerInBlocklist } from '@server/lib/blocklist.js'
|
||||
import { ServerModel } from '@server/models/server/server.js'
|
||||
import { AccountModel } from '@server/models/account/account.js'
|
||||
import { isValidActorHandle } from '@server/helpers/custom-validators/activitypub/actor.js'
|
||||
import { isHostValid } from '@server/helpers/custom-validators/servers.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type ImportObject = { handle: string | null, host: string | null, archiveFiles?: never }
|
||||
|
||||
export class BlocklistImporter extends AbstractUserImporter<BlocklistExportJSON, ImportObject, ImportObject> {
|
||||
protected getImportObjects (json: BlocklistExportJSON) {
|
||||
return [
|
||||
...json.actors.map(o => ({ handle: o.handle, host: null })),
|
||||
...json.instances.map(o => ({ handle: null, host: o.host }))
|
||||
]
|
||||
}
|
||||
|
||||
protected sanitize (blocklistImportData: ImportObject) {
|
||||
if (!isValidActorHandle(blocklistImportData.handle) && !isHostValid(blocklistImportData.host)) return undefined
|
||||
|
||||
return pick(blocklistImportData, [ 'handle', 'host' ])
|
||||
}
|
||||
|
||||
protected async importObject (blocklistImportData: ImportObject) {
|
||||
if (blocklistImportData.handle) {
|
||||
await this.importAccountBlock(blocklistImportData.handle)
|
||||
} else {
|
||||
await this.importServerBlock(blocklistImportData.host)
|
||||
}
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
|
||||
private async importAccountBlock (handle: string) {
|
||||
const accountToBlock = await AccountModel.loadByHandle(handle)
|
||||
if (!accountToBlock) {
|
||||
logger.info('Account %s was not blocked on user import because it cannot be found in the database.', handle, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
await addAccountInBlocklist({
|
||||
byAccountId: this.user.Account.id,
|
||||
targetAccountId: accountToBlock.id,
|
||||
removeNotificationOfUserId: this.user.id
|
||||
})
|
||||
|
||||
logger.info('Account %s blocked on user import.', handle, lTags())
|
||||
}
|
||||
|
||||
private async importServerBlock (hostToBlock: string) {
|
||||
const serverToBlock = await ServerModel.loadOrCreateByHost(hostToBlock)
|
||||
|
||||
await addServerInBlocklist({
|
||||
byAccountId: this.user.Account.id,
|
||||
targetServerId: serverToBlock.id,
|
||||
removeNotificationOfUserId: this.user.id
|
||||
})
|
||||
|
||||
logger.info('Server %s blocked on user import.', hostToBlock, lTags())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { AccountExportJSON, ActorImageType } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
import { updateLocalActorImageFiles } from '@server/lib/local-actor.js'
|
||||
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
|
||||
import { MAccountDefault } from '@server/types/models/index.js'
|
||||
import { isUserDescriptionValid, isUserDisplayNameValid } from '@server/helpers/custom-validators/users.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type SanitizedObject = Pick<AccountExportJSON, 'description' | 'displayName' | 'archiveFiles'>
|
||||
|
||||
export class AccountImporter extends AbstractUserImporter <AccountExportJSON, AccountExportJSON, SanitizedObject> {
|
||||
|
||||
protected getImportObjects (json: AccountExportJSON) {
|
||||
return [ json ]
|
||||
}
|
||||
|
||||
protected sanitize (blocklistImportData: AccountExportJSON) {
|
||||
if (!isUserDisplayNameValid(blocklistImportData.displayName)) return undefined
|
||||
|
||||
if (!isUserDescriptionValid(blocklistImportData.description)) blocklistImportData.description = null
|
||||
|
||||
return pick(blocklistImportData, [ 'displayName', 'description', 'archiveFiles' ])
|
||||
}
|
||||
|
||||
protected async importObject (accountImportData: SanitizedObject) {
|
||||
const account = this.user.Account
|
||||
|
||||
account.name = accountImportData.displayName
|
||||
account.description = accountImportData.description
|
||||
|
||||
await saveInTransactionWithRetries(account)
|
||||
|
||||
await this.importAvatar(account, accountImportData)
|
||||
|
||||
logger.info('Account %s imported.', account.name, lTags())
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
|
||||
private async importAvatar (account: MAccountDefault, accountImportData: SanitizedObject) {
|
||||
const avatarPath = this.getSafeArchivePathOrThrow(accountImportData.archiveFiles.avatar)
|
||||
if (!avatarPath) return undefined
|
||||
|
||||
if (!await this.isFileValidOrLog(avatarPath, CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max)) return undefined
|
||||
|
||||
await updateLocalActorImageFiles({
|
||||
accountOrChannel: account,
|
||||
imagePhysicalFile: { path: avatarPath },
|
||||
type: ActorImageType.AVATAR,
|
||||
sendActorUpdate: false
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { ActorImageType, ChannelExportJSON } from '@peertube/peertube-models'
|
||||
import { isPlayerChannelThemeSettingValid } from '@server/helpers/custom-validators/player-settings.js'
|
||||
import {
|
||||
isVideoChannelDescriptionValid,
|
||||
isVideoChannelDisplayNameValid,
|
||||
isVideoChannelSupportValid,
|
||||
isVideoChannelUsernameValid
|
||||
} from '@server/helpers/custom-validators/video-channels.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
|
||||
import { sequelizeTypescript } from '@server/initializers/database.js'
|
||||
import { JobQueue } from '@server/lib/job-queue/job-queue.js'
|
||||
import { updateLocalActorImageFiles } from '@server/lib/local-actor.js'
|
||||
import { createLocalVideoChannelWithoutKeys } from '@server/lib/video-channel.js'
|
||||
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
import { MChannelId } from '@server/types/models/index.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type SanitizedObject = Pick<
|
||||
ChannelExportJSON['channels'][0],
|
||||
'name' | 'displayName' | 'description' | 'support' | 'playerSettings' | 'archiveFiles'
|
||||
>
|
||||
|
||||
export class ChannelsImporter extends AbstractUserImporter<ChannelExportJSON, ChannelExportJSON['channels'][0], SanitizedObject> {
|
||||
protected getImportObjects (json: ChannelExportJSON) {
|
||||
return json.channels
|
||||
}
|
||||
|
||||
protected sanitize (channelImportData: ChannelExportJSON['channels'][0]) {
|
||||
if (!isVideoChannelUsernameValid(channelImportData.name)) return undefined
|
||||
if (!isVideoChannelDisplayNameValid(channelImportData.displayName)) return undefined
|
||||
|
||||
if (!isVideoChannelDescriptionValid(channelImportData.description)) channelImportData.description = null
|
||||
if (!isVideoChannelSupportValid(channelImportData.support)) channelImportData.support = null
|
||||
|
||||
if (channelImportData.playerSettings) {
|
||||
if (!isPlayerChannelThemeSettingValid(channelImportData.playerSettings.theme)) channelImportData.playerSettings.theme = undefined
|
||||
}
|
||||
|
||||
return pick(channelImportData, [ 'name', 'displayName', 'description', 'support', 'playerSettings', 'archiveFiles' ])
|
||||
}
|
||||
|
||||
protected async importObject (channelImportData: SanitizedObject) {
|
||||
const account = this.user.Account
|
||||
const existingChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(channelImportData.name)
|
||||
|
||||
if (existingChannel) {
|
||||
logger.info(`Do not import channel ${existingChannel.name} that already exists on this PeerTube instance`, lTags())
|
||||
} else {
|
||||
const videoChannelCreated = await sequelizeTypescript.transaction(async t => {
|
||||
return createLocalVideoChannelWithoutKeys(pick(channelImportData, [ 'displayName', 'name', 'description', 'support' ]), account, t)
|
||||
})
|
||||
|
||||
await this.importPlayerSettings(videoChannelCreated, channelImportData)
|
||||
|
||||
await JobQueue.Instance.createJob({ type: 'actor-keys', payload: { actorId: videoChannelCreated.Actor.id } })
|
||||
|
||||
for (const type of [ ActorImageType.AVATAR, ActorImageType.BANNER ]) {
|
||||
const relativePath = type === ActorImageType.AVATAR
|
||||
? channelImportData.archiveFiles.avatar
|
||||
: channelImportData.archiveFiles.banner
|
||||
|
||||
if (!relativePath) continue
|
||||
|
||||
const absolutePath = this.getSafeArchivePathOrThrow(relativePath)
|
||||
if (!await this.isFileValidOrLog(absolutePath, CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max)) continue
|
||||
|
||||
await updateLocalActorImageFiles({
|
||||
accountOrChannel: videoChannelCreated,
|
||||
imagePhysicalFile: { path: absolutePath },
|
||||
type,
|
||||
sendActorUpdate: false
|
||||
})
|
||||
}
|
||||
|
||||
logger.info('Video channel %s imported.', channelImportData.name, lTags())
|
||||
}
|
||||
|
||||
return {
|
||||
duplicate: !!existingChannel
|
||||
}
|
||||
}
|
||||
|
||||
private async importPlayerSettings (channel: MChannelId, channelImportData: SanitizedObject) {
|
||||
const playerSettings = channelImportData.playerSettings
|
||||
if (!playerSettings?.theme) return
|
||||
|
||||
await PlayerSettingModel.create({
|
||||
theme: playerSettings.theme,
|
||||
channelId: channel.id
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { DislikesExportJSON } from '@peertube/peertube-models'
|
||||
import { AbstractRatesImporter, SanitizedRateObject } from './abstract-rates-importer.js'
|
||||
|
||||
export class DislikesImporter extends AbstractRatesImporter <DislikesExportJSON, DislikesExportJSON['dislikes'][0]> {
|
||||
|
||||
protected getImportObjects (json: DislikesExportJSON) {
|
||||
return json.dislikes
|
||||
}
|
||||
|
||||
protected sanitize (o: DislikesExportJSON['dislikes'][0]) {
|
||||
return this.sanitizeRate(o)
|
||||
}
|
||||
|
||||
protected async importObject (dislikesImportData: SanitizedRateObject) {
|
||||
return this.importRate(dislikesImportData, 'dislike')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { FollowingExportJSON } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
import { JobQueue } from '@server/lib/job-queue/job-queue.js'
|
||||
import { isValidActorHandle } from '@server/helpers/custom-validators/activitypub/actor.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type SanitizedObject = Pick<FollowingExportJSON['following'][0], 'targetHandle'>
|
||||
|
||||
export class FollowingImporter extends AbstractUserImporter <FollowingExportJSON, FollowingExportJSON['following'][0], SanitizedObject> {
|
||||
|
||||
protected getImportObjects (json: FollowingExportJSON) {
|
||||
return json.following
|
||||
}
|
||||
|
||||
protected sanitize (followingImportData: FollowingExportJSON['following'][0]) {
|
||||
if (!isValidActorHandle(followingImportData.targetHandle)) return undefined
|
||||
|
||||
return pick(followingImportData, [ 'targetHandle' ])
|
||||
}
|
||||
|
||||
protected async importObject (followingImportData: SanitizedObject) {
|
||||
const [ name, host ] = followingImportData.targetHandle.split('@')
|
||||
|
||||
const payload = {
|
||||
name,
|
||||
host,
|
||||
assertIsChannel: true,
|
||||
followerActorId: this.user.Account.Actor.id
|
||||
}
|
||||
|
||||
await JobQueue.Instance.createJob({ type: 'activitypub-follow', payload })
|
||||
|
||||
logger.info('Subscription job of %s created on user import.', followingImportData.targetHandle, lTags())
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
}
|
||||
10
server/core/lib/user-import-export/importers/index.ts
Normal file
10
server/core/lib/user-import-export/importers/index.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
export * from './account-blocklist-importer.js'
|
||||
export * from './account-importer.js'
|
||||
export * from './channels-importer.js'
|
||||
export * from './dislikes-importer.js'
|
||||
export * from './following-importer.js'
|
||||
export * from './likes-importer.js'
|
||||
export * from './user-settings-importer.js'
|
||||
export * from './user-video-history-importer.js'
|
||||
export * from './video-playlists-importer.js'
|
||||
export * from './videos-importer.js'
|
||||
@@ -0,0 +1,17 @@
|
||||
import { LikesExportJSON } from '@peertube/peertube-models'
|
||||
import { AbstractRatesImporter, SanitizedRateObject } from './abstract-rates-importer.js'
|
||||
|
||||
export class LikesImporter extends AbstractRatesImporter <LikesExportJSON, LikesExportJSON['likes'][0]> {
|
||||
|
||||
protected getImportObjects (json: LikesExportJSON) {
|
||||
return json.likes
|
||||
}
|
||||
|
||||
protected sanitize (o: LikesExportJSON['likes'][0]) {
|
||||
return this.sanitizeRate(o)
|
||||
}
|
||||
|
||||
protected async importObject (likesImportData: SanitizedRateObject) {
|
||||
return this.importRate(likesImportData, 'like')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { AutoTagPoliciesJSON, AutomaticTagPolicy } from '@peertube/peertube-models'
|
||||
import { isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
|
||||
import { setAccountAutomaticTagsPolicy } from '@server/lib/automatic-tags/automatic-tags.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
|
||||
type SanitizedObject = AutoTagPoliciesJSON['reviewComments']
|
||||
|
||||
export class ReviewCommentsTagPoliciesImporter
|
||||
extends AbstractUserImporter<AutoTagPoliciesJSON, AutoTagPoliciesJSON['reviewComments'] & { archiveFiles?: never }, SanitizedObject>
|
||||
{
|
||||
protected getImportObjects (json: AutoTagPoliciesJSON) {
|
||||
if (!json.reviewComments) return []
|
||||
|
||||
return [ json.reviewComments ]
|
||||
}
|
||||
|
||||
protected sanitize (data: AutoTagPoliciesJSON['reviewComments']) {
|
||||
return data.filter(d => isWatchedWordListNameValid(d.name))
|
||||
}
|
||||
|
||||
protected async importObject (data: SanitizedObject) {
|
||||
await setAccountAutomaticTagsPolicy({
|
||||
account: this.user.Account,
|
||||
policy: AutomaticTagPolicy.REVIEW_COMMENT,
|
||||
tags: data.map(v => v.name)
|
||||
})
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { UserNotificationSetting, UserSettingsExportJSON } from '@peertube/peertube-models'
|
||||
import { exists } from '@server/helpers/custom-validators/misc.js'
|
||||
import { isThemeNameValid } from '@server/helpers/custom-validators/plugins.js'
|
||||
import { isUserNotificationSettingValid } from '@server/helpers/custom-validators/user-notifications.js'
|
||||
import {
|
||||
isUserAutoPlayNextVideoPlaylistValid,
|
||||
isUserAutoPlayNextVideoValid,
|
||||
isUserAutoPlayVideoValid,
|
||||
isUserLanguage,
|
||||
isUserNSFWPolicyValid,
|
||||
isUserP2PEnabledValid,
|
||||
isUserVideoLanguages,
|
||||
isUserVideosHistoryEnabledValid
|
||||
} from '@server/helpers/custom-validators/users.js'
|
||||
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { isThemeRegistered } from '@server/lib/plugins/theme-utils.js'
|
||||
import { UserNotificationSettingModel } from '@server/models/user/user-notification-setting.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type SanitizedObject = Pick<
|
||||
UserSettingsExportJSON,
|
||||
| 'nsfwPolicy'
|
||||
| 'autoPlayVideo'
|
||||
| 'autoPlayNextVideo'
|
||||
| 'autoPlayNextVideoPlaylist'
|
||||
| 'p2pEnabled'
|
||||
| 'videosHistoryEnabled'
|
||||
| 'videoLanguages'
|
||||
| 'language'
|
||||
| 'theme'
|
||||
| 'notificationSettings'
|
||||
>
|
||||
|
||||
export class UserSettingsImporter extends AbstractUserImporter<UserSettingsExportJSON, UserSettingsExportJSON, SanitizedObject> {
|
||||
protected getImportObjects (json: UserSettingsExportJSON) {
|
||||
return [ json ]
|
||||
}
|
||||
|
||||
protected sanitize (o: UserSettingsExportJSON) {
|
||||
if (!isUserNSFWPolicyValid(o.nsfwPolicy)) o.nsfwPolicy = undefined
|
||||
if (!isUserAutoPlayVideoValid(o.autoPlayVideo)) o.autoPlayVideo = undefined
|
||||
if (!isUserAutoPlayNextVideoValid(o.autoPlayNextVideo)) o.autoPlayNextVideo = undefined
|
||||
if (!isUserAutoPlayNextVideoPlaylistValid(o.autoPlayNextVideoPlaylist)) o.autoPlayNextVideoPlaylist = undefined
|
||||
if (!isUserP2PEnabledValid(o.p2pEnabled)) o.p2pEnabled = undefined
|
||||
if (!isUserVideosHistoryEnabledValid(o.videosHistoryEnabled)) o.videosHistoryEnabled = undefined
|
||||
if (!isUserVideoLanguages(o.videoLanguages)) o.videoLanguages = undefined
|
||||
if (!isUserLanguage(o.language)) o.language = undefined
|
||||
if (!isThemeNameValid(o.theme) || !isThemeRegistered(o.theme)) o.theme = undefined
|
||||
|
||||
for (const key of Object.keys(o.notificationSettings || {})) {
|
||||
if (!isUserNotificationSettingValid(o.notificationSettings[key])) (o.notificationSettings[key] as any) = undefined
|
||||
}
|
||||
|
||||
return pick(o, [
|
||||
'nsfwPolicy',
|
||||
'autoPlayVideo',
|
||||
'autoPlayNextVideo',
|
||||
'autoPlayNextVideoPlaylist',
|
||||
'p2pEnabled',
|
||||
'videosHistoryEnabled',
|
||||
'videoLanguages',
|
||||
'language',
|
||||
'theme',
|
||||
'notificationSettings'
|
||||
])
|
||||
}
|
||||
|
||||
protected async importObject (userImportData: SanitizedObject) {
|
||||
if (exists(userImportData.nsfwPolicy)) this.user.nsfwPolicy = userImportData.nsfwPolicy
|
||||
if (exists(userImportData.autoPlayVideo)) this.user.autoPlayVideo = userImportData.autoPlayVideo
|
||||
if (exists(userImportData.autoPlayNextVideo)) this.user.autoPlayNextVideo = userImportData.autoPlayNextVideo
|
||||
if (exists(userImportData.autoPlayNextVideoPlaylist)) this.user.autoPlayNextVideoPlaylist = userImportData.autoPlayNextVideoPlaylist
|
||||
if (exists(userImportData.p2pEnabled)) this.user.p2pEnabled = userImportData.p2pEnabled
|
||||
if (exists(userImportData.videosHistoryEnabled)) this.user.videosHistoryEnabled = userImportData.videosHistoryEnabled
|
||||
if (exists(userImportData.videoLanguages)) this.user.videoLanguages = userImportData.videoLanguages
|
||||
if (exists(userImportData.language)) this.user.language = userImportData.language
|
||||
if (exists(userImportData.theme)) this.user.theme = userImportData.theme
|
||||
|
||||
await saveInTransactionWithRetries(this.user)
|
||||
|
||||
await this.updateSettings(userImportData.notificationSettings)
|
||||
|
||||
logger.info('Settings of user %s imported.', this.user.username, lTags())
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
|
||||
private async updateSettings (settingsImportData: UserSettingsExportJSON['notificationSettings']) {
|
||||
const values: UserNotificationSetting = {
|
||||
newVideoFromSubscription: settingsImportData.newVideoFromSubscription,
|
||||
newCommentOnMyVideo: settingsImportData.newCommentOnMyVideo,
|
||||
myVideoImportFinished: settingsImportData.myVideoImportFinished,
|
||||
myVideoPublished: settingsImportData.myVideoPublished,
|
||||
abuseAsModerator: settingsImportData.abuseAsModerator,
|
||||
videoAutoBlacklistAsModerator: settingsImportData.videoAutoBlacklistAsModerator,
|
||||
blacklistOnMyVideo: settingsImportData.blacklistOnMyVideo,
|
||||
newUserRegistration: settingsImportData.newUserRegistration,
|
||||
commentMention: settingsImportData.commentMention,
|
||||
newFollow: settingsImportData.newFollow,
|
||||
newInstanceFollower: settingsImportData.newInstanceFollower,
|
||||
abuseNewMessage: settingsImportData.abuseNewMessage,
|
||||
abuseStateChange: settingsImportData.abuseStateChange,
|
||||
autoInstanceFollowing: settingsImportData.autoInstanceFollowing,
|
||||
newPeerTubeVersion: settingsImportData.newPeerTubeVersion,
|
||||
newPluginVersion: settingsImportData.newPluginVersion,
|
||||
myVideoStudioEditionFinished: settingsImportData.myVideoStudioEditionFinished,
|
||||
myVideoTranscriptionGenerated: settingsImportData.myVideoTranscriptionGenerated
|
||||
}
|
||||
|
||||
await UserNotificationSettingModel.updateUserSettings(values, this.user.id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
|
||||
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
|
||||
import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
|
||||
import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
|
||||
type SanitizedObject = Pick<UserVideoHistoryExportJSON['watchedVideos'][0], 'videoUrl' | 'lastTimecode' | 'archiveFiles'>
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export class UserVideoHistoryImporter extends AbstractUserImporter <UserVideoHistoryExportJSON, UserVideoHistoryExportJSON['watchedVideos'][0], SanitizedObject> {
|
||||
|
||||
protected getImportObjects (json: UserVideoHistoryExportJSON) {
|
||||
return json.watchedVideos
|
||||
}
|
||||
|
||||
protected sanitize (data: UserVideoHistoryExportJSON['watchedVideos'][0]) {
|
||||
if (!isUrlValid(data.videoUrl)) return undefined
|
||||
|
||||
return pick(data, [ 'videoUrl', 'lastTimecode' ])
|
||||
}
|
||||
|
||||
protected async importObject (data: SanitizedObject) {
|
||||
if (!this.user.videosHistoryEnabled) return { duplicate: false }
|
||||
|
||||
const videoUrl = data.videoUrl
|
||||
const videoImmutable = await loadOrCreateVideoIfAllowedForUser(videoUrl)
|
||||
|
||||
if (!videoImmutable) {
|
||||
throw new Error(`Cannot get or create video ${videoUrl} to import user history`)
|
||||
}
|
||||
|
||||
await UserVideoHistoryModel.upsert({
|
||||
videoId: videoImmutable.id,
|
||||
userId: this.user.id,
|
||||
currentTime: data.lastTimecode
|
||||
})
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
import { VideoPlaylistPrivacy, VideoPlaylistType, VideoPlaylistsExportJSON } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { buildUUID } from '@peertube/peertube-node-utils'
|
||||
import { MChannelBannerAccountDefault, MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'
|
||||
import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js'
|
||||
import { sequelizeTypescript } from '@server/initializers/database.js'
|
||||
import { updateLocalPlaylistMiniatureFromExisting } from '@server/lib/thumbnail.js'
|
||||
import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js'
|
||||
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
|
||||
import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
|
||||
import {
|
||||
isVideoPlaylistDescriptionValid,
|
||||
isVideoPlaylistNameValid,
|
||||
isVideoPlaylistPrivacyValid,
|
||||
isVideoPlaylistTimestampValid,
|
||||
isVideoPlaylistTypeValid
|
||||
} from '@server/helpers/custom-validators/video-playlists.js'
|
||||
import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js'
|
||||
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
|
||||
import { isArray } from '@server/helpers/custom-validators/misc.js'
|
||||
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { generateThumbnailForPlaylist } from '@server/lib/video-playlist.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type ImportObject = VideoPlaylistsExportJSON['videoPlaylists'][0]
|
||||
type SanitizedObject = Pick<ImportObject, 'type' | 'displayName' | 'privacy' | 'elements' | 'description' | 'channel' | 'archiveFiles'>
|
||||
|
||||
export class VideoPlaylistsImporter extends AbstractUserImporter<VideoPlaylistsExportJSON, ImportObject, SanitizedObject> {
|
||||
protected getImportObjects (json: VideoPlaylistsExportJSON) {
|
||||
return json.videoPlaylists
|
||||
}
|
||||
|
||||
protected sanitize (o: ImportObject) {
|
||||
if (!isVideoPlaylistTypeValid(o.type)) return undefined
|
||||
if (!isVideoPlaylistNameValid(o.displayName)) return undefined
|
||||
if (!isVideoPlaylistPrivacyValid(o.privacy)) return undefined
|
||||
if (!isArray(o.elements)) return undefined
|
||||
|
||||
if (o.channel?.name && !isActorPreferredUsernameValid(o.channel.name)) o.channel = undefined
|
||||
if (!isVideoPlaylistDescriptionValid(o.description)) o.description = undefined
|
||||
|
||||
o.elements = o.elements.filter(e => {
|
||||
if (!isUrlValid(e.videoUrl)) return false
|
||||
if (e.startTimestamp && !isVideoPlaylistTimestampValid(e.startTimestamp)) return false
|
||||
if (e.stopTimestamp && !isVideoPlaylistTimestampValid(e.stopTimestamp)) return false
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
return pick(o, [ 'type', 'displayName', 'privacy', 'elements', 'channel', 'description', 'archiveFiles' ])
|
||||
}
|
||||
|
||||
protected async importObject (playlistImportData: SanitizedObject) {
|
||||
const existingPlaylist = await VideoPlaylistModel.loadRegularByAccountAndName(this.user.Account, playlistImportData.displayName)
|
||||
|
||||
if (existingPlaylist) {
|
||||
logger.info(`Do not import playlist ${playlistImportData.displayName} that already exists in the account`, lTags())
|
||||
return { duplicate: true }
|
||||
}
|
||||
|
||||
const videoPlaylist = playlistImportData.type === VideoPlaylistType.WATCH_LATER
|
||||
? await this.getWatchLaterPlaylist()
|
||||
: await this.createPlaylist(playlistImportData)
|
||||
|
||||
await this.createElements(videoPlaylist, playlistImportData)
|
||||
|
||||
await sendCreateVideoPlaylist(videoPlaylist, undefined)
|
||||
|
||||
logger.info('Video playlist %s imported.', videoPlaylist.name, lTags(videoPlaylist.uuid))
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
|
||||
private async createPlaylist (playlistImportData: SanitizedObject) {
|
||||
let videoChannel: MChannelBannerAccountDefault
|
||||
|
||||
if (playlistImportData.channel.name) {
|
||||
videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(playlistImportData.channel.name)
|
||||
|
||||
if (!videoChannel) throw new Error(`Channel ${playlistImportData} not found`)
|
||||
if (videoChannel.accountId !== this.user.Account.id) {
|
||||
throw new Error(`Channel ${videoChannel.name} is not owned by user ${this.user.username}`)
|
||||
}
|
||||
} else if (playlistImportData.privacy !== VideoPlaylistPrivacy.PRIVATE) {
|
||||
throw new Error('Cannot create a non private playlist without channel')
|
||||
}
|
||||
|
||||
const playlist: MVideoPlaylistFull = new VideoPlaylistModel({
|
||||
name: playlistImportData.displayName,
|
||||
description: playlistImportData.description,
|
||||
privacy: playlistImportData.privacy,
|
||||
|
||||
uuid: buildUUID(),
|
||||
videoChannelId: videoChannel?.id,
|
||||
ownerAccountId: this.user.Account.id
|
||||
})
|
||||
playlist.url = getLocalVideoPlaylistActivityPubUrl(playlist)
|
||||
playlist.VideoChannel = videoChannel
|
||||
playlist.OwnerAccount = this.user.Account
|
||||
|
||||
await saveInTransactionWithRetries(playlist)
|
||||
|
||||
await this.createThumbnail(playlist, playlistImportData)
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
private async getWatchLaterPlaylist () {
|
||||
return VideoPlaylistModel.loadWatchLaterOf(this.user.Account)
|
||||
}
|
||||
|
||||
private async createThumbnail (playlist: MVideoPlaylistThumbnail, playlistImportData: SanitizedObject) {
|
||||
const thumbnailPath = this.getSafeArchivePathOrThrow(playlistImportData.archiveFiles.thumbnail)
|
||||
if (!thumbnailPath) return undefined
|
||||
|
||||
if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.FILE_SIZE.max)) return undefined
|
||||
|
||||
const thumbnail = await updateLocalPlaylistMiniatureFromExisting({
|
||||
inputPath: thumbnailPath,
|
||||
playlist,
|
||||
automaticallyGenerated: false
|
||||
})
|
||||
|
||||
await playlist.setAndSaveThumbnail(thumbnail, undefined)
|
||||
}
|
||||
|
||||
private async createElements (playlist: MVideoPlaylistThumbnail, playlistImportData: SanitizedObject) {
|
||||
const elementsToCreate: { videoId: number, startTimestamp: number, stopTimestamp: number }[] = []
|
||||
|
||||
for (const element of playlistImportData.elements.slice(0, USER_IMPORT.MAX_PLAYLIST_ELEMENTS)) {
|
||||
const video = await loadOrCreateVideoIfAllowedForUser(element.videoUrl)
|
||||
|
||||
if (!video) {
|
||||
logger.debug(`Cannot get or create video ${element.videoUrl} to create playlist element in user import`, lTags())
|
||||
continue
|
||||
}
|
||||
|
||||
elementsToCreate.push({
|
||||
videoId: video.id,
|
||||
startTimestamp: element.startTimestamp,
|
||||
stopTimestamp: element.stopTimestamp
|
||||
})
|
||||
}
|
||||
|
||||
await sequelizeTypescript.transaction(async t => {
|
||||
let position = await VideoPlaylistElementModel.getNextPositionOf(playlist.id, t)
|
||||
|
||||
for (const elementToCreate of elementsToCreate) {
|
||||
const playlistElement = new VideoPlaylistElementModel({
|
||||
position,
|
||||
startTimestamp: elementToCreate.startTimestamp,
|
||||
stopTimestamp: elementToCreate.stopTimestamp,
|
||||
videoPlaylistId: playlist.id,
|
||||
videoId: elementToCreate.videoId
|
||||
})
|
||||
await playlistElement.save({ transaction: t })
|
||||
|
||||
playlistElement.url = getLocalVideoPlaylistElementActivityPubUrl(playlist, playlistElement)
|
||||
await playlistElement.save({ transaction: t })
|
||||
|
||||
if (playlist.shouldGenerateThumbnailWithNewElement(playlistElement)) {
|
||||
const video = await VideoModel.loadFull(elementToCreate.videoId)
|
||||
|
||||
generateThumbnailForPlaylist(playlist, video)
|
||||
.catch(err => logger.error('Cannot generate thumbnail from playlist', { err }))
|
||||
}
|
||||
|
||||
position++
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
354
server/core/lib/user-import-export/importers/videos-importer.ts
Normal file
354
server/core/lib/user-import-export/importers/videos-importer.ts
Normal file
@@ -0,0 +1,354 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
|
||||
import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models'
|
||||
import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
|
||||
import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js'
|
||||
import { isPlayerVideoThemeSettingValid } from '@server/helpers/custom-validators/player-settings.js'
|
||||
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
|
||||
import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js'
|
||||
import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js'
|
||||
import { isLiveLatencyModeValid, isLiveScheduleValid } from '@server/helpers/custom-validators/video-lives.js'
|
||||
import {
|
||||
isPasswordValid,
|
||||
isVideoCategoryValid,
|
||||
isVideoCommentsPolicyValid,
|
||||
isVideoDescriptionValid,
|
||||
isVideoDurationValid,
|
||||
isVideoLanguageValid,
|
||||
isVideoLicenceValid,
|
||||
isVideoNameValid,
|
||||
isVideoOriginallyPublishedAtValid,
|
||||
isVideoPrivacyValid,
|
||||
isVideoReplayPrivacyValid,
|
||||
isVideoSourceFilenameValid,
|
||||
isVideoSupportValid,
|
||||
isVideoTagValid
|
||||
} from '@server/helpers/custom-validators/videos.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
|
||||
import { LocalVideoCreator, ThumbnailOptions } from '@server/lib/local-video-creator.js'
|
||||
import { isLocalVideoFileAccepted } from '@server/lib/moderation.js'
|
||||
import { Hooks } from '@server/lib/plugins/hooks.js'
|
||||
import { isUserQuotaValid } from '@server/lib/user.js'
|
||||
import { createLocalCaption, updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
|
||||
import { buildNextVideoState } from '@server/lib/video-state.js'
|
||||
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MChannelId, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { parse } from 'path'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
type ImportObject = VideoExportJSON['videos'][0]
|
||||
type SanitizedObject = Pick<
|
||||
ImportObject,
|
||||
| 'name'
|
||||
| 'duration'
|
||||
| 'channel'
|
||||
| 'privacy'
|
||||
| 'archiveFiles'
|
||||
| 'captions'
|
||||
| 'category'
|
||||
| 'licence'
|
||||
| 'language'
|
||||
| 'description'
|
||||
| 'support'
|
||||
| 'nsfw'
|
||||
| 'isLive'
|
||||
| 'commentsPolicy'
|
||||
| 'downloadEnabled'
|
||||
| 'waitTranscoding'
|
||||
| 'originallyPublishedAt'
|
||||
| 'tags'
|
||||
| 'live'
|
||||
| 'passwords'
|
||||
| 'source'
|
||||
| 'chapters'
|
||||
| 'playerSettings'
|
||||
>
|
||||
|
||||
export class VideosImporter extends AbstractUserImporter<VideoExportJSON, ImportObject, SanitizedObject> {
|
||||
protected getImportObjects (json: VideoExportJSON) {
|
||||
return json.videos
|
||||
}
|
||||
|
||||
protected sanitize (o: ImportObject) {
|
||||
if (!isVideoNameValid(o.name)) return undefined
|
||||
if (!isVideoDurationValid(o.duration + '')) return undefined
|
||||
if (!isVideoChannelUsernameValid(o.channel?.name)) return undefined
|
||||
if (!isVideoPrivacyValid(o.privacy)) return undefined
|
||||
if (o.isLive !== true && !o.archiveFiles?.videoFile) return undefined
|
||||
|
||||
if (!isVideoCategoryValid(o.category)) o.category = null
|
||||
if (!o.licence || !isVideoLicenceValid(o.licence)) o.licence = CONFIG.DEFAULTS.PUBLISH.LICENCE
|
||||
if (!isVideoLanguageValid(o.language)) o.language = null
|
||||
if (!isVideoDescriptionValid(o.description)) o.description = null
|
||||
if (!isVideoSupportValid(o.support)) o.support = null
|
||||
|
||||
if (!isBooleanValid(o.nsfw)) o.nsfw = false
|
||||
if (!isBooleanValid(o.isLive)) o.isLive = false
|
||||
if (!isBooleanValid(o.downloadEnabled)) o.downloadEnabled = CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED
|
||||
if (!isBooleanValid(o.waitTranscoding)) o.waitTranscoding = true
|
||||
|
||||
if (!o.commentsPolicy || !isVideoCommentsPolicyValid(o.commentsPolicy)) {
|
||||
o.commentsPolicy = CONFIG.DEFAULTS.PUBLISH.COMMENTS_POLICY
|
||||
}
|
||||
|
||||
if (!isVideoSourceFilenameValid(o.source?.inputFilename)) o.source = undefined
|
||||
|
||||
if (!isVideoOriginallyPublishedAtValid(o.originallyPublishedAt)) o.originallyPublishedAt = null
|
||||
|
||||
if (!isArray(o.tags)) o.tags = []
|
||||
if (!isArray(o.captions)) o.captions = []
|
||||
if (!isArray(o.chapters)) o.chapters = []
|
||||
|
||||
o.tags = o.tags.filter(t => isVideoTagValid(t))
|
||||
|
||||
o.captions = o.captions.filter(c => isVideoCaptionLanguageValid(c.language))
|
||||
o.chapters = o.chapters.filter(c => isVideoChapterTimecodeValid(c.timecode) && isVideoChapterTitleValid(c.title))
|
||||
|
||||
if (o.isLive) {
|
||||
if (!o.live) return undefined
|
||||
if (!isBooleanValid(o.live.permanentLive)) return undefined
|
||||
|
||||
if (!isBooleanValid(o.live.saveReplay)) o.live.saveReplay = false
|
||||
if (o.live.saveReplay && !isVideoReplayPrivacyValid(o.live.replaySettings.privacy)) return undefined
|
||||
|
||||
if (!o.live.latencyMode || !isLiveLatencyModeValid(o.live.latencyMode)) o.live.latencyMode = LiveVideoLatencyMode.DEFAULT
|
||||
|
||||
if (!o.live.streamKey) o.live.streamKey = buildUUID()
|
||||
else if (!isUUIDValid(o.live.streamKey)) return undefined
|
||||
|
||||
if (!isArray(o.live.schedules)) o.live.schedules = []
|
||||
|
||||
o.live.schedules = o.live.schedules.filter(s => isLiveScheduleValid(s))
|
||||
}
|
||||
|
||||
if (o.playerSettings) {
|
||||
if (!isPlayerVideoThemeSettingValid(o.playerSettings.theme)) o.playerSettings.theme = undefined
|
||||
}
|
||||
|
||||
if (o.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
|
||||
if (!isArray(o.passwords)) return undefined
|
||||
// Refuse the import rather than handle only a portion of the passwords, which can be difficult for video owners to debug
|
||||
if (o.passwords.some(p => !isPasswordValid(p))) return undefined
|
||||
}
|
||||
|
||||
return pick(o, [
|
||||
'name',
|
||||
'duration',
|
||||
'channel',
|
||||
'privacy',
|
||||
'archiveFiles',
|
||||
'category',
|
||||
'licence',
|
||||
'language',
|
||||
'description',
|
||||
'support',
|
||||
'nsfw',
|
||||
'isLive',
|
||||
'commentsPolicy',
|
||||
'downloadEnabled',
|
||||
'waitTranscoding',
|
||||
'originallyPublishedAt',
|
||||
'tags',
|
||||
'captions',
|
||||
'live',
|
||||
'passwords',
|
||||
'source',
|
||||
'chapters',
|
||||
'playerSettings'
|
||||
])
|
||||
}
|
||||
|
||||
protected async importObject (videoImportData: SanitizedObject) {
|
||||
const videoFilePath = !videoImportData.isLive
|
||||
? this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.videoFile)
|
||||
: null
|
||||
|
||||
const videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(videoImportData.channel.name)
|
||||
if (!videoChannel) throw new Error(`Channel ${videoImportData} not found`)
|
||||
if (videoChannel.accountId !== this.user.Account.id) {
|
||||
throw new Error(`Channel ${videoChannel.name} is not owned by user ${this.user.username}`)
|
||||
}
|
||||
|
||||
const existingVideo = await VideoModel.loadByNameAndChannel(videoChannel, videoImportData.name)
|
||||
if (existingVideo && Math.abs(existingVideo.duration - videoImportData.duration) <= 1) {
|
||||
logger.info(`Do not import video ${videoImportData.name} that already exists in the account`, lTags())
|
||||
return { duplicate: true }
|
||||
}
|
||||
|
||||
const videoSize = videoFilePath
|
||||
? await getFileSize(videoFilePath)
|
||||
: undefined
|
||||
|
||||
let duration = 0
|
||||
|
||||
let ffprobe: FfprobeData
|
||||
if (videoFilePath) {
|
||||
if (await isUserQuotaValid({ userId: this.user.id, uploadSize: videoSize, checkDaily: false }) === false) {
|
||||
throw new Error(`Cannot import video ${videoImportData.name} for user ${this.user.username} because of exceeded quota`)
|
||||
}
|
||||
|
||||
await this.checkVideoFileIsAcceptedOrThrow({ videoFilePath, size: videoSize, channel: videoChannel, videoImportData })
|
||||
|
||||
ffprobe = await ffprobePromise(videoFilePath)
|
||||
duration = await getVideoStreamDuration(videoFilePath, ffprobe)
|
||||
}
|
||||
|
||||
const thumbnailPath = this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.thumbnail)
|
||||
|
||||
const thumbnails: ThumbnailOptions = []
|
||||
for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) {
|
||||
if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max)) continue
|
||||
|
||||
thumbnails.push({
|
||||
path: thumbnailPath,
|
||||
automaticallyGenerated: false,
|
||||
keepOriginal: true,
|
||||
type
|
||||
})
|
||||
}
|
||||
|
||||
const localVideoCreator = new LocalVideoCreator({
|
||||
lTags,
|
||||
|
||||
videoFile: videoFilePath
|
||||
? { path: videoFilePath, probe: ffprobe }
|
||||
: undefined,
|
||||
|
||||
user: this.user,
|
||||
channel: videoChannel,
|
||||
|
||||
chapters: videoImportData.chapters,
|
||||
fallbackChapters: {
|
||||
fromDescription: false,
|
||||
finalFallback: undefined
|
||||
},
|
||||
|
||||
videoAttributes: {
|
||||
...pick(videoImportData, [
|
||||
'name',
|
||||
'category',
|
||||
'licence',
|
||||
'language',
|
||||
'privacy',
|
||||
'description',
|
||||
'support',
|
||||
'isLive',
|
||||
'nsfw',
|
||||
'tags',
|
||||
'commentsPolicy',
|
||||
'downloadEnabled',
|
||||
'waitTranscoding',
|
||||
'originallyPublishedAt'
|
||||
]),
|
||||
|
||||
videoPasswords: videoImportData.passwords,
|
||||
duration,
|
||||
|
||||
inputFilename: videoImportData.source?.inputFilename,
|
||||
|
||||
state: videoImportData.isLive
|
||||
? VideoState.WAITING_FOR_LIVE
|
||||
: buildNextVideoState()
|
||||
},
|
||||
|
||||
liveAttributes: videoImportData.live,
|
||||
|
||||
videoAttributeResultHook: 'filter:api.video.user-import.video-attribute.result',
|
||||
|
||||
thumbnails
|
||||
})
|
||||
|
||||
const { video } = await localVideoCreator.create()
|
||||
|
||||
await this.importCaptions(video, videoImportData)
|
||||
await this.importPlayerSettings(video, videoImportData)
|
||||
|
||||
logger.info('Video %s imported.', video.name, lTags(video.uuid))
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
|
||||
private async importCaptions (video: MVideoFullLight, videoImportData: SanitizedObject) {
|
||||
const captionPaths: string[] = []
|
||||
let updateHLS = false
|
||||
|
||||
for (const captionImport of videoImportData.captions) {
|
||||
const relativeFilePath = videoImportData.archiveFiles?.captions?.[captionImport.language]
|
||||
|
||||
if (!relativeFilePath) {
|
||||
logger.warn('Cannot import caption ' + captionImport.language + ': file does not exist in the archive', lTags(video.uuid))
|
||||
continue
|
||||
}
|
||||
|
||||
const absoluteFilePath = this.getSafeArchivePathOrThrow(relativeFilePath)
|
||||
|
||||
if (!await this.isFileValidOrLog(absoluteFilePath, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)) continue
|
||||
|
||||
const caption = await createLocalCaption({
|
||||
video,
|
||||
language: captionImport.language,
|
||||
path: absoluteFilePath,
|
||||
automaticallyGenerated: captionImport.automaticallyGenerated === true
|
||||
})
|
||||
|
||||
captionPaths.push(absoluteFilePath)
|
||||
|
||||
if (caption.m3u8Filename) updateHLS = true
|
||||
}
|
||||
|
||||
if (updateHLS && video.getHLSPlaylist()) {
|
||||
await updateHLSMasterOnCaptionChange(video, video.getHLSPlaylist())
|
||||
}
|
||||
|
||||
return captionPaths
|
||||
}
|
||||
|
||||
private async importPlayerSettings (video: MVideoFullLight, videoImportData: SanitizedObject) {
|
||||
const playerSettings = videoImportData.playerSettings
|
||||
if (!playerSettings?.theme) return
|
||||
|
||||
await PlayerSettingModel.create({
|
||||
theme: playerSettings.theme,
|
||||
videoId: video.id
|
||||
})
|
||||
}
|
||||
|
||||
private async checkVideoFileIsAcceptedOrThrow (options: {
|
||||
videoFilePath: string
|
||||
size: number
|
||||
channel: MChannelId
|
||||
videoImportData: SanitizedObject
|
||||
}) {
|
||||
const { videoFilePath, size, videoImportData, channel } = options
|
||||
|
||||
// Check we accept this video
|
||||
const acceptParameters = {
|
||||
videoBody: {
|
||||
...videoImportData,
|
||||
|
||||
channelId: channel.id
|
||||
},
|
||||
videoFile: {
|
||||
path: videoFilePath,
|
||||
filename: parse(videoFilePath).name,
|
||||
size,
|
||||
originalname: null
|
||||
},
|
||||
user: this.user
|
||||
}
|
||||
const acceptedResult = await Hooks.wrapFun(isLocalVideoFileAccepted, acceptParameters, 'filter:api.video.user-import.accept.result')
|
||||
|
||||
if (acceptedResult?.accepted !== true) {
|
||||
logger.info('Refused local video file to import.', { acceptedResult, acceptParameters, ...lTags() })
|
||||
|
||||
throw new Error('Video file is not accepted: ' + acceptedResult.errorMessage || 'unknown reason')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { WatchedWordsListsJSON } from '@peertube/peertube-models'
|
||||
import { areWatchedWordsValid, isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
|
||||
import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
|
||||
import { AbstractUserImporter } from './abstract-user-importer.js'
|
||||
|
||||
type SanitizedObject = Pick<WatchedWordsListsJSON['watchedWordLists'][0], 'listName' | 'words'>
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export class WatchedWordsListsImporter extends AbstractUserImporter <WatchedWordsListsJSON, WatchedWordsListsJSON['watchedWordLists'][0], SanitizedObject> {
|
||||
|
||||
protected getImportObjects (json: WatchedWordsListsJSON) {
|
||||
return json.watchedWordLists
|
||||
}
|
||||
|
||||
protected sanitize (data: WatchedWordsListsJSON['watchedWordLists'][0]) {
|
||||
if (!isWatchedWordListNameValid(data.listName)) return undefined
|
||||
if (!areWatchedWordsValid(data.words)) return undefined
|
||||
|
||||
return pick(data, [ 'listName', 'words' ])
|
||||
}
|
||||
|
||||
protected async importObject (data: SanitizedObject) {
|
||||
const accountId = this.user.Account.id
|
||||
const existing = await WatchedWordsListModel.loadByListName({ listName: data.listName, accountId })
|
||||
|
||||
if (existing) {
|
||||
await existing.updateList({ listName: data.listName, words: data.words })
|
||||
} else {
|
||||
await WatchedWordsListModel.createList({ accountId, listName: data.listName, words: data.words })
|
||||
}
|
||||
|
||||
return { duplicate: false }
|
||||
}
|
||||
}
|
||||
302
server/core/lib/user-import-export/user-exporter.ts
Normal file
302
server/core/lib/user-import-export/user-exporter.ts
Normal file
@@ -0,0 +1,302 @@
|
||||
import { FileStorage, UserExportState } from '@peertube/peertube-models'
|
||||
import { getFileSize } from '@peertube/peertube-node-utils'
|
||||
import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
|
||||
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { MUserDefault, MUserExport } from '@server/types/models/index.js'
|
||||
import archiver, { Archiver } from 'archiver'
|
||||
import { createWriteStream } from 'fs'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { join, parse } from 'path'
|
||||
import { PassThrough, Readable, Writable } from 'stream'
|
||||
import { activityPubCollection } from '../activitypub/collection.js'
|
||||
import { getContextFilter } from '../activitypub/context.js'
|
||||
import { getUserExportFileObjectStorageSize, removeUserExportObjectStorage, storeUserExportFile } from '../object-storage/user-export.js'
|
||||
import { getFSUserExportFilePath } from '../paths.js'
|
||||
import {
|
||||
AbstractUserExporter,
|
||||
AccountExporter,
|
||||
AutoTagPoliciesExporter,
|
||||
BlocklistExporter,
|
||||
ChannelsExporter,
|
||||
CommentsExporter,
|
||||
DislikesExporter,
|
||||
ExportResult,
|
||||
FollowersExporter,
|
||||
FollowingExporter,
|
||||
LikesExporter,
|
||||
UserSettingsExporter,
|
||||
UserVideoHistoryExporter,
|
||||
VideoPlaylistsExporter,
|
||||
VideosExporter,
|
||||
WatchedWordsListsExporter
|
||||
} from './exporters/index.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-export')
|
||||
|
||||
export class UserExporter {
|
||||
|
||||
private archive: Archiver
|
||||
|
||||
async export (exportModel: MUserExport) {
|
||||
try {
|
||||
exportModel.state = UserExportState.PROCESSING
|
||||
await saveInTransactionWithRetries(exportModel)
|
||||
|
||||
const user = await UserModel.loadByIdFull(exportModel.userId)
|
||||
|
||||
let endPromise: Promise<any>
|
||||
let output: Writable
|
||||
|
||||
if (exportModel.storage === FileStorage.FILE_SYSTEM) {
|
||||
output = createWriteStream(getFSUserExportFilePath(exportModel))
|
||||
endPromise = new Promise<string>(res => output.on('close', () => res('')))
|
||||
} else {
|
||||
output = new PassThrough()
|
||||
endPromise = storeUserExportFile(output as PassThrough, exportModel)
|
||||
}
|
||||
|
||||
await this.createZip({ exportModel, user, output })
|
||||
|
||||
const fileUrl = await endPromise
|
||||
|
||||
if (exportModel.storage === FileStorage.OBJECT_STORAGE) {
|
||||
exportModel.fileUrl = fileUrl
|
||||
exportModel.size = await getUserExportFileObjectStorageSize(exportModel)
|
||||
} else if (exportModel.storage === FileStorage.FILE_SYSTEM) {
|
||||
exportModel.size = await getFileSize(getFSUserExportFilePath(exportModel))
|
||||
}
|
||||
|
||||
exportModel.state = UserExportState.COMPLETED
|
||||
|
||||
await saveInTransactionWithRetries(exportModel)
|
||||
} catch (err) {
|
||||
logger.error('Cannot generate an export', { err, ...lTags() })
|
||||
|
||||
try {
|
||||
exportModel.state = UserExportState.ERRORED
|
||||
exportModel.error = err.message
|
||||
|
||||
await saveInTransactionWithRetries(exportModel)
|
||||
} catch (innerErr) {
|
||||
logger.error('Cannot set export error state', { err: innerErr, ...lTags() })
|
||||
}
|
||||
|
||||
try {
|
||||
if (exportModel.storage === FileStorage.FILE_SYSTEM) {
|
||||
await remove(getFSUserExportFilePath(exportModel))
|
||||
} else {
|
||||
await removeUserExportObjectStorage(exportModel)
|
||||
}
|
||||
} catch (innerErr) {
|
||||
logger.error('Cannot remove archive path after failure', { err: innerErr, ...lTags() })
|
||||
}
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
private createZip (options: {
|
||||
exportModel: MUserExport
|
||||
user: MUserDefault
|
||||
output: Writable
|
||||
}) {
|
||||
const { output, exportModel, user } = options
|
||||
|
||||
let activityPubOutboxStore: ExportResult<any>['activityPubOutbox'] = []
|
||||
|
||||
this.archive = archiver('zip', {
|
||||
zlib: {
|
||||
level: 9
|
||||
}
|
||||
})
|
||||
|
||||
return new Promise<void>(async (res, rej) => {
|
||||
this.archive.on('warning', err => {
|
||||
logger.warn('Warning to archive a file in ' + exportModel.filename, { ...lTags(), err })
|
||||
})
|
||||
|
||||
this.archive.on('error', err => {
|
||||
rej(err)
|
||||
})
|
||||
|
||||
this.archive.pipe(output)
|
||||
|
||||
try {
|
||||
for (const { exporter, jsonFilename } of this.buildExporters(exportModel, user)) {
|
||||
const { json, staticFiles, activityPub, activityPubOutbox } = await exporter.export()
|
||||
|
||||
logger.debug(`Adding JSON file ${jsonFilename} in archive ${exportModel.filename}`, lTags())
|
||||
this.appendJSON(json, join('peertube', jsonFilename))
|
||||
|
||||
if (activityPub) {
|
||||
const activityPubFilename = exporter.getActivityPubFilename()
|
||||
if (!activityPubFilename) throw new Error('ActivityPub filename is required for exporter that export activity pub data')
|
||||
|
||||
this.appendJSON(activityPub, join('activity-pub', activityPubFilename))
|
||||
}
|
||||
|
||||
if (activityPubOutbox) {
|
||||
activityPubOutboxStore = activityPubOutboxStore.concat(activityPubOutbox)
|
||||
}
|
||||
|
||||
for (const file of staticFiles) {
|
||||
const archivePath = join('files', parse(jsonFilename).name, file.archivePath)
|
||||
|
||||
logger.debug(`Adding static file ${archivePath} in archive`, lTags())
|
||||
|
||||
try {
|
||||
await this.addToArchiveAndWait(await file.readStreamFactory(), archivePath)
|
||||
} catch (err) {
|
||||
logger.error(`Cannot add ${archivePath} in archive`, { err, ...lTags() })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.appendJSON(
|
||||
await activityPubContextify(activityPubCollection('outbox.json', activityPubOutboxStore), 'Video', getContextFilter()),
|
||||
join('activity-pub', 'outbox.json')
|
||||
)
|
||||
|
||||
await this.archive.finalize()
|
||||
|
||||
res()
|
||||
} catch (err) {
|
||||
this.archive.abort()
|
||||
|
||||
rej(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private buildExporters (exportModel: MUserExport, user: MUserDefault) {
|
||||
const options = {
|
||||
user,
|
||||
activityPubFilenames: {
|
||||
dislikes: 'dislikes.json',
|
||||
likes: 'likes.json',
|
||||
outbox: 'outbox.json',
|
||||
following: 'following.json',
|
||||
account: 'actor.json'
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
jsonFilename: 'videos.json',
|
||||
|
||||
exporter: new VideosExporter({
|
||||
...options,
|
||||
|
||||
relativeStaticDirPath: '../files/videos',
|
||||
withVideoFiles: exportModel.withVideoFiles
|
||||
})
|
||||
},
|
||||
{
|
||||
jsonFilename: 'channels.json',
|
||||
exporter: new ChannelsExporter({
|
||||
...options,
|
||||
|
||||
relativeStaticDirPath: '../files/channels'
|
||||
})
|
||||
},
|
||||
{
|
||||
jsonFilename: 'account.json',
|
||||
exporter: new AccountExporter({
|
||||
...options,
|
||||
|
||||
relativeStaticDirPath: '../files/account'
|
||||
})
|
||||
},
|
||||
{
|
||||
jsonFilename: 'blocklist.json',
|
||||
exporter: new BlocklistExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'likes.json',
|
||||
exporter: new LikesExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'dislikes.json',
|
||||
exporter: new DislikesExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'follower.json',
|
||||
exporter: new FollowersExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'following.json',
|
||||
exporter: new FollowingExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'user-settings.json',
|
||||
exporter: new UserSettingsExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'comments.json',
|
||||
exporter: new CommentsExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'video-playlists.json',
|
||||
exporter: new VideoPlaylistsExporter({
|
||||
...options,
|
||||
|
||||
relativeStaticDirPath: '../files/video-playlists'
|
||||
})
|
||||
},
|
||||
{
|
||||
jsonFilename: 'video-history.json',
|
||||
exporter: new UserVideoHistoryExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'watched-words-lists.json',
|
||||
exporter: new WatchedWordsListsExporter(options)
|
||||
},
|
||||
{
|
||||
jsonFilename: 'automatic-tag-policies.json',
|
||||
exporter: new AutoTagPoliciesExporter(options)
|
||||
}
|
||||
] as { jsonFilename: string, exporter: AbstractUserExporter<any> }[]
|
||||
}
|
||||
|
||||
private addToArchiveAndWait (stream: Readable, archivePath: string) {
|
||||
let errored = false
|
||||
|
||||
return new Promise<void>((res, rej) => {
|
||||
const self = this
|
||||
|
||||
function cleanup () {
|
||||
self.archive.off('entry', entryListener)
|
||||
}
|
||||
|
||||
function entryListener ({ name }) {
|
||||
if (name !== archivePath) return
|
||||
|
||||
cleanup()
|
||||
|
||||
return res()
|
||||
}
|
||||
|
||||
stream.once('error', err => {
|
||||
cleanup()
|
||||
|
||||
errored = true
|
||||
return rej(err)
|
||||
})
|
||||
|
||||
this.archive.on('entry', entryListener)
|
||||
|
||||
// Prevent sending a stream that has an error on open resulting in a stucked archiving process
|
||||
stream.once('readable', () => {
|
||||
if (errored) return
|
||||
|
||||
this.archive.append(stream, { name: archivePath })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private appendJSON (json: any, name: string) {
|
||||
this.archive.append(JSON.stringify(json, undefined, 2), { name })
|
||||
}
|
||||
}
|
||||
171
server/core/lib/user-import-export/user-importer.ts
Normal file
171
server/core/lib/user-import-export/user-importer.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
import { UserImportResultSummary, UserImportState } from '@peertube/peertube-models'
|
||||
import { getFilenameWithoutExt, getFileSize } from '@peertube/peertube-node-utils'
|
||||
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { unzip } from '@server/helpers/unzip.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { MUserDefault, MUserImport } from '@server/types/models/index.js'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { dirname, join } from 'path'
|
||||
import { getFSUserImportFilePath } from '../paths.js'
|
||||
import { BlocklistImporter } from './importers/account-blocklist-importer.js'
|
||||
import { AccountImporter } from './importers/account-importer.js'
|
||||
import { ChannelsImporter } from './importers/channels-importer.js'
|
||||
import { DislikesImporter } from './importers/dislikes-importer.js'
|
||||
import { FollowingImporter } from './importers/following-importer.js'
|
||||
import { LikesImporter } from './importers/likes-importer.js'
|
||||
import { ReviewCommentsTagPoliciesImporter } from './importers/review-comments-tag-policies-importer.js'
|
||||
import { UserSettingsImporter } from './importers/user-settings-importer.js'
|
||||
import { UserVideoHistoryImporter } from './importers/user-video-history-importer.js'
|
||||
import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js'
|
||||
import { VideosImporter } from './importers/videos-importer.js'
|
||||
import { WatchedWordsListsImporter } from './importers/watched-words-lists-importer.js'
|
||||
import { parseBytes } from '@server/helpers/core-utils.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
export class UserImporter {
|
||||
private extractedDirectory: string
|
||||
|
||||
async import (importModel: MUserImport) {
|
||||
const resultSummary: UserImportResultSummary = {
|
||||
stats: {
|
||||
blocklist: this.buildSummary(),
|
||||
channels: this.buildSummary(),
|
||||
likes: this.buildSummary(),
|
||||
dislikes: this.buildSummary(),
|
||||
following: this.buildSummary(),
|
||||
videoPlaylists: this.buildSummary(),
|
||||
videos: this.buildSummary(),
|
||||
account: this.buildSummary(),
|
||||
userSettings: this.buildSummary(),
|
||||
userVideoHistory: this.buildSummary(),
|
||||
watchedWordsLists: this.buildSummary(),
|
||||
commentAutoTagPolicies: this.buildSummary()
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
importModel.state = UserImportState.PROCESSING
|
||||
await saveInTransactionWithRetries(importModel)
|
||||
|
||||
const inputZip = getFSUserImportFilePath(importModel)
|
||||
this.extractedDirectory = join(dirname(inputZip), getFilenameWithoutExt(inputZip))
|
||||
|
||||
await unzip({
|
||||
source: inputZip,
|
||||
destination: this.extractedDirectory,
|
||||
// Videos that take a lot of space don't have a good compression ratio
|
||||
// Keep a minimum of 1GB if the archive doesn't contain video files
|
||||
maxSize: Math.max(await getFileSize(inputZip) * 2, parseBytes('1GB')),
|
||||
maxFiles: 10000
|
||||
})
|
||||
|
||||
const user = await UserModel.loadByIdFull(importModel.userId)
|
||||
|
||||
for (const { name, importer } of this.buildImporters(user)) {
|
||||
try {
|
||||
const { duplicates, errors, success } = await importer.import()
|
||||
|
||||
resultSummary.stats[name].duplicates += duplicates
|
||||
resultSummary.stats[name].errors += errors
|
||||
resultSummary.stats[name].success += success
|
||||
} catch (err) {
|
||||
logger.error(`Cannot import ${importer.getJSONFilePath()} from ${inputZip}`, { err, ...lTags() })
|
||||
|
||||
resultSummary.stats[name].errors++
|
||||
}
|
||||
}
|
||||
|
||||
importModel.state = UserImportState.COMPLETED
|
||||
importModel.resultSummary = resultSummary
|
||||
await saveInTransactionWithRetries(importModel)
|
||||
} catch (err) {
|
||||
logger.error('Cannot import user archive', { err, ...lTags() })
|
||||
|
||||
try {
|
||||
importModel.state = UserImportState.ERRORED
|
||||
importModel.error = err.message
|
||||
|
||||
await saveInTransactionWithRetries(importModel)
|
||||
} catch (innerErr) {
|
||||
logger.error('Cannot set import error state', { err: innerErr, ...lTags() })
|
||||
}
|
||||
|
||||
throw err
|
||||
} finally {
|
||||
try {
|
||||
await remove(getFSUserImportFilePath(importModel))
|
||||
await remove(this.extractedDirectory)
|
||||
} catch (innerErr) {
|
||||
logger.error('Cannot remove import archive and directory after failure', { err: innerErr, ...lTags() })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private buildImporters (user: MUserDefault) {
|
||||
// Keep consistency in import order (don't import videos before channels for example)
|
||||
return [
|
||||
{
|
||||
name: 'account' as const,
|
||||
importer: new AccountImporter(this.buildImporterOptions(user, 'account.json'))
|
||||
},
|
||||
{
|
||||
name: 'userSettings' as const,
|
||||
importer: new UserSettingsImporter(this.buildImporterOptions(user, 'user-settings.json'))
|
||||
},
|
||||
{
|
||||
name: 'channels' as const,
|
||||
importer: new ChannelsImporter(this.buildImporterOptions(user, 'channels.json'))
|
||||
},
|
||||
{
|
||||
name: 'blocklist' as const,
|
||||
importer: new BlocklistImporter(this.buildImporterOptions(user, 'blocklist.json'))
|
||||
},
|
||||
{
|
||||
name: 'following' as const,
|
||||
importer: new FollowingImporter(this.buildImporterOptions(user, 'following.json'))
|
||||
},
|
||||
{
|
||||
name: 'videos' as const,
|
||||
importer: new VideosImporter(this.buildImporterOptions(user, 'videos.json'))
|
||||
},
|
||||
{
|
||||
name: 'likes' as const,
|
||||
importer: new LikesImporter(this.buildImporterOptions(user, 'likes.json'))
|
||||
},
|
||||
{
|
||||
name: 'dislikes' as const,
|
||||
importer: new DislikesImporter(this.buildImporterOptions(user, 'dislikes.json'))
|
||||
},
|
||||
{
|
||||
name: 'videoPlaylists' as const,
|
||||
importer: new VideoPlaylistsImporter(this.buildImporterOptions(user, 'video-playlists.json'))
|
||||
},
|
||||
{
|
||||
name: 'userVideoHistory' as const,
|
||||
importer: new UserVideoHistoryImporter(this.buildImporterOptions(user, 'video-history.json'))
|
||||
},
|
||||
{
|
||||
name: 'watchedWordsLists' as const,
|
||||
importer: new WatchedWordsListsImporter(this.buildImporterOptions(user, 'watched-words-lists.json'))
|
||||
},
|
||||
{
|
||||
name: 'commentAutoTagPolicies' as const,
|
||||
importer: new ReviewCommentsTagPoliciesImporter(this.buildImporterOptions(user, 'automatic-tag-policies.json'))
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
private buildImporterOptions (user: MUserDefault, jsonFilename: string) {
|
||||
return {
|
||||
extractedDirectory: this.extractedDirectory,
|
||||
user,
|
||||
jsonFilePath: join(this.extractedDirectory, 'peertube', jsonFilename)
|
||||
}
|
||||
}
|
||||
|
||||
private buildSummary () {
|
||||
return { success: 0, duplicates: 0, errors: 0 }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user