From 83f328de8a1536c9fbae0605e97ec4af51bd84a4 Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Sun, 12 Nov 2023 15:07:32 +0100 Subject: [PATCH] add: Importing of Posts - Supports Instagram, Mastodon/Pleroma/Akkoma, Twitter and *key --- locales/en-US.yml | 1 + locales/index.d.ts | 1 + locales/ja-JP.yml | 1 + .../backend/src/core/NoteCreateService.ts | 268 ++++++++++ packages/backend/src/core/QueueService.ts | 44 +- packages/backend/src/core/RoleService.ts | 3 + .../backend/src/queue/QueueProcessorModule.ts | 2 + .../src/queue/QueueProcessorService.ts | 8 + .../processors/ImportNotesProcessorService.ts | 490 ++++++++++++++++++ packages/backend/src/queue/types.ts | 17 + .../backend/src/server/api/EndpointsModule.ts | 4 + packages/backend/src/server/api/endpoints.ts | 2 + .../server/api/endpoints/i/import-notes.ts | 72 +++ packages/frontend/src/const.ts | 1 + .../frontend/src/pages/admin/roles.editor.vue | 20 + packages/frontend/src/pages/admin/roles.vue | 8 + .../src/pages/settings/import-export.vue | 34 +- .../megalodon/src/misskey/entities/meta.ts | 1 + 18 files changed, 971 insertions(+), 6 deletions(-) create mode 100644 packages/backend/src/queue/processors/ImportNotesProcessorService.ts create mode 100644 packages/backend/src/server/api/endpoints/i/import-notes.ts diff --git a/locales/en-US.yml b/locales/en-US.yml index 2726641a6..1c069e55d 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1594,6 +1594,7 @@ _role: gtlAvailable: "Can view the global timeline" ltlAvailable: "Can view the local timeline" canPublicNote: "Can send public notes" + canImportNotes: "Can import notes" canInvite: "Can create instance invite codes" inviteLimit: "Invite limit" inviteLimitCycle: "Invite limit cooldown" diff --git a/locales/index.d.ts b/locales/index.d.ts index dfdf3119b..6cacf1ab3 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1696,6 +1696,7 @@ export interface Locale { "gtlAvailable": string; "ltlAvailable": string; "canPublicNote": string; + "canImportNotes": string; "canInvite": string; "inviteLimit": string; "inviteLimitCycle": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index fb2a64b39..039937e40 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1605,6 +1605,7 @@ _role: gtlAvailable: "グローバルタイムラインの閲覧" ltlAvailable: "ローカルタイムラインの閲覧" canPublicNote: "パブリック投稿の許可" + canImportNotes: "ノートのインポートが可能" canInvite: "サーバー招待コードの発行" inviteLimit: "招待コードの作成可能数" inviteLimitCycle: "招待コードの発行間隔" diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 3e7bba9e1..9398b5c2c 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -378,6 +378,167 @@ export class NoteCreateService implements OnApplicationShutdown { return note; } + @bindThis + public async import(user: { + id: MiUser['id']; + username: MiUser['username']; + host: MiUser['host']; + isBot: MiUser['isBot']; + isIndexable: MiUser['isIndexable']; + }, data: Option, silent = false): Promise { + // チャンネル外にリプライしたら対象のスコープに合わせる + // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) + if (data.reply && data.channel && data.reply.channelId !== data.channel.id) { + if (data.reply.channelId) { + data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); + } else { + data.channel = null; + } + } + + // チャンネル内にリプライしたら対象のスコープに合わせる + // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) + if (data.reply && (data.channel == null) && data.reply.channelId) { + data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); + } + + if (data.createdAt == null) data.createdAt = new Date(); + if (data.visibility == null) data.visibility = 'public'; + if (data.localOnly == null) data.localOnly = false; + if (data.channel != null) data.visibility = 'public'; + if (data.channel != null) data.visibleUsers = []; + if (data.channel != null) data.localOnly = true; + + const meta = await this.metaService.fetch(); + + if (data.visibility === 'public' && data.channel == null) { + const sensitiveWords = meta.sensitiveWords; + if (this.isSensitive(data, sensitiveWords)) { + data.visibility = 'home'; + } else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) { + data.visibility = 'home'; + } + } + + const inSilencedInstance = this.utilityService.isSilencedHost(meta.silencedHosts, user.host); + + if (data.visibility === 'public' && inSilencedInstance && user.host !== null) { + data.visibility = 'home'; + } + + if (data.renote) { + switch (data.renote.visibility) { + case 'public': + // public noteは無条件にrenote可能 + break; + case 'home': + // home noteはhome以下にrenote可能 + if (data.visibility === 'public') { + data.visibility = 'home'; + } + break; + case 'followers': + // 他人のfollowers noteはreject + if (data.renote.userId !== user.id) { + throw new Error('Renote target is not public or home'); + } + + // Renote対象がfollowersならfollowersにする + data.visibility = 'followers'; + break; + case 'specified': + // specified / direct noteはreject + throw new Error('Renote target is not public or home'); + } + } + + // Check blocking + if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) { + if (data.renote.userHost === null) { + if (data.renote.userId !== user.id) { + const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id); + if (blocked) { + throw new Error('blocked'); + } + } + } + } + + // 返信対象がpublicではないならhomeにする + if (data.reply && data.reply.visibility !== 'public' && data.visibility === 'public') { + data.visibility = 'home'; + } + + // ローカルのみをRenoteしたらローカルのみにする + if (data.renote && data.renote.localOnly && data.channel == null) { + data.localOnly = true; + } + + // ローカルのみにリプライしたらローカルのみにする + if (data.reply && data.reply.localOnly && data.channel == null) { + data.localOnly = true; + } + + if (data.text) { + if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) { + data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH); + } + data.text = data.text.trim(); + } else { + data.text = null; + } + + let tags = data.apHashtags; + let emojis = data.apEmojis; + let mentionedUsers = data.apMentions; + + // Parse MFM if needed + if (!tags || !emojis || !mentionedUsers) { + const tokens = (data.text ? mfm.parse(data.text)! : []); + const cwTokens = data.cw ? mfm.parse(data.cw)! : []; + const choiceTokens = data.poll && data.poll.choices + ? concat(data.poll.choices.map(choice => mfm.parse(choice)!)) + : []; + + const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens); + + tags = data.apHashtags ?? extractHashtags(combinedTokens); + + emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens); + + mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens); + } + + tags = tags.filter(tag => Array.from(tag).length <= 128).splice(0, 32); + + if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) { + mentionedUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId })); + } + + if (data.visibility === 'specified') { + if (data.visibleUsers == null) throw new Error('invalid param'); + + for (const u of data.visibleUsers) { + if (!mentionedUsers.some(x => x.id === u.id)) { + mentionedUsers.push(u); + } + } + + if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) { + data.visibleUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId })); + } + } + + const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); + + setImmediate('post created', { signal: this.#shutdownController.signal }).then( + () => this.postNoteImported(note, user, data, silent, tags!, mentionedUsers!), + () => { /* aborted, ignore this */ }, + ); + + return note; + } + @bindThis private async insertNote(user: { id: MiUser['id']; host: MiUser['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) { const insert = new MiNote({ @@ -715,6 +876,113 @@ export class NoteCreateService implements OnApplicationShutdown { if (user.isIndexable) this.index(note); } + @bindThis + private async postNoteImported(note: MiNote, user: { + id: MiUser['id']; + username: MiUser['username']; + host: MiUser['host']; + isBot: MiUser['isBot']; + isIndexable: MiUser['isIndexable']; + }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + const meta = await this.metaService.fetch(); + + this.notesChart.update(note, true); + if (meta.enableChartsForRemoteUser || (user.host == null)) { + this.perUserNotesChart.update(user, note, true); + } + + // Register host + if (this.userEntityService.isRemoteUser(user)) { + this.federatedInstanceService.fetch(user.host).then(async i => { + if (note.renote && note.text) { + this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); + } else if (!note.renote) { + this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); + } + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateNote(i.host, note, true); + } + }); + } + + if (data.renote && data.text) { + // Increment notes count (user) + this.incNotesCountOfUser(user); + } else if (!data.renote) { + // Increment notes count (user) + this.incNotesCountOfUser(user); + } + + this.pushToTl(note, user); + + this.antennaService.addNoteToAntennas(note, user); + + if (data.reply) { + this.saveReply(data.reply, note); + } + + if (data.reply == null) { + // TODO: キャッシュ + this.followingsRepository.findBy({ + followeeId: user.id, + notify: 'normal', + }).then(followings => { + for (const following of followings) { + // TODO: ワードミュート考慮 + this.notificationService.createNotification(following.followerId, 'note', { + noteId: note.id, + }, user.id); + } + }); + } + + if (data.renote && data.text == null && data.renote.userId !== user.id && !user.isBot) { + this.incRenoteCount(data.renote); + } + + if (data.poll && data.poll.expiresAt) { + const delay = data.poll.expiresAt.getTime() - Date.now(); + this.queueService.endedPollNotificationQueue.add(note.id, { + noteId: note.id, + }, { + delay, + removeOnComplete: true, + }); + } + + if (!silent) { + if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user); + + // Pack the note + const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true }); + + this.globalEventService.publishNotesStream(noteObj); + + this.roleService.addNoteToRoleTimeline(noteObj); + } + + if (data.channel) { + this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + this.channelsRepository.update(data.channel.id, { + lastNotedAt: new Date(), + }); + + this.notesRepository.countBy({ + userId: user.id, + channelId: data.channel.id, + }).then(count => { + // この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる + // TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい + if (count === 1) { + this.channelsRepository.increment({ id: data.channel!.id }, 'usersCount', 1); + } + }); + } + + // Register to search database + if (user.isIndexable) this.index(note); + } + @bindThis private isSensitive(note: Option, sensitiveWord: string[]): boolean { if (sensitiveWord.length > 0) { diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index c5830168b..ed24cfa56 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -258,6 +258,48 @@ export class QueueService { }); } + @bindThis + public createImportNotesJob(user: ThinUser, fileId: MiDriveFile['id'], type: string | null | undefined) { + return this.dbQueue.add('importNotes', { + user: { id: user.id }, + fileId: fileId, + type: type, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + + @bindThis + public createImportTweetsToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportMastoToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportPleroToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportIGToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importIGToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + @bindThis public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) { const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies })); @@ -293,7 +335,7 @@ export class QueueService { } @bindThis - private generateToDbJobData>(name: T, data: D): { + private generateToDbJobData>(name: T, data: D): { name: string, data: D, opts: Bull.JobsOptions, diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index d6a414694..4c5f88335 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -47,6 +47,7 @@ export type RolePolicies = { userListLimit: number; userEachUserListsLimit: number; rateLimitFactor: number; + canImportNotes: boolean; }; export const DEFAULT_POLICIES: RolePolicies = { @@ -73,6 +74,7 @@ export const DEFAULT_POLICIES: RolePolicies = { userListLimit: 10, userEachUserListsLimit: 50, rateLimitFactor: 1, + canImportNotes: true, }; @Injectable() @@ -323,6 +325,7 @@ export class RoleService implements OnApplicationShutdown { userListLimit: calc('userListLimit', vs => Math.max(...vs)), userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)), rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)), + canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)), }; } diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index 5c61eb9e9..29dc78605 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -29,6 +29,7 @@ import { ExportUserListsProcessorService } from './processors/ExportUserListsPro import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js'; import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js'; import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js'; +import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js'; import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js'; import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; @@ -61,6 +62,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor ExportBlockingProcessorService, ExportUserListsProcessorService, ExportAntennasProcessorService, + ImportNotesProcessorService, ImportFollowingProcessorService, ImportMutingProcessorService, ImportBlockingProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 7e45509fb..d0e1a46a1 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -41,6 +41,7 @@ import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; import { QUEUE, baseQueueOptions } from './const.js'; +import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 function httpRelatedBackoff(attemptsMade: number) { @@ -100,6 +101,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private exportUserListsProcessorService: ExportUserListsProcessorService, private exportAntennasProcessorService: ExportAntennasProcessorService, private importFollowingProcessorService: ImportFollowingProcessorService, + private importNotesProcessorService: ImportNotesProcessorService, private importMutingProcessorService: ImportMutingProcessorService, private importBlockingProcessorService: ImportBlockingProcessorService, private importUserListsProcessorService: ImportUserListsProcessorService, @@ -174,6 +176,12 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'exportUserLists': return this.exportUserListsProcessorService.process(job); case 'exportAntennas': return this.exportAntennasProcessorService.process(job); case 'importFollowing': return this.importFollowingProcessorService.process(job); + case 'importNotes': return this.importNotesProcessorService.process(job); + case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job); + case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job); + case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job); + case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job); + case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job); case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); case 'importMuting': return this.importMutingProcessorService.process(job); case 'importBlocking': return this.importBlockingProcessorService.process(job); diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts new file mode 100644 index 000000000..b8e68ae19 --- /dev/null +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -0,0 +1,490 @@ +import * as fs from 'node:fs'; +import * as vm from 'node:vm'; +import { Inject, Injectable } from '@nestjs/common'; +import { IsNull } from 'typeorm'; +import { ZipReader } from 'slacc'; +import { DI } from '@/di-symbols.js'; +import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote } from '@/models/_.js'; +import type Logger from '@/logger.js'; +import { DownloadService } from '@/core/DownloadService.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; +import { createTemp, createTempDir } from '@/misc/create-temp.js'; +import { NoteCreateService } from '@/core/NoteCreateService.js'; +import { DriveService } from '@/core/DriveService.js'; +import { MfmService } from '@/core/MfmService.js'; +import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; +import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbNoteImportToDbJobData, DbNoteImportJobData } from '../types.js'; + +@Injectable() +export class ImportNotesProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + + private queueService: QueueService, + private utilityService: UtilityService, + private noteCreateService: NoteCreateService, + private mfmService: MfmService, + private apNoteService: ApNoteService, + private driveService: DriveService, + private downloadService: DownloadService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('import-notes'); + } + @bindThis + private async _keepTweet(tweet: any) { + if (!tweet.created_at.endsWith(new Date().getFullYear())) { + return false; + } + + return !tweet.full_text.startsWith('@'); + } + + @bindThis + private async uploadFiles(dir: any, user: any) { + const fileList = fs.readdirSync(dir); + for (const file of fileList) { + const name = `${dir}/${file}`; + if (fs.statSync(name).isDirectory()) { + await this.uploadFiles(name, user); + } else { + const exists = await this.driveFilesRepository.findOneBy({ name: file, userId: user.id }); + + if (file.endsWith('.srt')) return; + + if (!exists) { + await this.driveService.addFile({ + user: user, + path: name, + name: file, + }); + } + } + } + } + + @bindThis + private isIterable(obj: any) { + if (obj == null) { + return false; + } + return typeof obj[Symbol.iterator] === 'function'; + } + + @bindThis + public async process(job: Bull.Job): Promise { + this.logger.info(`Importing following of ${job.data.user.id} ...`); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const file = await this.driveFilesRepository.findOneBy({ + id: job.data.fileId, + }); + if (file == null) { + return; + } + + if (job.data.type === 'Twitter' || file.name.startsWith('twitter') && file.name.endsWith('.zip')) { + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + const destPath = path + '/twitter.zip'; + + try { + fs.writeFileSync(destPath, '', 'binary'); + await this.downloadService.downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const outputPath = path + '/twitter'; + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); + const fakeWindow: any = { + window: { + YTD: { + tweets: { + part0: {}, + }, + }, + }, + }; + const script = new vm.Script(fs.readFileSync(outputPath + '/data/tweets.js', 'utf-8')); + const context = vm.createContext(fakeWindow); + script.runInContext(context); + const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => { + return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet); + }, []).filter(this._keepTweet); + this.queueService.createImportTweetsToDbJob({ id: user.id }, tweets); + } finally { + cleanup(); + } + } else if (file.name.endsWith('.zip')) { + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + const destPath = path + '/unknown.zip'; + + try { + fs.writeFileSync(destPath, '', 'binary'); + await this.downloadService.downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const outputPath = path + '/unknown'; + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); + const isInstagram = job.data.type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses'); + const isOutbox = job.data.type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json'); + if (isInstagram) { + const postsJson = fs.readFileSync(outputPath + '/content/posts_1.json', 'utf-8'); + const posts = JSON.parse(postsJson); + await this.uploadFiles(outputPath + '/media/posts', user); + this.queueService.createImportIGToDbJob({ id: user.id }, posts); + } else if (isOutbox) { + const actorJson = fs.readFileSync(outputPath + '/actor.json', 'utf-8'); + const actor = JSON.parse(actorJson); + const isPleroma = actor['@context'].some((v: any) => typeof v === 'string' && v.match(/litepub(.*)/)); + if (isPleroma) { + const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8'); + const outbox = JSON.parse(outboxJson); + this.queueService.createImportPleroToDbJob({ id: user.id }, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note')); + } else { + const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8'); + const outbox = JSON.parse(outboxJson); + if (fs.existsSync(outputPath + '/media_attachments/files')) await this.uploadFiles(outputPath + '/media_attachments/files', user); + this.queueService.createImportMastoToDbJob({ id: user.id }, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note')); + } + } + } finally { + cleanup(); + } + } else if (job.data.type === 'Misskey' || file.name.startsWith('notes-') && file.name.endsWith('.json')) { + const [path, cleanup] = await createTemp(); + + this.logger.info(`Temp dir is ${path}`); + + try { + fs.writeFileSync(path, '', 'utf-8'); + await this.downloadService.downloadUrl(file.url, path); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const notesJson = fs.readFileSync(path, 'utf-8'); + const notes = JSON.parse(notesJson); + this.queueService.createImportKeyNotesToDbJob({ id: user.id }, notes); + cleanup(); + } + + this.logger.succ('Import jobs created'); + } + + @bindThis + public async processKeyNotesToDb(job: Bull.Job): Promise { + const note = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const files: MiDriveFile[] = []; + const date = new Date(note.createdAt); + + if (note.files && this.isIterable(note.files)) { + for await (const file of note.files) { + const [filePath, cleanup] = await createTemp(); + const slashdex = file.url.lastIndexOf('/'); + const name = file.url.substring(slashdex + 1); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + + if (!exists) { + try { + await this.downloadService.downloadUrl(file.url, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + }); + files.push(driveFile); + } else { + files.push(exists); + } + + cleanup(); + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw }); + if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes); + } + + @bindThis + public async processMastoToDb(job: Bull.Job): Promise { + const toot = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const date = new Date(toot.object.published); + let text = undefined; + const files: MiDriveFile[] = []; + let reply: MiNote | null = null; + + if (toot.object.inReplyTo != null) { + try { + reply = await this.apNoteService.resolveNote(toot.object.inReplyTo); + } catch (error) { + reply = null; + } + } + + if (toot.directMessage) return; + + const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null); + + try { + text = await this.mfmService.fromHtml(toot.object.content, hashtags); + } catch (error) { + text = undefined; + } + + if (toot.object.attachment && this.isIterable(toot.object.attachment)) { + for await (const file of toot.object.attachment) { + const slashdex = file.url.lastIndexOf('/'); + const name = file.url.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + if (exists) { + files.push(exists); + } + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: toot.object.sensitive ? toot.object.summary : null, reply: reply }); + } + + @bindThis + public async processPleroToDb(job: Bull.Job): Promise { + const post = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const date = new Date(post.object.published); + let text = undefined; + const files: MiDriveFile[] = []; + let reply: MiNote | null = null; + + if (post.object.inReplyTo != null) { + try { + reply = await this.apNoteService.resolveNote(post.object.inReplyTo); + } catch (error) { + reply = null; + } + } + + if (post.directMessage) return; + + const hashtags = extractApHashtagObjects(post.object.tag).map((x) => x.name).filter((x): x is string => x != null); + + try { + text = await this.mfmService.fromHtml(post.object.content, hashtags); + } catch (error) { + text = undefined; + } + + if (post.object.attachment && this.isIterable(post.object.attachment)) { + for await (const file of post.object.attachment) { + const slashdex = file.url.lastIndexOf('/'); + const name = file.url.substring(slashdex + 1); + const [filePath, cleanup] = await createTemp(); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + + if (!exists) { + try { + await this.downloadService.downloadUrl(file.url, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + }); + files.push(driveFile); + } else { + files.push(exists); + } + + cleanup(); + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: post.object.sensitive ? post.object.summary : null, reply: reply }); + } + + @bindThis + public async processIGDb(job: Bull.Job): Promise { + const post = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + let date; + let title; + const files: MiDriveFile[] = []; + + if (post.media && this.isIterable(post.media) && post.media.length > 1) { + date = new Date(post.creation_timestamp * 1000); + title = post.title; + for await (const file of post.media) { + const slashdex = file.uri.lastIndexOf('/'); + const name = file.uri.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id }); + if (exists) { + files.push(exists); + } + } + } else if (post.media && this.isIterable(post.media) && !(post.media.length > 1)) { + date = new Date(post.media[0].creation_timestamp * 1000); + title = post.media[0].title; + const slashdex = post.media[0].uri.lastIndexOf('/'); + const name = post.media[0].uri.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id }); + if (exists) { + files.push(exists); + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: title, files: files }); + } + + @bindThis + public async processTwitterDb(job: Bull.Job): Promise { + const tweet = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + if (tweet.in_reply_to_status_id_str) return; + + async function replaceTwitterUrls(full_text: string, urls: any) { + let full_textedit = full_text; + urls.forEach((url: any) => { + full_textedit = full_textedit.replaceAll(url.url, url.expanded_url); + }); + return full_textedit; + } + + async function replaceTwitterMentions(full_text: string, mentions: any) { + let full_textedit = full_text; + mentions.forEach((mention: any) => { + full_textedit = full_textedit.replaceAll(`@${mention.screen_name}`, `[@${mention.screen_name}](https://nitter.net/${mention.screen_name})`); + }); + return full_textedit; + } + + try { + const date = new Date(tweet.created_at); + const textReplaceURLs = tweet.entities.urls && tweet.entities.urls.length > 0 ? await replaceTwitterUrls(tweet.full_text, tweet.entities.urls) : tweet.full_text; + const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs; + const files: MiDriveFile[] = []; + + if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) { + for await (const file of tweet.extended_entities.media) { + if (file.video_info) { + const [filePath, cleanup] = await createTemp(); + const slashdex = file.video_info.variants[0].url.lastIndexOf('/'); + const name = file.video_info.variants[0].url.substring(slashdex + 1); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + + const videos = file.video_info.variants.filter((x: any) => x.content_type === 'video/mp4'); + + if (!exists) { + try { + await this.downloadService.downloadUrl(videos[0].url, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + }); + files.push(driveFile); + } else { + files.push(exists); + } + + cleanup(); + } else if (file.media_url_https) { + const [filePath, cleanup] = await createTemp(); + const slashdex = file.media_url_https.lastIndexOf('/'); + const name = file.media_url_https.substring(slashdex + 1); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + + if (!exists) { + try { + await this.downloadService.downloadUrl(file.media_url_https, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + }); + files.push(driveFile); + } else { + files.push(exists); + } + cleanup(); + } + } + } + await this.noteCreateService.import(user, { createdAt: date, text: text, files: files }); + } catch (e) { + this.logger.warn(`Error: ${e}`); + } + } +} diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 94a95d8b9..8da71492d 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -49,6 +49,12 @@ export type DbJobMap = { exportBlocking: DbJobDataWithUser; exportUserLists: DbJobDataWithUser; importAntennas: DBAntennaImportJobData; + importNotes: DbUserImportJobData; + importTweetsToDb: DbNoteImportToDbJobData; + importIGToDb: DbNoteImportToDbJobData; + importMastoToDb: DbNoteImportToDbJobData; + importPleroToDb: DbNoteImportToDbJobData; + importKeyNotesToDb: DbNoteImportToDbJobData; importFollowing: DbUserImportJobData; importFollowingToDb: DbUserImportToDbJobData; importMuting: DbUserImportJobData; @@ -84,6 +90,12 @@ export type DbUserImportJobData = { withReplies?: boolean; }; +export type DbNoteImportJobData = { + user: ThinUser; + fileId: MiDriveFile['id']; + type?: string; +}; + export type DBAntennaImportJobData = { user: ThinUser, antenna: Antenna @@ -95,6 +107,11 @@ export type DbUserImportToDbJobData = { withReplies?: boolean; }; +export type DbNoteImportToDbJobData = { + user: ThinUser; + target: any; +}; + export type ObjectStorageJobData = ObjectStorageFileJobData | Record; export type ObjectStorageFileJobData = { diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 09a8d8c37..77048ec01 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -219,6 +219,7 @@ import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js'; import * as ep___i_importBlocking from './endpoints/i/import-blocking.js'; import * as ep___i_importFollowing from './endpoints/i/import-following.js'; +import * as ep___i_importNotes from './endpoints/i/import-notes.js'; import * as ep___i_importMuting from './endpoints/i/import-muting.js'; import * as ep___i_importUserLists from './endpoints/i/import-user-lists.js'; import * as ep___i_importAntennas from './endpoints/i/import-antennas.js'; @@ -587,6 +588,7 @@ const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep const $i_gallery_posts: Provider = { provide: 'ep:i/gallery/posts', useClass: ep___i_gallery_posts.default }; const $i_importBlocking: Provider = { provide: 'ep:i/import-blocking', useClass: ep___i_importBlocking.default }; const $i_importFollowing: Provider = { provide: 'ep:i/import-following', useClass: ep___i_importFollowing.default }; +const $i_importNotes: Provider = { provide: 'ep:i/import-notes', useClass: ep___i_importNotes.default }; const $i_importMuting: Provider = { provide: 'ep:i/import-muting', useClass: ep___i_importMuting.default }; const $i_importUserLists: Provider = { provide: 'ep:i/import-user-lists', useClass: ep___i_importUserLists.default }; const $i_importAntennas: Provider = { provide: 'ep:i/import-antennas', useClass: ep___i_importAntennas.default }; @@ -959,6 +961,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de $i_gallery_posts, $i_importBlocking, $i_importFollowing, + $i_importNotes, $i_importMuting, $i_importUserLists, $i_importAntennas, @@ -1325,6 +1328,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de $i_gallery_posts, $i_importBlocking, $i_importFollowing, + $i_importNotes, $i_importMuting, $i_importUserLists, $i_importAntennas, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 527235264..18a0bff84 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -219,6 +219,7 @@ import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js'; import * as ep___i_importBlocking from './endpoints/i/import-blocking.js'; import * as ep___i_importFollowing from './endpoints/i/import-following.js'; +import * as ep___i_importNotes from './endpoints/i/import-notes.js'; import * as ep___i_importMuting from './endpoints/i/import-muting.js'; import * as ep___i_importUserLists from './endpoints/i/import-user-lists.js'; import * as ep___i_importAntennas from './endpoints/i/import-antennas.js'; @@ -585,6 +586,7 @@ const eps = [ ['i/gallery/posts', ep___i_gallery_posts], ['i/import-blocking', ep___i_importBlocking], ['i/import-following', ep___i_importFollowing], + ['i/import-notes', ep___i_importNotes], ['i/import-muting', ep___i_importMuting], ['i/import-user-lists', ep___i_importUserLists], ['i/import-antennas', ep___i_importAntennas], diff --git a/packages/backend/src/server/api/endpoints/i/import-notes.ts b/packages/backend/src/server/api/endpoints/i/import-notes.ts new file mode 100644 index 000000000..1e572d3d7 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/import-notes.ts @@ -0,0 +1,72 @@ +import { Inject, Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; +import type { DriveFilesRepository } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '../../error.js'; + +export const meta = { + secure: true, + requireCredential: true, + prohibitMoved: true, + limit: { + duration: ms('1hour'), + max: 1, + }, + + errors: { + noSuchFile: { + message: 'No such file.', + code: 'NO_SUCH_FILE', + id: 'b98644cf-a5ac-4277-a502-0b8054a709a3', + }, + + emptyFile: { + message: 'That file is empty.', + code: 'EMPTY_FILE', + id: '31a1b42c-06f7-42ae-8a38-a661c5c9f691', + }, + + notPermitted: { + message: 'You are not allowed to import notes.', + code: 'NO_PERMISSION', + id: '31a1b42c-06f7-42ae-8a38-a661c5c9f692', + }, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: { + fileId: { type: 'string', format: 'misskey:id' }, + type: { type: 'string', nullable: true }, + }, + required: ['fileId'], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + + private queueService: QueueService, + private roleService: RoleService, + ) { + super(meta, paramDef, async (ps, me) => { + const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + + if (file.size === 0) throw new ApiError(meta.errors.emptyFile); + + if ((await this.roleService.getUserPolicies(me.id)).canImportNotes === false) { + throw new ApiError(meta.errors.notPermitted); + } + + this.queueService.createImportNotesJob(me, file.id, ps.type); + }); + } +} diff --git a/packages/frontend/src/const.ts b/packages/frontend/src/const.ts index dee0461a3..26f1451e9 100644 --- a/packages/frontend/src/const.ts +++ b/packages/frontend/src/const.ts @@ -112,6 +112,7 @@ export const ROLE_POLICIES = [ 'gtlAvailable', 'ltlAvailable', 'canPublicNote', + 'canImportNotes', 'canInvite', 'inviteLimit', 'inviteLimitCycle', diff --git a/packages/frontend/src/pages/admin/roles.editor.vue b/packages/frontend/src/pages/admin/roles.editor.vue index be630c05a..8c656e917 100644 --- a/packages/frontend/src/pages/admin/roles.editor.vue +++ b/packages/frontend/src/pages/admin/roles.editor.vue @@ -160,6 +160,26 @@ SPDX-License-Identifier: AGPL-3.0-only + + + +
+ + + + + + + + + +
+ +