merge: GDPR Data export for users (#143)

feat: GDPR Data export for users

locales/en-US.yml

@@ -2401,3 +2401,9 @@ _animatedMFM:
   _alert:
     text: "Animated MFMs could include flashing lights and fast moving text/emojis."
     confirm: "Animate"
+
+_dataRequest:
+  title: "Request Data"
+  warn: "Data requests are only possible every 3 days."
+  text: "Once the data is ready to download, an email will be sent to the email address registered to this account."
+  button: "Request"

locales/index.d.ts

@@ -2509,6 +2509,12 @@ export interface Locale {
             "confirm": string;
         };
     };
+    "_dataRequest": {
+        "title": string;
+        "warn": string;
+        "text": string;
+        "button": string;
+    };
 }
 declare const locales: {
     [lang: string]: Locale;

locales/ja-JP.yml

@@ -2395,3 +2395,9 @@ _animatedMFM:
   _alert:
     text: "アニメーションMFMには、点滅するライトや高速で動くテキスト／絵文字を含めることができる。"
     confirm: "アニメイト"
+
+_dataRequest:
+  title: "リクエストデータ"
+  warn: "データのリクエストは3日ごとにしかできない。"
+  text: "データのダウンロードが完了すると、このアカウントに登録されているEメールアドレスにEメールが送信されます。"
+  button: "リクエスト"

packages/backend/src/core/QueueService.ts

@@ -164,6 +164,16 @@ export class QueueService {
 		});
 	}
 
+	@bindThis
+	public createExportAccountDataJob(user: ThinUser) {
+		return this.dbQueue.add('exportAccountData', {
+			user: { id: user.id },
+		}, {
+			removeOnComplete: true,
+			removeOnFail: true,
+		});
+	}
+
 	@bindThis
 	public createExportNotesJob(user: ThinUser) {
 		return this.dbQueue.add('exportNotes', {

packages/backend/src/queue/QueueProcessorModule.ts

@@ -19,6 +19,7 @@ import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesP
 import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
 import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
 import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js';
+import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js';
 import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
 import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
 import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
@@ -51,6 +52,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
 		CheckExpiredMutingsProcessorService,
 		CleanProcessorService,
 		DeleteDriveFilesProcessorService,
+		ExportAccountDataProcessorService,
 		ExportCustomEmojisProcessorService,
 		ExportNotesProcessorService,
 		ExportFavoritesProcessorService,

packages/backend/src/queue/QueueProcessorService.ts

@@ -14,6 +14,7 @@ import { EndedPollNotificationProcessorService } from './processors/EndedPollNot
 import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
 import { InboxProcessorService } from './processors/InboxProcessorService.js';
 import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
+import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js';
 import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
 import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
 import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
@@ -89,6 +90,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 		private deliverProcessorService: DeliverProcessorService,
 		private inboxProcessorService: InboxProcessorService,
 		private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
+		private exportAccountDataProcessorService: ExportAccountDataProcessorService,
 		private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
 		private exportNotesProcessorService: ExportNotesProcessorService,
 		private exportFavoritesProcessorService: ExportFavoritesProcessorService,
@@ -162,6 +164,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 		this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
 			switch (job.name) {
 				case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
+				case 'exportAccountData': return this.exportAccountDataProcessorService.process(job);
 				case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
 				case 'exportNotes': return this.exportNotesProcessorService.process(job);
 				case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);

packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts

@@ -0,0 +1,776 @@
+/* eslint-disable no-constant-condition */
+/* eslint-disable @typescript-eslint/no-unnecessary-condition */
+
+import * as fs from 'node:fs';
+import { Inject, Injectable } from '@nestjs/common';
+import { In, IsNull, MoreThan, Not } from 'typeorm';
+import { format as dateFormat } from 'date-fns';
+import mime from 'mime-types';
+import archiver from 'archiver';
+import { DI } from '@/di-symbols.js';
+import type { AntennasRepository, BlockingsRepository, DriveFilesRepository, FollowingsRepository, MiBlocking, MiFollowing, MiMuting, MiNote, MiNoteFavorite, MiPoll, MiUser, MutingsRepository, NoteFavoritesRepository, NotesRepository, PollsRepository, UserListMembershipsRepository, UserListsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
+import type { Config } from '@/config.js';
+import type Logger from '@/logger.js';
+import { DriveService } from '@/core/DriveService.js';
+import { IdService } from '@/core/IdService.js';
+import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
+import { createTemp, createTempDir } from '@/misc/create-temp.js';
+import { bindThis } from '@/decorators.js';
+import { Packed } from '@/misc/json-schema.js';
+import { UtilityService } from '@/core/UtilityService.js';
+import { DownloadService } from '@/core/DownloadService.js';
+import { QueueLoggerService } from '../QueueLoggerService.js';
+import type * as Bull from 'bullmq';
+import { EmailService } from '@/core/EmailService.js';
+
+@Injectable()
+export class ExportAccountDataProcessorService {
+	private logger: Logger;
+
+	constructor(
+		@Inject(DI.config)
+		private config: Config,
+
+		@Inject(DI.usersRepository)
+		private usersRepository: UsersRepository,
+
+		@Inject(DI.userProfilesRepository)
+		private userProfilesRepository: UserProfilesRepository,
+
+		@Inject(DI.notesRepository)
+		private notesRepository: NotesRepository,
+
+		@Inject(DI.noteFavoritesRepository)
+		private noteFavoritesRepository: NoteFavoritesRepository,
+
+		@Inject(DI.pollsRepository)
+		private pollsRepository: PollsRepository,
+
+		@Inject(DI.followingsRepository)
+		private followingsRepository: FollowingsRepository,
+
+		@Inject(DI.mutingsRepository)
+		private mutingsRepository: MutingsRepository,
+
+		@Inject(DI.blockingsRepository)
+		private blockingsRepository: BlockingsRepository,
+
+		@Inject(DI.driveFilesRepository)
+		private driveFilesRepository: DriveFilesRepository,
+
+		@Inject(DI.antennasRepository)
+		private antennasRepository: AntennasRepository,
+
+		@Inject(DI.userListsRepository)
+		private userListsRepository: UserListsRepository,
+
+		@Inject(DI.userListMembershipsRepository)
+		private userListMembershipsRepository: UserListMembershipsRepository,
+
+		private utilityService: UtilityService,
+		private driveService: DriveService,
+		private idService: IdService,
+		private driveFileEntityService: DriveFileEntityService,
+		private downloadService: DownloadService,
+		private emailService: EmailService,
+		private queueLoggerService: QueueLoggerService,
+	) {
+		this.logger = this.queueLoggerService.logger.createSubLogger('export-account-data');
+	}
+
+	@bindThis
+	public async process(job: Bull.Job): Promise<void> {
+		this.logger.info('Exporting Account Data...');
+
+		const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
+		if (user == null) {
+			return;
+		}
+
+		const profile = await this.userProfilesRepository.findOneBy({ userId: job.data.user.id });
+		if (profile == null) {
+			return;
+		}
+
+		const [path, cleanup] = await createTempDir();
+
+		this.logger.info(`Temp dir is ${path}`);
+
+		// User Export
+
+		const userPath = path + '/user.json';
+
+		fs.writeFileSync(userPath, '', 'utf-8');
+
+		const userStream = fs.createWriteStream(userPath, { flags: 'a' });
+
+		const writeUser = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				userStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeUser(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","user":[`);
+
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		const { host, uri, sharedInbox, followersUri, lastFetchedAt, inbox, ...userTrimmed } = user;
+
+		await writeUser(JSON.stringify(userTrimmed));
+
+		await writeUser(']}');
+
+		userStream.end();
+
+		// Profile Export
+
+		const profilePath = path + '/profile.json';
+
+		fs.writeFileSync(profilePath, '', 'utf-8');
+
+		const profileStream = fs.createWriteStream(profilePath, { flags: 'a' });
+
+		const writeProfile = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				profileStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		const { emailVerifyCode, twoFactorBackupSecret, twoFactorSecret, password, twoFactorTempSecret, userHost, ...profileTrimmed } = profile;
+
+		await writeProfile(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","profile":[`);
+
+		await writeProfile(JSON.stringify(profileTrimmed));
+
+		await writeProfile(']}');
+
+		profileStream.end();
+
+		// Note Export
+
+		const notesPath = path + '/notes.json';
+
+		fs.writeFileSync(notesPath, '', 'utf-8');
+
+		const notesStream = fs.createWriteStream(notesPath, { flags: 'a' });
+
+		const writeNotes = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				notesStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeNotes(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","notes":[`);
+
+		let noteCursor: MiNote['id'] | null = null;
+		let exportedNotesCount = 0;
+
+		while (true) {
+			const notes = await this.notesRepository.find({
+				where: {
+					userId: user.id,
+					...(noteCursor ? { id: MoreThan(noteCursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			}) as MiNote[];
+
+			if (notes.length === 0) {
+				break;
+			}
+
+			noteCursor = notes.at(-1)?.id ?? null;
+
+			for (const note of notes) {
+				let poll: MiPoll | undefined;
+				if (note.hasPoll) {
+					poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id });
+				}
+				const files = await this.driveFileEntityService.packManyByIds(note.fileIds);
+				const content = JSON.stringify(this.noteSerialize(note, poll, files));
+				const isFirst = exportedNotesCount === 0;
+				await writeNotes(isFirst ? content : ',\n' + content);
+				exportedNotesCount++;
+			}
+		}
+
+		await writeNotes(']}');
+
+		notesStream.end();
+
+		// Following Export
+
+		const followingsPath = path + '/followings.json';
+
+		fs.writeFileSync(followingsPath, '', 'utf-8');
+
+		const followingStream = fs.createWriteStream(followingsPath, { flags: 'a' });
+
+		const writeFollowing = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				followingStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeFollowing(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followings":[`);
+
+		let followingsCursor: MiFollowing['id'] | null = null;
+		let exportedFollowingsCount = 0;
+
+		const mutings = await this.mutingsRepository.findBy({
+			muterId: user.id,
+		});
+		
+		while (true) {
+			const followings = await this.followingsRepository.find({
+				where: {
+					followerId: user.id,
+					...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}),
+					...(followingsCursor ? { id: MoreThan(followingsCursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			}) as MiFollowing[];
+
+			if (followings.length === 0) {
+				break;
+			}
+
+			followingsCursor = followings.at(-1)?.id ?? null;
+
+			for (const following of followings) {
+				const u = await this.usersRepository.findOneBy({ id: following.followeeId });
+				if (u == null) {
+					continue;
+				}
+
+				if (u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) {
+					continue;
+				}
+
+				const isFirst = exportedFollowingsCount === 0;
+				const content = this.utilityService.getFullApAccount(u.username, u.host);
+				await writeFollowing(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+				exportedFollowingsCount++;
+			}
+		}
+
+		await writeFollowing(']}');
+
+		followingStream.end();
+
+		// Followers Export
+
+		const followersPath = path + '/followers.json';
+
+		fs.writeFileSync(followersPath, '', 'utf-8');
+
+		const followerStream = fs.createWriteStream(followersPath, { flags: 'a' });
+
+		const writeFollowers = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				followerStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeFollowers(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followers":[`);
+
+		let followersCursor: MiFollowing['id'] | null = null;
+		let exportedFollowersCount = 0;
+		
+		while (true) {
+			const followers = await this.followingsRepository.find({
+				where: {
+					followeeId: user.id,
+					...(followersCursor ? { id: MoreThan(followersCursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			}) as MiFollowing[];
+
+			if (followers.length === 0) {
+				break;
+			}
+
+			followersCursor = followers.at(-1)?.id ?? null;
+
+			for (const follower of followers) {
+				const u = await this.usersRepository.findOneBy({ id: follower.followerId });
+				if (u == null) {
+					continue;
+				}
+
+				const isFirst = exportedFollowersCount === 0;
+				const content = this.utilityService.getFullApAccount(u.username, u.host);
+				await writeFollowers(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+				exportedFollowersCount++;
+			}
+		}
+
+		await writeFollowers(']}');
+
+		followerStream.end();
+
+		// Drive Export
+
+		const filesPath = path + '/drive.json';
+
+		fs.writeFileSync(filesPath, '', 'utf-8');
+
+		const filesStream = fs.createWriteStream(filesPath, { flags: 'a' });
+
+		const writeDrive = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				filesStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		fs.mkdirSync(`${path}/files`);
+
+		await writeDrive(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","drive":[`);
+
+		const driveFiles = await this.driveFilesRepository.find({ where: { userId: user.id } });
+
+		for (const file of driveFiles) {
+			const ext = mime.extension(file.type);
+			const fileName = file.name + '.' + ext;
+			const filePath = path + '/files/' + fileName;
+			fs.writeFileSync(filePath, '', 'binary');
+			let downloaded = false;
+
+			try {
+				await this.downloadService.downloadUrl(file.url, filePath);
+				downloaded = true;
+			} catch (e) {
+				this.logger.error(e instanceof Error ? e : new Error(e as string));
+			}
+
+			if (!downloaded) {
+				fs.unlinkSync(filePath);
+			}
+
+			const content = JSON.stringify({
+				fileName: fileName,
+				file: file,
+			});
+			const isFirst = driveFiles.indexOf(file) === 0;
+
+			await writeDrive(isFirst ? content : ',\n' + content);
+		}
+
+		await writeDrive(']}');
+
+		filesStream.end();
+
+		// Muting Export
+
+		const mutingPath = path + '/mutings.json';
+
+		fs.writeFileSync(mutingPath, '', 'utf-8');
+
+		const mutingStream = fs.createWriteStream(mutingPath, { flags: 'a' });
+
+		const writeMuting = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				mutingStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeMuting(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","mutings":[`);
+
+		let exportedMutingCount = 0;
+		let mutingCursor: MiMuting['id'] | null = null;
+
+		while (true) {
+			const mutes = await this.mutingsRepository.find({
+				where: {
+					muterId: user.id,
+					expiresAt: IsNull(),
+					...(mutingCursor ? { id: MoreThan(mutingCursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			});
+
+			if (mutes.length === 0) {
+				break;
+			}
+
+			mutingCursor = mutes.at(-1)?.id ?? null;
+
+			for (const mute of mutes) {
+				const u = await this.usersRepository.findOneBy({ id: mute.muteeId });
+
+				if (u == null) {
+					exportedMutingCount++; continue;
+				}
+
+				const content = this.utilityService.getFullApAccount(u.username, u.host);
+				const isFirst = exportedMutingCount === 0;
+				await writeMuting(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+				exportedMutingCount++;
+			}
+		}
+
+		await writeMuting(']}');
+
+		mutingStream.end();
+
+		// Blockings Export
+
+		const blockingPath = path + '/blockings.json';
+
+		fs.writeFileSync(blockingPath, '', 'utf-8');
+
+		const blockingStream = fs.createWriteStream(blockingPath, { flags: 'a' });
+
+		const writeBlocking = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				blockingStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeBlocking(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","blockings":[`);
+
+		let exportedBlockingCount = 0;
+		let blockingCursor: MiBlocking['id'] | null = null;
+
+		while (true) {
+			const blockings = await this.blockingsRepository.find({
+				where: {
+					blockerId: user.id,
+					...(blockingCursor ? { id: MoreThan(blockingCursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			});
+
+			if (blockings.length === 0) {
+				break;
+			}
+
+			blockingCursor = blockings.at(-1)?.id ?? null;
+
+			for (const block of blockings) {
+				const u = await this.usersRepository.findOneBy({ id: block.blockeeId });
+
+				if (u == null) {
+					exportedBlockingCount++; continue;
+				}
+
+				const content = this.utilityService.getFullApAccount(u.username, u.host);
+				const isFirst = exportedBlockingCount === 0;
+				await writeBlocking(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+				exportedBlockingCount++;
+			}
+		}
+
+		await writeBlocking(']}');
+
+		blockingStream.end();
+
+		// Favorites export
+
+		const favoritePath = path + '/favorites.json';
+
+		fs.writeFileSync(favoritePath, '', 'utf-8');
+
+		const favoriteStream = fs.createWriteStream(favoritePath, { flags: 'a' });
+
+		const writeFavorite = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				favoriteStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeFavorite(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","favorites":[`);
+
+		let exportedFavoritesCount = 0;
+		let favoriteCursor: MiNoteFavorite['id'] | null = null;
+
+		while (true) {
+			const favorites = await this.noteFavoritesRepository.find({
+				where: {
+					userId: user.id,
+					...(favoriteCursor ? { id: MoreThan(favoriteCursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+				relations: ['note', 'note.user'],
+			}) as (MiNoteFavorite & { note: MiNote & { user: MiUser } })[];
+
+			if (favorites.length === 0) {
+				break;
+			}
+
+			favoriteCursor = favorites.at(-1)?.id ?? null;
+
+			for (const favorite of favorites) {
+				let poll: MiPoll | undefined;
+				if (favorite.note.hasPoll) {
+					poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id });
+				}
+				const content = JSON.stringify(this.favoriteSerialize(favorite, poll));
+				const isFirst = exportedFavoritesCount === 0;
+				await writeFavorite(isFirst ? content : ',\n' + content);
+				exportedFavoritesCount++;
+			}
+		}
+
+		await writeFavorite(']}');
+
+		favoriteStream.end();
+
+		// Antennas export
+
+		const antennaPath = path + '/antennas.json';
+
+		fs.writeFileSync(antennaPath, '', 'utf-8');
+
+		const antennaStream = fs.createWriteStream(antennaPath, { flags: 'a' });
+
+		const writeAntenna = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				antennaStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		await writeAntenna(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","antennas":[`);
+
+		const antennas = await this.antennasRepository.findBy({ userId: user.id });
+
+		for (const [index, antenna] of antennas.entries()) {
+			let users: MiUser[] | undefined;
+			if (antenna.userListId !== null) {
+				const memberships = await this.userListMembershipsRepository.findBy({ userListId: antenna.userListId });
+				users = await this.usersRepository.findBy({
+					id: In(memberships.map(j => j.userId)),
+				});
+			}
+
+			await writeAntenna(JSON.stringify({
+				name: antenna.name,
+				src: antenna.src,
+				keywords: antenna.keywords,
+				excludeKeywords: antenna.excludeKeywords,
+				users: antenna.users,
+				userListAccts: typeof users !== 'undefined' ? users.map((u) => {
+					return this.utilityService.getFullApAccount(u.username, u.host); // acct
+				}) : null,
+				caseSensitive: antenna.caseSensitive,
+				localOnly: antenna.localOnly,
+				withReplies: antenna.withReplies,
+				withFile: antenna.withFile,
+				notify: antenna.notify,
+			}));
+
+			if (antennas.length - 1 !== index) {
+				await writeAntenna(', ');
+			}
+		}
+
+		await writeAntenna(']}');
+
+		antennaStream.end();
+
+		// Lists export
+
+		const listPath = path + '/lists.csv';
+
+		fs.writeFileSync(listPath, '', 'utf-8');
+
+		const listStream = fs.createWriteStream(listPath, { flags: 'a' });
+
+		const writeList = (text: string): Promise<void> => {
+			return new Promise<void>((res, rej) => {
+				listStream.write(text, err => {
+					if (err) {
+						this.logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+		};
+
+		const lists = await this.userListsRepository.findBy({
+			userId: user.id,
+		});
+
+		for (const list of lists) {
+			const memberships = await this.userListMembershipsRepository.findBy({ userListId: list.id });
+			const users = await this.usersRepository.findBy({
+				id: In(memberships.map(j => j.userId)),
+			});
+
+			for (const u of users) {
+				const acct = this.utilityService.getFullApAccount(u.username, u.host);
+				const content = `${list.name},${acct}`;
+				await writeList(content + '\n');
+			}
+		}
+
+		listStream.end();
+
+		// Create archive
+		await new Promise<void>(async (resolve) => {
+			const [archivePath, archiveCleanup] = await createTemp();
+			const archiveStream = fs.createWriteStream(archivePath);
+			const archive = archiver('zip', {
+				zlib: { level: 0 },
+			});
+			archiveStream.on('close', async () => {
+				this.logger.succ(`Exported to: ${archivePath}`);
+
+				const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
+				const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
+
+				this.logger.succ(`Exported to: ${driveFile.id}`);
+				cleanup();
+				archiveCleanup();
+				if (profile.email) {
+					this.emailService.sendEmail(profile.email, 
+						'Your data archive is ready', 
+						`Click the following link to download the archive: ${driveFile.url}<br/>It is also available in your drive.`, 
+						`Click the following link to download the archive: ${driveFile.url}\r\n\r\nIt is also available in your drive.`,
+					);
+				}
+				resolve();
+			});
+			archive.pipe(archiveStream);
+			archive.directory(path, false);
+			archive.finalize();
+		});
+	}
+
+	private noteSerialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record<string, unknown> {
+		return {
+			id: note.id,
+			text: note.text,
+			createdAt: this.idService.parse(note.id).date.toISOString(),
+			fileIds: note.fileIds,
+			files: files,
+			replyId: note.replyId,
+			renoteId: note.renoteId,
+			poll: poll,
+			cw: note.cw,
+			visibility: note.visibility,
+			visibleUserIds: note.visibleUserIds,
+			localOnly: note.localOnly,
+			reactionAcceptance: note.reactionAcceptance,
+		};
+	}
+
+	private favoriteSerialize(favorite: MiNoteFavorite & { note: MiNote & { user: MiUser } }, poll: MiPoll | null = null): Record<string, unknown> {
+		return {
+			id: favorite.id,
+			createdAt: this.idService.parse(favorite.id).date.toISOString(),
+			note: {
+				id: favorite.note.id,
+				text: favorite.note.text,
+				createdAt: this.idService.parse(favorite.note.id).date.toISOString(),
+				fileIds: favorite.note.fileIds,
+				replyId: favorite.note.replyId,
+				renoteId: favorite.note.renoteId,
+				poll: poll,
+				cw: favorite.note.cw,
+				visibility: favorite.note.visibility,
+				visibleUserIds: favorite.note.visibleUserIds,
+				localOnly: favorite.note.localOnly,
+				reactionAcceptance: favorite.note.reactionAcceptance,
+				uri: favorite.note.uri,
+				url: favorite.note.url,
+				user: {
+					id: favorite.note.user.id,
+					name: favorite.note.user.name,
+					username: favorite.note.user.username,
+					host: favorite.note.user.host,
+					uri: favorite.note.user.uri,
+				},
+			},
+		};
+	}
+}

packages/backend/src/queue/types.ts

@@ -39,6 +39,7 @@ export type DbJobData<T extends keyof DbJobMap> = DbJobMap[T];
 
 export type DbJobMap = {
 	deleteDriveFiles: DbJobDataWithUser;
+	exportAccountData: DbJobDataWithUser;
 	exportCustomEmojis: DbJobDataWithUser;
 	exportAntennas: DBExportAntennasData;
 	exportNotes: DbJobDataWithUser;

packages/backend/src/server/api/EndpointsModule.ts

@@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js';
 import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js';
 import * as ep___i_changePassword from './endpoints/i/change-password.js';
 import * as ep___i_deleteAccount from './endpoints/i/delete-account.js';
+import * as ep___i_exportData from './endpoints/i/export-data.js';
 import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
 import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
 import * as ep___i_exportMute from './endpoints/i/export-mute.js';
@@ -573,6 +574,7 @@ const $i_authorizedApps: Provider = { provide: 'ep:i/authorized-apps', useClass:
 const $i_claimAchievement: Provider = { provide: 'ep:i/claim-achievement', useClass: ep___i_claimAchievement.default };
 const $i_changePassword: Provider = { provide: 'ep:i/change-password', useClass: ep___i_changePassword.default };
 const $i_deleteAccount: Provider = { provide: 'ep:i/delete-account', useClass: ep___i_deleteAccount.default };
+const $i_exportData: Provider = { provide: 'ep:i/export-data', useClass: ep___i_exportData.default };
 const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: ep___i_exportBlocking.default };
 const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
 const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
@@ -944,6 +946,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
 		$i_claimAchievement,
 		$i_changePassword,
 		$i_deleteAccount,
+		$i_exportData,
 		$i_exportBlocking,
 		$i_exportFollowing,
 		$i_exportMute,
@@ -1309,6 +1312,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
 		$i_claimAchievement,
 		$i_changePassword,
 		$i_deleteAccount,
+		$i_exportData,
 		$i_exportBlocking,
 		$i_exportFollowing,
 		$i_exportMute,

packages/backend/src/server/api/endpoints.ts

@@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js';
 import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js';
 import * as ep___i_changePassword from './endpoints/i/change-password.js';
 import * as ep___i_deleteAccount from './endpoints/i/delete-account.js';
+import * as ep___i_exportData from './endpoints/i/export-data.js';
 import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
 import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
 import * as ep___i_exportMute from './endpoints/i/export-mute.js';
@@ -571,6 +572,7 @@ const eps = [
 	['i/claim-achievement', ep___i_claimAchievement],
 	['i/change-password', ep___i_changePassword],
 	['i/delete-account', ep___i_deleteAccount],
+	['i/export-data', ep___i_exportData],
 	['i/export-blocking', ep___i_exportBlocking],
 	['i/export-following', ep___i_exportFollowing],
 	['i/export-mute', ep___i_exportMute],

packages/backend/src/server/api/endpoints/i/export-data.ts

@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Injectable } from '@nestjs/common';
+import ms from 'ms';
+import { Endpoint } from '@/server/api/endpoint-base.js';
+import { QueueService } from '@/core/QueueService.js';
+
+export const meta = {
+	secure: true,
+	requireCredential: true,
+	limit: {
+		duration: ms('3days'),
+		max: 1,
+	},
+} as const;
+
+export const paramDef = {
+	type: 'object',
+	properties: {},
+	required: [],
+} as const;
+
+@Injectable()
+export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
+	constructor(
+		private queueService: QueueService,
+	) {
+		super(meta, paramDef, async (ps, me) => {
+			this.queueService.createExportAccountDataJob(me);
+		});
+	}
+}

packages/frontend/src/pages/settings/other.vue

@@ -34,6 +34,17 @@ SPDX-License-Identifier: AGPL-3.0-only
 				</div>
 			</MkFolder>
 
+			<MkFolder>
+				<template #icon><i class="ph-database ph-bold ph-lg"></i></template>
+				<template #label>{{ i18n.ts._dataRequest.title }}</template>
+
+				<div class="_gaps_m">
+					<FormInfo warn>{{ i18n.ts._dataRequest.warn }}</FormInfo>
+					<FormInfo>{{ i18n.ts._dataRequest.text }}</FormInfo>
+					<MkButton primary @click="exportData">{{ i18n.ts._dataRequest.button }}</MkButton>
+				</div>
+			</MkFolder>
+
 			<MkFolder>
 				<template #icon><i class="ph-warning ph-bold ph-lg"></i></template>
 				<template #label>{{ i18n.ts.closeAccount }}</template>
@@ -156,6 +167,20 @@ async function updateRepliesAll(withReplies: boolean) {
 	await os.api('following/update-all', { withReplies });
 }
 
+const exportData = () => {
+	os.api('i/export-data', {}).then(() => {
+		os.alert({
+			type: 'info',
+			text: i18n.ts.exportRequested,
+		});
+	}).catch((ev) => {
+		os.alert({
+			type: 'error',
+			text: ev.message,
+		});
+	});
+};
+
 watch([
 	enableCondensedLineForAcct,
 ], async () => {