merge: GDPR Data export for users (#143)
feat: GDPR Data export for users
This commit is contained in:
Marie
GitHub
commit
739f958842
12 changed files with 876 additions and 0 deletions
|
|
@ -2401,3 +2401,9 @@ _animatedMFM:
|
|||
_alert:
|
||||
text: "Animated MFMs could include flashing lights and fast moving text/emojis."
|
||||
confirm: "Animate"
|
||||
|
||||
_dataRequest:
|
||||
title: "Request Data"
|
||||
warn: "Data requests are only possible every 3 days."
|
||||
text: "Once the data is ready to download, an email will be sent to the email address registered to this account."
|
||||
button: "Request"
|
||||
|
|
|
|||
6
locales/index.d.ts
vendored
6
locales/index.d.ts
vendored
|
|
@ -2509,6 +2509,12 @@ export interface Locale {
|
|||
"confirm": string;
|
||||
};
|
||||
};
|
||||
"_dataRequest": {
|
||||
"title": string;
|
||||
"warn": string;
|
||||
"text": string;
|
||||
"button": string;
|
||||
};
|
||||
}
|
||||
declare const locales: {
|
||||
[lang: string]: Locale;
|
||||
|
|
|
|||
|
|
@ -2395,3 +2395,9 @@ _animatedMFM:
|
|||
_alert:
|
||||
text: "アニメーションMFMには、点滅するライトや高速で動くテキスト／絵文字を含めることができる。"
|
||||
confirm: "アニメイト"
|
||||
|
||||
_dataRequest:
|
||||
title: "リクエストデータ"
|
||||
warn: "データのリクエストは3日ごとにしかできない。"
|
||||
text: "データのダウンロードが完了すると、このアカウントに登録されているEメールアドレスにEメールが送信されます。"
|
||||
button: "リクエスト"
|
||||
|
|
|
|||
|
|
@ -164,6 +164,16 @@ export class QueueService {
|
|||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createExportAccountDataJob(user: ThinUser) {
|
||||
return this.dbQueue.add('exportAccountData', {
|
||||
user: { id: user.id },
|
||||
}, {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createExportNotesJob(user: ThinUser) {
|
||||
return this.dbQueue.add('exportNotes', {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesP
|
|||
import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
|
||||
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
|
||||
import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js';
|
||||
import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js';
|
||||
import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
|
||||
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
|
||||
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
|
||||
|
|
@ -51,6 +52,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
|
|||
CheckExpiredMutingsProcessorService,
|
||||
CleanProcessorService,
|
||||
DeleteDriveFilesProcessorService,
|
||||
ExportAccountDataProcessorService,
|
||||
ExportCustomEmojisProcessorService,
|
||||
ExportNotesProcessorService,
|
||||
ExportFavoritesProcessorService,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { EndedPollNotificationProcessorService } from './processors/EndedPollNot
|
|||
import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
|
||||
import { InboxProcessorService } from './processors/InboxProcessorService.js';
|
||||
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
|
||||
import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js';
|
||||
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
|
||||
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
|
||||
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
|
||||
|
|
@ -89,6 +90,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
private deliverProcessorService: DeliverProcessorService,
|
||||
private inboxProcessorService: InboxProcessorService,
|
||||
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
|
||||
private exportAccountDataProcessorService: ExportAccountDataProcessorService,
|
||||
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
|
||||
private exportNotesProcessorService: ExportNotesProcessorService,
|
||||
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
|
||||
|
|
@ -162,6 +164,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
|
||||
switch (job.name) {
|
||||
case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
|
||||
case 'exportAccountData': return this.exportAccountDataProcessorService.process(job);
|
||||
case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
|
||||
case 'exportNotes': return this.exportNotesProcessorService.process(job);
|
||||
case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,776 @@
|
|||
/* eslint-disable no-constant-condition */
|
||||
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { In, IsNull, MoreThan, Not } from 'typeorm';
|
||||
import { format as dateFormat } from 'date-fns';
|
||||
import mime from 'mime-types';
|
||||
import archiver from 'archiver';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { AntennasRepository, BlockingsRepository, DriveFilesRepository, FollowingsRepository, MiBlocking, MiFollowing, MiMuting, MiNote, MiNoteFavorite, MiPoll, MiUser, MutingsRepository, NoteFavoritesRepository, NotesRepository, PollsRepository, UserListMembershipsRepository, UserListsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
|
||||
import type { Config } from '@/config.js';
|
||||
import type Logger from '@/logger.js';
|
||||
import { DriveService } from '@/core/DriveService.js';
|
||||
import { IdService } from '@/core/IdService.js';
|
||||
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
|
||||
import { createTemp, createTempDir } from '@/misc/create-temp.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import { Packed } from '@/misc/json-schema.js';
|
||||
import { UtilityService } from '@/core/UtilityService.js';
|
||||
import { DownloadService } from '@/core/DownloadService.js';
|
||||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||
import type * as Bull from 'bullmq';
|
||||
import { EmailService } from '@/core/EmailService.js';
|
||||
|
||||
@Injectable()
|
||||
export class ExportAccountDataProcessorService {
|
||||
private logger: Logger;
|
||||
|
||||
constructor(
|
||||
@Inject(DI.config)
|
||||
private config: Config,
|
||||
|
||||
@Inject(DI.usersRepository)
|
||||
private usersRepository: UsersRepository,
|
||||
|
||||
@Inject(DI.userProfilesRepository)
|
||||
private userProfilesRepository: UserProfilesRepository,
|
||||
|
||||
@Inject(DI.notesRepository)
|
||||
private notesRepository: NotesRepository,
|
||||
|
||||
@Inject(DI.noteFavoritesRepository)
|
||||
private noteFavoritesRepository: NoteFavoritesRepository,
|
||||
|
||||
@Inject(DI.pollsRepository)
|
||||
private pollsRepository: PollsRepository,
|
||||
|
||||
@Inject(DI.followingsRepository)
|
||||
private followingsRepository: FollowingsRepository,
|
||||
|
||||
@Inject(DI.mutingsRepository)
|
||||
private mutingsRepository: MutingsRepository,
|
||||
|
||||
@Inject(DI.blockingsRepository)
|
||||
private blockingsRepository: BlockingsRepository,
|
||||
|
||||
@Inject(DI.driveFilesRepository)
|
||||
private driveFilesRepository: DriveFilesRepository,
|
||||
|
||||
@Inject(DI.antennasRepository)
|
||||
private antennasRepository: AntennasRepository,
|
||||
|
||||
@Inject(DI.userListsRepository)
|
||||
private userListsRepository: UserListsRepository,
|
||||
|
||||
@Inject(DI.userListMembershipsRepository)
|
||||
private userListMembershipsRepository: UserListMembershipsRepository,
|
||||
|
||||
private utilityService: UtilityService,
|
||||
private driveService: DriveService,
|
||||
private idService: IdService,
|
||||
private driveFileEntityService: DriveFileEntityService,
|
||||
private downloadService: DownloadService,
|
||||
private emailService: EmailService,
|
||||
private queueLoggerService: QueueLoggerService,
|
||||
) {
|
||||
this.logger = this.queueLoggerService.logger.createSubLogger('export-account-data');
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async process(job: Bull.Job): Promise<void> {
|
||||
this.logger.info('Exporting Account Data...');
|
||||
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const profile = await this.userProfilesRepository.findOneBy({ userId: job.data.user.id });
|
||||
if (profile == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const [path, cleanup] = await createTempDir();
|
||||
|
||||
this.logger.info(`Temp dir is ${path}`);
|
||||
|
||||
// User Export
|
||||
|
||||
const userPath = path + '/user.json';
|
||||
|
||||
fs.writeFileSync(userPath, '', 'utf-8');
|
||||
|
||||
const userStream = fs.createWriteStream(userPath, { flags: 'a' });
|
||||
|
||||
const writeUser = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
userStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeUser(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","user":[`);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { host, uri, sharedInbox, followersUri, lastFetchedAt, inbox, ...userTrimmed } = user;
|
||||
|
||||
await writeUser(JSON.stringify(userTrimmed));
|
||||
|
||||
await writeUser(']}');
|
||||
|
||||
userStream.end();
|
||||
|
||||
// Profile Export
|
||||
|
||||
const profilePath = path + '/profile.json';
|
||||
|
||||
fs.writeFileSync(profilePath, '', 'utf-8');
|
||||
|
||||
const profileStream = fs.createWriteStream(profilePath, { flags: 'a' });
|
||||
|
||||
const writeProfile = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
profileStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { emailVerifyCode, twoFactorBackupSecret, twoFactorSecret, password, twoFactorTempSecret, userHost, ...profileTrimmed } = profile;
|
||||
|
||||
await writeProfile(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","profile":[`);
|
||||
|
||||
await writeProfile(JSON.stringify(profileTrimmed));
|
||||
|
||||
await writeProfile(']}');
|
||||
|
||||
profileStream.end();
|
||||
|
||||
// Note Export
|
||||
|
||||
const notesPath = path + '/notes.json';
|
||||
|
||||
fs.writeFileSync(notesPath, '', 'utf-8');
|
||||
|
||||
const notesStream = fs.createWriteStream(notesPath, { flags: 'a' });
|
||||
|
||||
const writeNotes = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
notesStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeNotes(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","notes":[`);
|
||||
|
||||
let noteCursor: MiNote['id'] | null = null;
|
||||
let exportedNotesCount = 0;
|
||||
|
||||
while (true) {
|
||||
const notes = await this.notesRepository.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...(noteCursor ? { id: MoreThan(noteCursor) } : {}),
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
}) as MiNote[];
|
||||
|
||||
if (notes.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
noteCursor = notes.at(-1)?.id ?? null;
|
||||
|
||||
for (const note of notes) {
|
||||
let poll: MiPoll | undefined;
|
||||
if (note.hasPoll) {
|
||||
poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id });
|
||||
}
|
||||
const files = await this.driveFileEntityService.packManyByIds(note.fileIds);
|
||||
const content = JSON.stringify(this.noteSerialize(note, poll, files));
|
||||
const isFirst = exportedNotesCount === 0;
|
||||
await writeNotes(isFirst ? content : ',\n' + content);
|
||||
exportedNotesCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeNotes(']}');
|
||||
|
||||
notesStream.end();
|
||||
|
||||
// Following Export
|
||||
|
||||
const followingsPath = path + '/followings.json';
|
||||
|
||||
fs.writeFileSync(followingsPath, '', 'utf-8');
|
||||
|
||||
const followingStream = fs.createWriteStream(followingsPath, { flags: 'a' });
|
||||
|
||||
const writeFollowing = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
followingStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeFollowing(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followings":[`);
|
||||
|
||||
let followingsCursor: MiFollowing['id'] | null = null;
|
||||
let exportedFollowingsCount = 0;
|
||||
|
||||
const mutings = await this.mutingsRepository.findBy({
|
||||
muterId: user.id,
|
||||
});
|
||||
|
||||
while (true) {
|
||||
const followings = await this.followingsRepository.find({
|
||||
where: {
|
||||
followerId: user.id,
|
||||
...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}),
|
||||
...(followingsCursor ? { id: MoreThan(followingsCursor) } : {}),
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
}) as MiFollowing[];
|
||||
|
||||
if (followings.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
followingsCursor = followings.at(-1)?.id ?? null;
|
||||
|
||||
for (const following of followings) {
|
||||
const u = await this.usersRepository.findOneBy({ id: following.followeeId });
|
||||
if (u == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const isFirst = exportedFollowingsCount === 0;
|
||||
const content = this.utilityService.getFullApAccount(u.username, u.host);
|
||||
await writeFollowing(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
|
||||
exportedFollowingsCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeFollowing(']}');
|
||||
|
||||
followingStream.end();
|
||||
|
||||
// Followers Export
|
||||
|
||||
const followersPath = path + '/followers.json';
|
||||
|
||||
fs.writeFileSync(followersPath, '', 'utf-8');
|
||||
|
||||
const followerStream = fs.createWriteStream(followersPath, { flags: 'a' });
|
||||
|
||||
const writeFollowers = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
followerStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeFollowers(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followers":[`);
|
||||
|
||||
let followersCursor: MiFollowing['id'] | null = null;
|
||||
let exportedFollowersCount = 0;
|
||||
|
||||
while (true) {
|
||||
const followers = await this.followingsRepository.find({
|
||||
where: {
|
||||
followeeId: user.id,
|
||||
...(followersCursor ? { id: MoreThan(followersCursor) } : {}),
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
}) as MiFollowing[];
|
||||
|
||||
if (followers.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
followersCursor = followers.at(-1)?.id ?? null;
|
||||
|
||||
for (const follower of followers) {
|
||||
const u = await this.usersRepository.findOneBy({ id: follower.followerId });
|
||||
if (u == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const isFirst = exportedFollowersCount === 0;
|
||||
const content = this.utilityService.getFullApAccount(u.username, u.host);
|
||||
await writeFollowers(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
|
||||
exportedFollowersCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeFollowers(']}');
|
||||
|
||||
followerStream.end();
|
||||
|
||||
// Drive Export
|
||||
|
||||
const filesPath = path + '/drive.json';
|
||||
|
||||
fs.writeFileSync(filesPath, '', 'utf-8');
|
||||
|
||||
const filesStream = fs.createWriteStream(filesPath, { flags: 'a' });
|
||||
|
||||
const writeDrive = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
filesStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
fs.mkdirSync(`${path}/files`);
|
||||
|
||||
await writeDrive(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","drive":[`);
|
||||
|
||||
const driveFiles = await this.driveFilesRepository.find({ where: { userId: user.id } });
|
||||
|
||||
for (const file of driveFiles) {
|
||||
const ext = mime.extension(file.type);
|
||||
const fileName = file.name + '.' + ext;
|
||||
const filePath = path + '/files/' + fileName;
|
||||
fs.writeFileSync(filePath, '', 'binary');
|
||||
let downloaded = false;
|
||||
|
||||
try {
|
||||
await this.downloadService.downloadUrl(file.url, filePath);
|
||||
downloaded = true;
|
||||
} catch (e) {
|
||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||
}
|
||||
|
||||
if (!downloaded) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
|
||||
const content = JSON.stringify({
|
||||
fileName: fileName,
|
||||
file: file,
|
||||
});
|
||||
const isFirst = driveFiles.indexOf(file) === 0;
|
||||
|
||||
await writeDrive(isFirst ? content : ',\n' + content);
|
||||
}
|
||||
|
||||
await writeDrive(']}');
|
||||
|
||||
filesStream.end();
|
||||
|
||||
// Muting Export
|
||||
|
||||
const mutingPath = path + '/mutings.json';
|
||||
|
||||
fs.writeFileSync(mutingPath, '', 'utf-8');
|
||||
|
||||
const mutingStream = fs.createWriteStream(mutingPath, { flags: 'a' });
|
||||
|
||||
const writeMuting = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
mutingStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeMuting(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","mutings":[`);
|
||||
|
||||
let exportedMutingCount = 0;
|
||||
let mutingCursor: MiMuting['id'] | null = null;
|
||||
|
||||
while (true) {
|
||||
const mutes = await this.mutingsRepository.find({
|
||||
where: {
|
||||
muterId: user.id,
|
||||
expiresAt: IsNull(),
|
||||
...(mutingCursor ? { id: MoreThan(mutingCursor) } : {}),
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
});
|
||||
|
||||
if (mutes.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
mutingCursor = mutes.at(-1)?.id ?? null;
|
||||
|
||||
for (const mute of mutes) {
|
||||
const u = await this.usersRepository.findOneBy({ id: mute.muteeId });
|
||||
|
||||
if (u == null) {
|
||||
exportedMutingCount++; continue;
|
||||
}
|
||||
|
||||
const content = this.utilityService.getFullApAccount(u.username, u.host);
|
||||
const isFirst = exportedMutingCount === 0;
|
||||
await writeMuting(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
|
||||
exportedMutingCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeMuting(']}');
|
||||
|
||||
mutingStream.end();
|
||||
|
||||
// Blockings Export
|
||||
|
||||
const blockingPath = path + '/blockings.json';
|
||||
|
||||
fs.writeFileSync(blockingPath, '', 'utf-8');
|
||||
|
||||
const blockingStream = fs.createWriteStream(blockingPath, { flags: 'a' });
|
||||
|
||||
const writeBlocking = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
blockingStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeBlocking(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","blockings":[`);
|
||||
|
||||
let exportedBlockingCount = 0;
|
||||
let blockingCursor: MiBlocking['id'] | null = null;
|
||||
|
||||
while (true) {
|
||||
const blockings = await this.blockingsRepository.find({
|
||||
where: {
|
||||
blockerId: user.id,
|
||||
...(blockingCursor ? { id: MoreThan(blockingCursor) } : {}),
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
});
|
||||
|
||||
if (blockings.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
blockingCursor = blockings.at(-1)?.id ?? null;
|
||||
|
||||
for (const block of blockings) {
|
||||
const u = await this.usersRepository.findOneBy({ id: block.blockeeId });
|
||||
|
||||
if (u == null) {
|
||||
exportedBlockingCount++; continue;
|
||||
}
|
||||
|
||||
const content = this.utilityService.getFullApAccount(u.username, u.host);
|
||||
const isFirst = exportedBlockingCount === 0;
|
||||
await writeBlocking(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
|
||||
exportedBlockingCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeBlocking(']}');
|
||||
|
||||
blockingStream.end();
|
||||
|
||||
// Favorites export
|
||||
|
||||
const favoritePath = path + '/favorites.json';
|
||||
|
||||
fs.writeFileSync(favoritePath, '', 'utf-8');
|
||||
|
||||
const favoriteStream = fs.createWriteStream(favoritePath, { flags: 'a' });
|
||||
|
||||
const writeFavorite = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
favoriteStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeFavorite(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","favorites":[`);
|
||||
|
||||
let exportedFavoritesCount = 0;
|
||||
let favoriteCursor: MiNoteFavorite['id'] | null = null;
|
||||
|
||||
while (true) {
|
||||
const favorites = await this.noteFavoritesRepository.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...(favoriteCursor ? { id: MoreThan(favoriteCursor) } : {}),
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
relations: ['note', 'note.user'],
|
||||
}) as (MiNoteFavorite & { note: MiNote & { user: MiUser } })[];
|
||||
|
||||
if (favorites.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
favoriteCursor = favorites.at(-1)?.id ?? null;
|
||||
|
||||
for (const favorite of favorites) {
|
||||
let poll: MiPoll | undefined;
|
||||
if (favorite.note.hasPoll) {
|
||||
poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id });
|
||||
}
|
||||
const content = JSON.stringify(this.favoriteSerialize(favorite, poll));
|
||||
const isFirst = exportedFavoritesCount === 0;
|
||||
await writeFavorite(isFirst ? content : ',\n' + content);
|
||||
exportedFavoritesCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeFavorite(']}');
|
||||
|
||||
favoriteStream.end();
|
||||
|
||||
// Antennas export
|
||||
|
||||
const antennaPath = path + '/antennas.json';
|
||||
|
||||
fs.writeFileSync(antennaPath, '', 'utf-8');
|
||||
|
||||
const antennaStream = fs.createWriteStream(antennaPath, { flags: 'a' });
|
||||
|
||||
const writeAntenna = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
antennaStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await writeAntenna(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","antennas":[`);
|
||||
|
||||
const antennas = await this.antennasRepository.findBy({ userId: user.id });
|
||||
|
||||
for (const [index, antenna] of antennas.entries()) {
|
||||
let users: MiUser[] | undefined;
|
||||
if (antenna.userListId !== null) {
|
||||
const memberships = await this.userListMembershipsRepository.findBy({ userListId: antenna.userListId });
|
||||
users = await this.usersRepository.findBy({
|
||||
id: In(memberships.map(j => j.userId)),
|
||||
});
|
||||
}
|
||||
|
||||
await writeAntenna(JSON.stringify({
|
||||
name: antenna.name,
|
||||
src: antenna.src,
|
||||
keywords: antenna.keywords,
|
||||
excludeKeywords: antenna.excludeKeywords,
|
||||
users: antenna.users,
|
||||
userListAccts: typeof users !== 'undefined' ? users.map((u) => {
|
||||
return this.utilityService.getFullApAccount(u.username, u.host); // acct
|
||||
}) : null,
|
||||
caseSensitive: antenna.caseSensitive,
|
||||
localOnly: antenna.localOnly,
|
||||
withReplies: antenna.withReplies,
|
||||
withFile: antenna.withFile,
|
||||
notify: antenna.notify,
|
||||
}));
|
||||
|
||||
if (antennas.length - 1 !== index) {
|
||||
await writeAntenna(', ');
|
||||
}
|
||||
}
|
||||
|
||||
await writeAntenna(']}');
|
||||
|
||||
antennaStream.end();
|
||||
|
||||
// Lists export
|
||||
|
||||
const listPath = path + '/lists.csv';
|
||||
|
||||
fs.writeFileSync(listPath, '', 'utf-8');
|
||||
|
||||
const listStream = fs.createWriteStream(listPath, { flags: 'a' });
|
||||
|
||||
const writeList = (text: string): Promise<void> => {
|
||||
return new Promise<void>((res, rej) => {
|
||||
listStream.write(text, err => {
|
||||
if (err) {
|
||||
this.logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const lists = await this.userListsRepository.findBy({
|
||||
userId: user.id,
|
||||
});
|
||||
|
||||
for (const list of lists) {
|
||||
const memberships = await this.userListMembershipsRepository.findBy({ userListId: list.id });
|
||||
const users = await this.usersRepository.findBy({
|
||||
id: In(memberships.map(j => j.userId)),
|
||||
});
|
||||
|
||||
for (const u of users) {
|
||||
const acct = this.utilityService.getFullApAccount(u.username, u.host);
|
||||
const content = `${list.name},${acct}`;
|
||||
await writeList(content + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
listStream.end();
|
||||
|
||||
// Create archive
|
||||
await new Promise<void>(async (resolve) => {
|
||||
const [archivePath, archiveCleanup] = await createTemp();
|
||||
const archiveStream = fs.createWriteStream(archivePath);
|
||||
const archive = archiver('zip', {
|
||||
zlib: { level: 0 },
|
||||
});
|
||||
archiveStream.on('close', async () => {
|
||||
this.logger.succ(`Exported to: ${archivePath}`);
|
||||
|
||||
const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
|
||||
const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
|
||||
|
||||
this.logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
archiveCleanup();
|
||||
if (profile.email) {
|
||||
this.emailService.sendEmail(profile.email,
|
||||
'Your data archive is ready',
|
||||
`Click the following link to download the archive: ${driveFile.url}<br/>It is also available in your drive.`,
|
||||
`Click the following link to download the archive: ${driveFile.url}\r\n\r\nIt is also available in your drive.`,
|
||||
);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
archive.pipe(archiveStream);
|
||||
archive.directory(path, false);
|
||||
archive.finalize();
|
||||
});
|
||||
}
|
||||
|
||||
private noteSerialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record<string, unknown> {
|
||||
return {
|
||||
id: note.id,
|
||||
text: note.text,
|
||||
createdAt: this.idService.parse(note.id).date.toISOString(),
|
||||
fileIds: note.fileIds,
|
||||
files: files,
|
||||
replyId: note.replyId,
|
||||
renoteId: note.renoteId,
|
||||
poll: poll,
|
||||
cw: note.cw,
|
||||
visibility: note.visibility,
|
||||
visibleUserIds: note.visibleUserIds,
|
||||
localOnly: note.localOnly,
|
||||
reactionAcceptance: note.reactionAcceptance,
|
||||
};
|
||||
}
|
||||
|
||||
private favoriteSerialize(favorite: MiNoteFavorite & { note: MiNote & { user: MiUser } }, poll: MiPoll | null = null): Record<string, unknown> {
|
||||
return {
|
||||
id: favorite.id,
|
||||
createdAt: this.idService.parse(favorite.id).date.toISOString(),
|
||||
note: {
|
||||
id: favorite.note.id,
|
||||
text: favorite.note.text,
|
||||
createdAt: this.idService.parse(favorite.note.id).date.toISOString(),
|
||||
fileIds: favorite.note.fileIds,
|
||||
replyId: favorite.note.replyId,
|
||||
renoteId: favorite.note.renoteId,
|
||||
poll: poll,
|
||||
cw: favorite.note.cw,
|
||||
visibility: favorite.note.visibility,
|
||||
visibleUserIds: favorite.note.visibleUserIds,
|
||||
localOnly: favorite.note.localOnly,
|
||||
reactionAcceptance: favorite.note.reactionAcceptance,
|
||||
uri: favorite.note.uri,
|
||||
url: favorite.note.url,
|
||||
user: {
|
||||
id: favorite.note.user.id,
|
||||
name: favorite.note.user.name,
|
||||
username: favorite.note.user.username,
|
||||
host: favorite.note.user.host,
|
||||
uri: favorite.note.user.uri,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -39,6 +39,7 @@ export type DbJobData<T extends keyof DbJobMap> = DbJobMap[T];
|
|||
|
||||
export type DbJobMap = {
|
||||
deleteDriveFiles: DbJobDataWithUser;
|
||||
exportAccountData: DbJobDataWithUser;
|
||||
exportCustomEmojis: DbJobDataWithUser;
|
||||
exportAntennas: DBExportAntennasData;
|
||||
exportNotes: DbJobDataWithUser;
|
||||
|
|
|
|||
|
|
@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js';
|
|||
import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js';
|
||||
import * as ep___i_changePassword from './endpoints/i/change-password.js';
|
||||
import * as ep___i_deleteAccount from './endpoints/i/delete-account.js';
|
||||
import * as ep___i_exportData from './endpoints/i/export-data.js';
|
||||
import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
|
||||
import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
|
||||
import * as ep___i_exportMute from './endpoints/i/export-mute.js';
|
||||
|
|
@ -573,6 +574,7 @@ const $i_authorizedApps: Provider = { provide: 'ep:i/authorized-apps', useClass:
|
|||
const $i_claimAchievement: Provider = { provide: 'ep:i/claim-achievement', useClass: ep___i_claimAchievement.default };
|
||||
const $i_changePassword: Provider = { provide: 'ep:i/change-password', useClass: ep___i_changePassword.default };
|
||||
const $i_deleteAccount: Provider = { provide: 'ep:i/delete-account', useClass: ep___i_deleteAccount.default };
|
||||
const $i_exportData: Provider = { provide: 'ep:i/export-data', useClass: ep___i_exportData.default };
|
||||
const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: ep___i_exportBlocking.default };
|
||||
const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
|
||||
const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
|
||||
|
|
@ -944,6 +946,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
|
|||
$i_claimAchievement,
|
||||
$i_changePassword,
|
||||
$i_deleteAccount,
|
||||
$i_exportData,
|
||||
$i_exportBlocking,
|
||||
$i_exportFollowing,
|
||||
$i_exportMute,
|
||||
|
|
@ -1309,6 +1312,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
|
|||
$i_claimAchievement,
|
||||
$i_changePassword,
|
||||
$i_deleteAccount,
|
||||
$i_exportData,
|
||||
$i_exportBlocking,
|
||||
$i_exportFollowing,
|
||||
$i_exportMute,
|
||||
|
|
|
|||
|
|
@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js';
|
|||
import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js';
|
||||
import * as ep___i_changePassword from './endpoints/i/change-password.js';
|
||||
import * as ep___i_deleteAccount from './endpoints/i/delete-account.js';
|
||||
import * as ep___i_exportData from './endpoints/i/export-data.js';
|
||||
import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
|
||||
import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
|
||||
import * as ep___i_exportMute from './endpoints/i/export-mute.js';
|
||||
|
|
@ -571,6 +572,7 @@ const eps = [
|
|||
['i/claim-achievement', ep___i_claimAchievement],
|
||||
['i/change-password', ep___i_changePassword],
|
||||
['i/delete-account', ep___i_deleteAccount],
|
||||
['i/export-data', ep___i_exportData],
|
||||
['i/export-blocking', ep___i_exportBlocking],
|
||||
['i/export-following', ep___i_exportFollowing],
|
||||
['i/export-mute', ep___i_exportMute],
|
||||
|
|
|
|||
35
packages/backend/src/server/api/endpoints/i/export-data.ts
Normal file
35
packages/backend/src/server/api/endpoints/i/export-data.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: syuilo and other misskey contributors
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import ms from 'ms';
|
||||
import { Endpoint } from '@/server/api/endpoint-base.js';
|
||||
import { QueueService } from '@/core/QueueService.js';
|
||||
|
||||
export const meta = {
|
||||
secure: true,
|
||||
requireCredential: true,
|
||||
limit: {
|
||||
duration: ms('3days'),
|
||||
max: 1,
|
||||
},
|
||||
} as const;
|
||||
|
||||
export const paramDef = {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
required: [],
|
||||
} as const;
|
||||
|
||||
@Injectable()
|
||||
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
|
||||
constructor(
|
||||
private queueService: QueueService,
|
||||
) {
|
||||
super(meta, paramDef, async (ps, me) => {
|
||||
this.queueService.createExportAccountDataJob(me);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -34,6 +34,17 @@ SPDX-License-Identifier: AGPL-3.0-only
|
|||
</div>
|
||||
</MkFolder>
|
||||
|
||||
<MkFolder>
|
||||
<template #icon><i class="ph-database ph-bold ph-lg"></i></template>
|
||||
<template #label>{{ i18n.ts._dataRequest.title }}</template>
|
||||
|
||||
<div class="_gaps_m">
|
||||
<FormInfo warn>{{ i18n.ts._dataRequest.warn }}</FormInfo>
|
||||
<FormInfo>{{ i18n.ts._dataRequest.text }}</FormInfo>
|
||||
<MkButton primary @click="exportData">{{ i18n.ts._dataRequest.button }}</MkButton>
|
||||
</div>
|
||||
</MkFolder>
|
||||
|
||||
<MkFolder>
|
||||
<template #icon><i class="ph-warning ph-bold ph-lg"></i></template>
|
||||
<template #label>{{ i18n.ts.closeAccount }}</template>
|
||||
|
|
@ -156,6 +167,20 @@ async function updateRepliesAll(withReplies: boolean) {
|
|||
await os.api('following/update-all', { withReplies });
|
||||
}
|
||||
|
||||
const exportData = () => {
|
||||
os.api('i/export-data', {}).then(() => {
|
||||
os.alert({
|
||||
type: 'info',
|
||||
text: i18n.ts.exportRequested,
|
||||
});
|
||||
}).catch((ev) => {
|
||||
os.alert({
|
||||
type: 'error',
|
||||
text: ev.message,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
watch([
|
||||
enableCondensedLineForAcct,
|
||||
], async () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue