feat(backend): allow disabling cache for sensitive files

locales/index.d.ts

@@ -158,6 +158,8 @@ export interface Locale {
     "settingGuide": string;
     "cacheRemoteFiles": string;
     "cacheRemoteFilesDescription": string;
+    "cacheRemoteSensitiveFiles": string;
+    "cacheRemoteSensitiveFilesDescription": string;
     "flagAsBot": string;
     "flagAsBotDescription": string;
     "flagAsCat": string;

locales/ja-JP.yml

@@ -155,6 +155,8 @@ addEmoji: "絵文字を追加"
 settingGuide: "おすすめ設定"
 cacheRemoteFiles: "リモートのファイルをキャッシュする"
 cacheRemoteFilesDescription: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。サーバーのストレージを節約できますが、サムネイルが生成されないので通信量が増加します。"
+cacheRemoteSensitiveFiles: "リモートのNSFWファイルをキャッシュする"
+cacheRemoteSensitiveFilesDescription: "この設定を無効にすると、リモートのNSFWファイルだけはキャッシュせず直リンクするようになります。"
 flagAsBot: "Botとして設定"
 flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。"
 flagAsCat: "にゃああああああああああああああ！！！！！！！！！！！！"

packages/backend/migration/1689102832143-nsfw-cache.js

@@ -0,0 +1,11 @@
+export class NsfwCache1689102832143 {
+	name = 'NsfwCache1689102832143'
+
+	async up(queryRunner) {
+			await queryRunner.query(`ALTER TABLE "meta" ADD "cacheRemoteSensitiveFiles" boolean NOT NULL DEFAULT true`);
+	}
+
+	async down(queryRunner) {
+			await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "cacheRemoteSensitiveFiles"`);
+	}
+}

packages/backend/src/core/activitypub/models/ApImageService.ts

@@ -1,7 +1,6 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { DI } from '@/di-symbols.js';
 import type { DriveFilesRepository } from '@/models/index.js';
-import type { Config } from '@/config.js';
 import type { RemoteUser } from '@/models/entities/User.js';
 import type { DriveFile } from '@/models/entities/DriveFile.js';
 import { MetaService } from '@/core/MetaService.js';
@@ -20,9 +19,6 @@ export class ApImageService {
 	private logger: Logger;
 
 	constructor(
-		@Inject(DI.config)
-		private config: Config,
-
 		@Inject(DI.driveFilesRepository)
 		private driveFilesRepository: DriveFilesRepository,
 
@@ -47,7 +43,7 @@ export class ApImageService {
 		const image = await this.apResolverService.createResolver().resolve(value);
 
 		if (image.url == null) {
-			throw new Error('invalid image: url not privided');
+			throw new Error('invalid image: url not provided');
 		}
 
 		if (typeof image.url !== 'string') {
@@ -62,12 +58,17 @@ export class ApImageService {
 
 		const instance = await this.metaService.fetch();
 
+		// Cache if remote file cache is on AND either
+		// 1. remote sensitive file is also on
+		// 2. or the image is not sensitive
+		const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive);
+
 		const file = await this.driveService.uploadFromUrl({
 			url: image.url,
 			user: actor,
 			uri: image.url,
 			sensitive: image.sensitive,
-			isLink: !instance.cacheRemoteFiles,
+			isLink: !shouldBeCached,
 			comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH),
 		});
 		if (!file.isLink || file.url === image.url) return file;

packages/backend/src/models/entities/Meta.ts

@@ -1,7 +1,6 @@
 import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
 import { id } from '../id.js';
 import { User } from './User.js';
-import type { Clip } from './Clip.js';
 
 @Entity()
 export class Meta {
@@ -126,6 +125,11 @@ export class Meta {
 	})
 	public cacheRemoteFiles: boolean;
 
+	@Column('boolean', {
+		default: true,
+	})
+	public cacheRemoteSensitiveFiles: boolean;
+
 	@Column({
 		...id(),
 		nullable: true,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -1,5 +1,4 @@
 import { Inject, Injectable } from '@nestjs/common';
-import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { MetaService } from '@/core/MetaService.js';
 import type { Config } from '@/config.js';
@@ -20,6 +19,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			cacheRemoteSensitiveFiles: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
 			emailRequiredForSignup: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -332,6 +335,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 				enableServiceWorker: instance.enableServiceWorker,
 				translatorAvailable: instance.deeplAuthKey != null,
 				cacheRemoteFiles: instance.cacheRemoteFiles,
+				cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles,
 				pinnedUsers: instance.pinnedUsers,
 				hiddenTags: instance.hiddenTags,
 				blockedHosts: instance.blockedHosts,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -43,6 +43,7 @@ export const paramDef = {
 		defaultLightTheme: { type: 'string', nullable: true },
 		defaultDarkTheme: { type: 'string', nullable: true },
 		cacheRemoteFiles: { type: 'boolean' },
+		cacheRemoteSensitiveFiles: { type: 'boolean' },
 		emailRequiredForSignup: { type: 'boolean' },
 		enableHcaptcha: { type: 'boolean' },
 		hcaptchaSiteKey: { type: 'string', nullable: true },
@@ -193,6 +194,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 				set.cacheRemoteFiles = ps.cacheRemoteFiles;
 			}
 
+			if (ps.cacheRemoteSensitiveFiles !== undefined) {
+				set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles;
+			}
+
 			if (ps.emailRequiredForSignup !== undefined) {
 				set.emailRequiredForSignup = ps.emailRequiredForSignup;
 			}

packages/backend/src/server/api/endpoints/meta.ts

@@ -83,6 +83,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			cacheRemoteSensitiveFiles: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
 			emailRequiredForSignup: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -272,7 +276,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 						.orWhere('ads.dayOfWeek = 0');
 				}))
 				.getMany();
-		
+
 			const response: any = {
 				maintainerName: instance.maintainerName,
 				maintainerEmail: instance.maintainerEmail,
@@ -329,6 +333,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 
 				...(ps.detail ? {
 					cacheRemoteFiles: instance.cacheRemoteFiles,
+					cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles,
 					requireSetup: (await this.usersRepository.countBy({
 						host: IsNull(),
 					})) === 0,

packages/backend/test/unit/activitypub.ts

@@ -4,6 +4,7 @@ import * as assert from 'assert';
 import { Test } from '@nestjs/testing';
 import { jest } from '@jest/globals';
 
+import { ApImageService } from '@/core/activitypub/models/ApImageService.js';
 import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
 import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
 import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
@@ -11,9 +12,12 @@ import { GlobalModule } from '@/GlobalModule.js';
 import { CoreModule } from '@/core/CoreModule.js';
 import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
 import { LoggerService } from '@/core/LoggerService.js';
-import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js';
+import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js';
-import { Note } from '@/models/index.js';
+import { Meta, Note } from '@/models/index.js';
 import { secureRndstr } from '@/misc/secure-rndstr.js';
+import { DownloadService } from '@/core/DownloadService.js';
+import { MetaService } from '@/core/MetaService.js';
+import type { RemoteUser } from '@/models/entities/User.js';
 import { MockResolver } from '../misc/mock-resolver.js';
 
 const host = 'https://host1.test';
@@ -63,16 +67,47 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe
 	};
 }
 
+async function createRandomRemoteUser(
+	resolver: MockResolver,
+	personService: ApPersonService,
+): Promise<RemoteUser> {
+	const actor = createRandomActor();
+	resolver.register(actor.id, actor);
+
+	return await personService.createPerson(actor.id, resolver);
+}
+
 describe('ActivityPub', () => {
+	let imageService: ApImageService;
 	let noteService: ApNoteService;
 	let personService: ApPersonService;
 	let rendererService: ApRendererService;
 	let resolver: MockResolver;
 
+	const metaInitial = {
+		cacheRemoteFiles: true,
+		cacheRemoteSensitiveFiles: true,
+		blockedHosts: [] as string[],
+		sensitiveWords: [] as string[],
+	} as Meta;
+	let meta = metaInitial;
+
 	beforeAll(async () => {
 		const app = await Test.createTestingModule({
 			imports: [GlobalModule, CoreModule],
-		}).compile();
+		})
+			.overrideProvider(DownloadService).useValue({
+				async downloadUrl(): Promise<{ filename: string }> {
+					return {
+						filename: 'dummy.tmp',
+					};
+				},
+			})
+			.overrideProvider(MetaService).useValue({
+				async fetch(): Promise<Meta> {
+					return meta;
+				},
+			}).compile();
 
 		await app.init();
 		app.enableShutdownHooks();
@@ -80,6 +115,7 @@ describe('ActivityPub', () => {
 		noteService = app.get<ApNoteService>(ApNoteService);
 		personService = app.get<ApPersonService>(ApPersonService);
 		rendererService = app.get<ApRendererService>(ApRendererService);
+		imageService = app.get<ApImageService>(ApImageService);
 		resolver = new MockResolver(await app.resolve<LoggerService>(LoggerService));
 
 		// Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error
@@ -219,4 +255,91 @@ describe('ActivityPub', () => {
 			assert.strictEqual(note.uri, actor2Note.id);
 		});
 	});
+
+	describe('Images', () => {
+		test('Create images', async () => {
+			const imageObject: IApDocument = {
+				type: 'Document',
+				mediaType: 'image/png',
+				url: 'http://host1.test/foo.png',
+				name: '',
+			};
+			const driveFile = await imageService.createImage(
+				await createRandomRemoteUser(resolver, personService),
+				imageObject,
+			);
+			assert.ok(!driveFile.isLink);
+
+			const sensitiveImageObject: IApDocument = {
+				type: 'Document',
+				mediaType: 'image/png',
+				url: 'http://host1.test/bar.png',
+				name: '',
+				sensitive: true,
+			};
+			const sensitiveDriveFile = await imageService.createImage(
+				await createRandomRemoteUser(resolver, personService),
+				sensitiveImageObject,
+			);
+			assert.ok(!sensitiveDriveFile.isLink);
+		});
+
+		test('cacheRemoteFiles=false disables caching', async () => {
+			meta = { ...metaInitial, cacheRemoteFiles: false };
+
+			const imageObject: IApDocument = {
+				type: 'Document',
+				mediaType: 'image/png',
+				url: 'http://host1.test/foo.png',
+				name: '',
+			};
+			const driveFile = await imageService.createImage(
+				await createRandomRemoteUser(resolver, personService),
+				imageObject,
+			);
+			assert.ok(driveFile.isLink);
+
+			const sensitiveImageObject: IApDocument = {
+				type: 'Document',
+				mediaType: 'image/png',
+				url: 'http://host1.test/bar.png',
+				name: '',
+				sensitive: true,
+			};
+			const sensitiveDriveFile = await imageService.createImage(
+				await createRandomRemoteUser(resolver, personService),
+				sensitiveImageObject,
+			);
+			assert.ok(sensitiveDriveFile.isLink);
+		});
+
+		test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => {
+			meta = { ...metaInitial, cacheRemoteSensitiveFiles: false };
+
+			const imageObject: IApDocument = {
+				type: 'Document',
+				mediaType: 'image/png',
+				url: 'http://host1.test/foo.png',
+				name: '',
+			};
+			const driveFile = await imageService.createImage(
+				await createRandomRemoteUser(resolver, personService),
+				imageObject,
+			);
+			assert.ok(!driveFile.isLink);
+
+			const sensitiveImageObject: IApDocument = {
+				type: 'Document',
+				mediaType: 'image/png',
+				url: 'http://host1.test/bar.png',
+				name: '',
+				sensitive: true,
+			};
+			const sensitiveDriveFile = await imageService.createImage(
+				await createRandomRemoteUser(resolver, personService),
+				sensitiveImageObject,
+			);
+			assert.ok(sensitiveDriveFile.isLink);
+		});
+	});
 });

packages/frontend/src/pages/admin/settings.vue

@@ -37,6 +37,13 @@
 								<template #label>{{ i18n.ts.cacheRemoteFiles }}</template>
 								<template #caption>{{ i18n.ts.cacheRemoteFilesDescription }}</template>
 							</MkSwitch>
+
+							<template v-if="cacheRemoteFiles">
+								<MkSwitch v-model="cacheRemoteSensitiveFiles">
+									<template #label>{{ i18n.ts.cacheRemoteSensitiveFiles }}</template>
+									<template #caption>{{ i18n.ts.cacheRemoteSensitiveFilesDescription }}</template>
+								</MkSwitch>
+							</template>
 						</div>
 					</FormSection>
 
@@ -104,7 +111,6 @@ import { fetchInstance } from '@/instance';
 import { i18n } from '@/i18n';
 import { definePageMetadata } from '@/scripts/page-metadata';
 import MkButton from '@/components/MkButton.vue';
-import MkColorInput from '@/components/MkColorInput.vue';
 
 let name: string | null = $ref(null);
 let description: string | null = $ref(null);
@@ -112,13 +118,14 @@ let maintainerName: string | null = $ref(null);
 let maintainerEmail: string | null = $ref(null);
 let pinnedUsers: string = $ref('');
 let cacheRemoteFiles: boolean = $ref(false);
+let cacheRemoteSensitiveFiles: boolean = $ref(false);
 let enableServiceWorker: boolean = $ref(false);
 let swPublicKey: any = $ref(null);
 let swPrivateKey: any = $ref(null);
 let deeplAuthKey: string = $ref('');
 let deeplIsPro: boolean = $ref(false);
 
-async function init() {
+async function init(): Promise<void> {
 	const meta = await os.api('admin/meta');
 	name = meta.name;
 	description = meta.description;
@@ -126,6 +133,7 @@ async function init() {
 	maintainerEmail = meta.maintainerEmail;
 	pinnedUsers = meta.pinnedUsers.join('\n');
 	cacheRemoteFiles = meta.cacheRemoteFiles;
+	cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles;
 	enableServiceWorker = meta.enableServiceWorker;
 	swPublicKey = meta.swPublickey;
 	swPrivateKey = meta.swPrivateKey;
@@ -133,7 +141,7 @@ async function init() {
 	deeplIsPro = meta.deeplIsPro;
 }
 
-function save() {
+function save(): void {
 	os.apiWithDialog('admin/update-meta', {
 		name,
 		description,
@@ -141,6 +149,7 @@ function save() {
 		maintainerEmail,
 		pinnedUsers: pinnedUsers.split('\n'),
 		cacheRemoteFiles,
+		cacheRemoteSensitiveFiles,
 		enableServiceWorker,
 		swPublicKey,
 		swPrivateKey,

packages/misskey-js/etc/misskey-js.api.md

@@ -271,6 +271,7 @@ type DetailedInstanceMetadata = LiteInstanceMetadata & {
     pinnedPages: string[];
     pinnedClipId: string | null;
     cacheRemoteFiles: boolean;
+    cacheRemoteSensitiveFiles: boolean;
     requireSetup: boolean;
     proxyAccountName: string | null;
     features: Record<string, any>;
@@ -327,6 +328,10 @@ export type Endpoints = {
         req: TODO;
         res: TODO;
     };
+    'admin/meta': {
+        req: TODO;
+        res: TODO;
+    };
     'admin/reset-password': {
         req: TODO;
         res: TODO;
@@ -2756,7 +2761,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u
 //
 // src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts
 // src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts
-// src/api.types.ts:620:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts
+// src/api.types.ts:621:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts
 // src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts
 
 // (No @packageDocumentation comment for this package)

packages/misskey-js/src/api.types.ts

@@ -20,6 +20,7 @@ export type Endpoints = {
 	'admin/get-table-stats': { req: TODO; res: TODO; };
 	'admin/invite': { req: TODO; res: TODO; };
 	'admin/logs': { req: TODO; res: TODO; };
+	'admin/meta': { req: TODO; res: TODO; };
 	'admin/reset-password': { req: TODO; res: TODO; };
 	'admin/resolve-abuse-user-report': { req: TODO; res: TODO; };
 	'admin/resync-chart': { req: TODO; res: TODO; };

packages/misskey-js/src/entities.ts

@@ -338,6 +338,7 @@ export type DetailedInstanceMetadata = LiteInstanceMetadata & {
 	pinnedPages: string[];
 	pinnedClipId: string | null;
 	cacheRemoteFiles: boolean;
+	cacheRemoteSensitiveFiles: boolean;
 	requireSetup: boolean;
 	proxyAccountName: string | null;
 	features: Record<string, any>;