monkee.ch/monkeeShark
1
0
Fork 0
Code Pull requests Activity

feat(backend): allow disabling cache for sensitive files

Browse source
This commit is contained in:
Kagami Sascha Rosylight 2023-07-12 02:13:27 +02:00
parent 1a096c557e
commit c277934328
13 changed files with 189 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
locales/index.d.ts vendored
View file

@ -158,6 +158,8 @@ export interface Locale {
"settingGuide": string;
"cacheRemoteFiles": string;
"cacheRemoteFilesDescription": string;
"cacheRemoteSensitiveFiles": string;
"cacheRemoteSensitiveFilesDescription": string;
"flagAsBot": string;
"flagAsBotDescription": string;
"flagAsCat": string;

2
locales/ja-JP.yml
View file

@ -155,6 +155,8 @@ addEmoji: "絵文字を追加"
settingGuide: "おすすめ設定"
cacheRemoteFiles: "リモートのファイルをキャッシュする"
cacheRemoteFilesDescription: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。サーバーのストレージを節約できますが、サムネイルが生成されないので通信量が増加します。"
cacheRemoteSensitiveFiles: "リモートのNSFWファイルをキャッシュする"
cacheRemoteSensitiveFilesDescription: "この設定を無効にすると、リモートのNSFWファイルだけはキャッシュせず直リンクするようになります。"
flagAsBot: "Botとして設定"
flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。"
flagAsCat: "にゃああああああああああああああ！！！！！！！！！！！！"

11
packages/backend/migration/1689102832143-nsfw-cache.js Normal file
View file

@ -0,0 +1,11 @@
export class NsfwCache1689102832143 {
name = 'NsfwCache1689102832143'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "cacheRemoteSensitiveFiles" boolean NOT NULL DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "cacheRemoteSensitiveFiles"`);
}
}

13
packages/backend/src/core/activitypub/models/ApImageService.ts
View file

@ -1,7 +1,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type { RemoteUser } from '@/models/entities/User.js';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import { MetaService } from '@/core/MetaService.js';
@ -20,9 +19,6 @@ export class ApImageService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@ -47,7 +43,7 @@ export class ApImageService {
const image = await this.apResolverService.createResolver().resolve(value);
if (image.url == null) {
throw new Error('invalid image: url not privided');
throw new Error('invalid image: url not provided');
}
if (typeof image.url !== 'string') {
@ -62,12 +58,17 @@ export class ApImageService {
const instance = await this.metaService.fetch();
// Cache if remote file cache is on AND either
// 1. remote sensitive file is also on
// 2. or the image is not sensitive
const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive);
const file = await this.driveService.uploadFromUrl({
url: image.url,
user: actor,
uri: image.url,
sensitive: image.sensitive,
isLink: !instance.cacheRemoteFiles,
isLink: !shouldBeCached,
comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH),
});
if (!file.isLink || file.url === image.url) return file;

6
packages/backend/src/models/entities/Meta.ts
View file

@ -1,7 +1,6 @@
import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
import { id } from '../id.js';
import { User } from './User.js';
import type { Clip } from './Clip.js';
@Entity()
export class Meta {
@ -126,6 +125,11 @@ export class Meta {
})
public cacheRemoteFiles: boolean;
@Column('boolean', {
default: true,
})
public cacheRemoteSensitiveFiles: boolean;
@Column({
...id(),
nullable: true,

6
packages/backend/src/server/api/endpoints/admin/meta.ts
View file

@ -1,5 +1,4 @@
import { Inject, Injectable } from '@nestjs/common';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { MetaService } from '@/core/MetaService.js';
import type { Config } from '@/config.js';
@ -20,6 +19,10 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
cacheRemoteSensitiveFiles: {
type: 'boolean',
optional: false, nullable: false,
},
emailRequiredForSignup: {
type: 'boolean',
optional: false, nullable: false,
@ -332,6 +335,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
enableServiceWorker: instance.enableServiceWorker,
translatorAvailable: instance.deeplAuthKey != null,
cacheRemoteFiles: instance.cacheRemoteFiles,
cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles,
pinnedUsers: instance.pinnedUsers,
hiddenTags: instance.hiddenTags,
blockedHosts: instance.blockedHosts,

5
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View file

@ -43,6 +43,7 @@ export const paramDef = {
defaultLightTheme: { type: 'string', nullable: true },
defaultDarkTheme: { type: 'string', nullable: true },
cacheRemoteFiles: { type: 'boolean' },
cacheRemoteSensitiveFiles: { type: 'boolean' },
emailRequiredForSignup: { type: 'boolean' },
enableHcaptcha: { type: 'boolean' },
hcaptchaSiteKey: { type: 'string', nullable: true },
@ -193,6 +194,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
set.cacheRemoteFiles = ps.cacheRemoteFiles;
}
if (ps.cacheRemoteSensitiveFiles !== undefined) {
set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles;
}
if (ps.emailRequiredForSignup !== undefined) {
set.emailRequiredForSignup = ps.emailRequiredForSignup;
}

7
packages/backend/src/server/api/endpoints/meta.ts
View file

@ -83,6 +83,10 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
cacheRemoteSensitiveFiles: {
type: 'boolean',
optional: false, nullable: false,
},
emailRequiredForSignup: {
type: 'boolean',
optional: false, nullable: false,
@ -272,7 +276,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
.orWhere('ads.dayOfWeek = 0');
}))
.getMany();
const response: any = {
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
@ -329,6 +333,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
...(ps.detail ? {
cacheRemoteFiles: instance.cacheRemoteFiles,
cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles,
requireSetup: (await this.usersRepository.countBy({
host: IsNull(),
})) === 0,

129
packages/backend/test/unit/activitypub.ts
View file

@ -4,6 +4,7 @@ import * as assert from 'assert';
import { Test } from '@nestjs/testing';
import { jest } from '@jest/globals';
import { ApImageService } from '@/core/activitypub/models/ApImageService.js';
import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
@ -11,9 +12,12 @@ import { GlobalModule } from '@/GlobalModule.js';
import { CoreModule } from '@/core/CoreModule.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js';
import { Note } from '@/models/index.js';
import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js';
import { Meta, Note } from '@/models/index.js';
import { secureRndstr } from '@/misc/secure-rndstr.js';
import { DownloadService } from '@/core/DownloadService.js';
import { MetaService } from '@/core/MetaService.js';
import type { RemoteUser } from '@/models/entities/User.js';
import { MockResolver } from '../misc/mock-resolver.js';
const host = 'https://host1.test';
@ -63,16 +67,47 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe
};
}
async function createRandomRemoteUser(
resolver: MockResolver,
personService: ApPersonService,
): Promise<RemoteUser> {
const actor = createRandomActor();
resolver.register(actor.id, actor);
return await personService.createPerson(actor.id, resolver);
}
describe('ActivityPub', () => {
let imageService: ApImageService;
let noteService: ApNoteService;
let personService: ApPersonService;
let rendererService: ApRendererService;
let resolver: MockResolver;
const metaInitial = {
cacheRemoteFiles: true,
cacheRemoteSensitiveFiles: true,
blockedHosts: [] as string[],
sensitiveWords: [] as string[],
} as Meta;
let meta = metaInitial;
beforeAll(async () => {
const app = await Test.createTestingModule({
imports: [GlobalModule, CoreModule],
}).compile();
})
.overrideProvider(DownloadService).useValue({
async downloadUrl(): Promise<{ filename: string }> {
return {
filename: 'dummy.tmp',
};
},
})
.overrideProvider(MetaService).useValue({
async fetch(): Promise<Meta> {
return meta;
},
}).compile();
await app.init();
app.enableShutdownHooks();
@ -80,6 +115,7 @@ describe('ActivityPub', () => {
noteService = app.get<ApNoteService>(ApNoteService);
personService = app.get<ApPersonService>(ApPersonService);
rendererService = app.get<ApRendererService>(ApRendererService);
imageService = app.get<ApImageService>(ApImageService);
resolver = new MockResolver(await app.resolve<LoggerService>(LoggerService));
// Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error
@ -219,4 +255,91 @@ describe('ActivityPub', () => {
assert.strictEqual(note.uri, actor2Note.id);
});
});
describe('Images', () => {
test('Create images', async () => {
const imageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/foo.png',
name: '',
};
const driveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
imageObject,
);
assert.ok(!driveFile.isLink);
const sensitiveImageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/bar.png',
name: '',
sensitive: true,
};
const sensitiveDriveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
sensitiveImageObject,
);
assert.ok(!sensitiveDriveFile.isLink);
});
test('cacheRemoteFiles=false disables caching', async () => {
meta = { ...metaInitial, cacheRemoteFiles: false };
const imageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/foo.png',
name: '',
};
const driveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
imageObject,
);
assert.ok(driveFile.isLink);
const sensitiveImageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/bar.png',
name: '',
sensitive: true,
};
const sensitiveDriveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
sensitiveImageObject,
);
assert.ok(sensitiveDriveFile.isLink);
});
test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => {
meta = { ...metaInitial, cacheRemoteSensitiveFiles: false };
const imageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/foo.png',
name: '',
};
const driveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
imageObject,
);
assert.ok(!driveFile.isLink);
const sensitiveImageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/bar.png',
name: '',
sensitive: true,
};
const sensitiveDriveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
sensitiveImageObject,
);
assert.ok(sensitiveDriveFile.isLink);
});
});
});

15
packages/frontend/src/pages/admin/settings.vue
View file

@ -37,6 +37,13 @@
<template #label>{{ i18n.ts.cacheRemoteFiles }}</template>
<template #caption>{{ i18n.ts.cacheRemoteFilesDescription }}</template>
</MkSwitch>
<template v-if="cacheRemoteFiles">
<MkSwitch v-model="cacheRemoteSensitiveFiles">
<template #label>{{ i18n.ts.cacheRemoteSensitiveFiles }}</template>
<template #caption>{{ i18n.ts.cacheRemoteSensitiveFilesDescription }}</template>
</MkSwitch>
</template>
</div>
</FormSection>
@ -104,7 +111,6 @@ import { fetchInstance } from '@/instance';
import { i18n } from '@/i18n';
import { definePageMetadata } from '@/scripts/page-metadata';
import MkButton from '@/components/MkButton.vue';
import MkColorInput from '@/components/MkColorInput.vue';
let name: string | null = $ref(null);
let description: string | null = $ref(null);
@ -112,13 +118,14 @@ let maintainerName: string | null = $ref(null);
let maintainerEmail: string | null = $ref(null);
let pinnedUsers: string = $ref('');
let cacheRemoteFiles: boolean = $ref(false);
let cacheRemoteSensitiveFiles: boolean = $ref(false);
let enableServiceWorker: boolean = $ref(false);
let swPublicKey: any = $ref(null);
let swPrivateKey: any = $ref(null);
let deeplAuthKey: string = $ref('');
let deeplIsPro: boolean = $ref(false);
async function init() {
async function init(): Promise<void> {
const meta = await os.api('admin/meta');
name = meta.name;
description = meta.description;
@ -126,6 +133,7 @@ async function init() {
maintainerEmail = meta.maintainerEmail;
pinnedUsers = meta.pinnedUsers.join('\n');
cacheRemoteFiles = meta.cacheRemoteFiles;
cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles;
enableServiceWorker = meta.enableServiceWorker;
swPublicKey = meta.swPublickey;
swPrivateKey = meta.swPrivateKey;
@ -133,7 +141,7 @@ async function init() {
deeplIsPro = meta.deeplIsPro;
}
function save() {
function save(): void {
os.apiWithDialog('admin/update-meta', {
name,
description,
@ -141,6 +149,7 @@ function save() {
maintainerEmail,
pinnedUsers: pinnedUsers.split('\n'),
cacheRemoteFiles,
cacheRemoteSensitiveFiles,
enableServiceWorker,
swPublicKey,
swPrivateKey,

7
packages/misskey-js/etc/misskey-js.api.md
View file

@ -271,6 +271,7 @@ type DetailedInstanceMetadata = LiteInstanceMetadata & {
pinnedPages: string[];
pinnedClipId: string | null;
cacheRemoteFiles: boolean;
cacheRemoteSensitiveFiles: boolean;
requireSetup: boolean;
proxyAccountName: string | null;
features: Record<string, any>;
@ -327,6 +328,10 @@ export type Endpoints = {
req: TODO;
res: TODO;
};
'admin/meta': {
req: TODO;
res: TODO;
};
'admin/reset-password': {
req: TODO;
res: TODO;
@ -2756,7 +2761,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u
//
// src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts
// src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts
// src/api.types.ts:620:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts
// src/api.types.ts:621:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts
// src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts
// (No @packageDocumentation comment for this package)

1
packages/misskey-js/src/api.types.ts
View file

@ -20,6 +20,7 @@ export type Endpoints = {
'admin/get-table-stats': { req: TODO; res: TODO; };
'admin/invite': { req: TODO; res: TODO; };
'admin/logs': { req: TODO; res: TODO; };
'admin/meta': { req: TODO; res: TODO; };
'admin/reset-password': { req: TODO; res: TODO; };
'admin/resolve-abuse-user-report': { req: TODO; res: TODO; };
'admin/resync-chart': { req: TODO; res: TODO; };

1
packages/misskey-js/src/entities.ts
View file

@ -338,6 +338,7 @@ export type DetailedInstanceMetadata = LiteInstanceMetadata & {
pinnedPages: string[];
pinnedClipId: string | null;
cacheRemoteFiles: boolean;
cacheRemoteSensitiveFiles: boolean;
requireSetup: boolean;
proxyAccountName: string | null;
features: Record<string, any>;