monkee
/
Sharkey
mirror of https://git.joinsharkey.org/Sharkey/Sharkey.git
1
0
Fork 1
Code Issues Packages Projects Wiki Activity

enhance(backend): improve featured system

This commit is contained in:
syuilo 2023-10-06 14:24:25 +09:00
parent e4dcab8671
commit dab205edb8
10 changed files with 208 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -34,7 +34,8 @@
- Fix: リアクションしたユーザ一覧のUIが稀に左上に残ってしまう不具合を修正
### Server
- Enhance: タイムライン取得時のパフォーマンスを改善
- Enhance: タイムライン取得時のパフォーマンスを大幅に向上
- Enhance: ハイライト取得時のパフォーマンスを大幅に向上
## 2023.9.3
### General

18
packages/backend/migration/1696569742153-clean-up.js Normal file
View File

@ -0,0 +1,18 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CleanUp1696569742153 {
name = 'CleanUp1696569742153'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_01f4581f114e0ebd2bbb876f0b"`);
await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "score"`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" ADD "score" integer NOT NULL DEFAULT '0'`);
await queryRunner.query(`CREATE INDEX "IDX_01f4581f114e0ebd2bbb876f0b" ON "note_reaction" ("createdAt") `);
}
}

6
packages/backend/src/core/CoreModule.ts
View File

@ -60,6 +60,7 @@ import { UtilityService } from './UtilityService.js';
import { FileInfoService } from './FileInfoService.js';
import { SearchService } from './SearchService.js';
import { ClipService } from './ClipService.js';
import { FeaturedService } from './FeaturedService.js';
import { ChartLoggerService } from './chart/ChartLoggerService.js';
import FederationChart from './chart/charts/federation.js';
import NotesChart from './chart/charts/notes.js';
@ -187,6 +188,7 @@ const $UtilityService: Provider = { provide: 'UtilityService', useExisting: Util
const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService };
const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService };
const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService };
const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService };
const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService };
const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart };
@ -318,6 +320,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
FileInfoService,
SearchService,
ClipService,
FeaturedService,
ChartLoggerService,
FederationChart,
NotesChart,
@ -442,6 +445,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$FileInfoService,
$SearchService,
$ClipService,
$FeaturedService,
$ChartLoggerService,
$FederationChart,
$NotesChart,
@ -567,6 +571,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
FileInfoService,
SearchService,
ClipService,
FeaturedService,
FederationChart,
NotesChart,
UsersChart,
@ -690,6 +695,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$FileInfoService,
$SearchService,
$ClipService,
$FeaturedService,
$FederationChart,
$NotesChart,
$UsersChart,

126
packages/backend/src/core/FeaturedService.ts Normal file
View File

@ -0,0 +1,126 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import type { MiNote } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
@Injectable()
export class FeaturedService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
) {
}
@bindThis
private getCurrentPerUserFriendRankingWindow(): number {
const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime();
return Math.floor(passed / (1000 * 60 * 60 * 24 * 7)); // 1週間ごと
}
@bindThis
private getCurrentGlobalNotesRankingWindow(): number {
const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime();
return Math.floor(passed / (1000 * 60 * 60 * 24 * 3)); // 3日ごと
}
@bindThis
public async updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> {
// TODO: フォロワー数の多い人が常にランキング上位になるのを防ぎたい
const currentWindow = this.getCurrentGlobalNotesRankingWindow();
const redisTransaction = this.redisClient.multi();
redisTransaction.zincrby(
`featuredGlobalNotesRanking:${currentWindow}`,
score.toString(),
noteId);
redisTransaction.expire(
`featuredGlobalNotesRanking:${currentWindow}`,
60 * 60 * 24 * 9, // 9日間保持
'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
await redisTransaction.exec();
}
@bindThis
public async updateInChannelNotesRanking(noteId: MiNote['id'], channelId: MiNote['channelId'], score = 1): Promise<void> {
const currentWindow = this.getCurrentGlobalNotesRankingWindow();
const redisTransaction = this.redisClient.multi();
redisTransaction.zincrby(
`featuredInChannelNotesRanking:${channelId}:${currentWindow}`,
score.toString(),
noteId);
redisTransaction.expire(
`featuredInChannelNotesRanking:${channelId}:${currentWindow}`,
60 * 60 * 24 * 9, // 9日間保持
'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
await redisTransaction.exec();
}
@bindThis
public async getGlobalNotesRanking(limit: number): Promise<MiNote['id'][]> {
const currentWindow = this.getCurrentGlobalNotesRankingWindow();
const previousWindow = currentWindow - 1;
const [currentRankingResult, previousRankingResult] = await Promise.all([
this.redisClient.zrange(
`featuredGlobalNotesRanking:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'),
this.redisClient.zrange(
`featuredGlobalNotesRanking:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'),
]);
const ranking = new Map<MiNote['id'], number>();
for (let i = 0; i < currentRankingResult.length; i += 2) {
const noteId = currentRankingResult[i];
const score = parseInt(currentRankingResult[i + 1], 10);
ranking.set(noteId, score);
}
for (let i = 0; i < previousRankingResult.length; i += 2) {
const noteId = previousRankingResult[i];
const score = parseInt(previousRankingResult[i + 1], 10);
const exist = ranking.get(noteId);
if (exist != null) {
ranking.set(noteId, (exist + score) / 2);
} else {
ranking.set(noteId, score);
}
}
return Array.from(ranking.keys());
}
@bindThis
public async getInChannelNotesRanking(channelId: MiNote['channelId'], limit: number): Promise<MiNote['id'][]> {
const currentWindow = this.getCurrentGlobalNotesRankingWindow();
const previousWindow = currentWindow - 1;
const [currentRankingResult, previousRankingResult] = await Promise.all([
this.redisClient.zrange(
`featuredInChannelNotesRanking:${channelId}:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'),
this.redisClient.zrange(
`featuredInChannelNotesRanking:${channelId}:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'),
]);
const ranking = new Map<MiNote['id'], number>();
for (let i = 0; i < currentRankingResult.length; i += 2) {
const noteId = currentRankingResult[i];
const score = parseInt(currentRankingResult[i + 1], 10);
ranking.set(noteId, score);
}
for (let i = 0; i < previousRankingResult.length; i += 2) {
const noteId = previousRankingResult[i];
const score = parseInt(previousRankingResult[i + 1], 10);
const exist = ranking.get(noteId);
if (exist != null) {
ranking.set(noteId, (exist + score) / 2);
} else {
ranking.set(noteId, score);
}
}
return Array.from(ranking.keys());
}
}

12
packages/backend/src/core/NoteCreateService.ts
View File

@ -53,6 +53,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { RoleService } from '@/core/RoleService.js';
import { MetaService } from '@/core/MetaService.js';
import { SearchService } from '@/core/SearchService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -200,6 +201,7 @@ export class NoteCreateService implements OnApplicationShutdown {
private hashtagService: HashtagService,
private antennaService: AntennaService,
private webhookService: WebhookService,
private featuredService: FeaturedService,
private remoteUserResolveService: RemoteUserResolveService,
private apDeliverManagerService: ApDeliverManagerService,
private apRendererService: ApRendererService,
@ -721,10 +723,18 @@ export class NoteCreateService implements OnApplicationShutdown {
this.notesRepository.createQueryBuilder().update()
.set({
renoteCount: () => '"renoteCount" + 1',
score: () => '"score" + 1',
})
.where('id = :id', { id: renote.id })
.execute();
// 30%の確率でハイライト用ランキング更新
if (Math.random() < 0.3) {
if (renote.channelId != null) {
this.featuredService.updateInChannelNotesRanking(renote.id, renote.channelId, 1);
} else if (renote.visibility === 'public' && renote.userHost == null) {
this.featuredService.updateGlobalNotesRanking(renote.id, 1);
}
}
}
@bindThis

1
packages/backend/src/core/NoteDeleteService.ts
View File

@ -67,7 +67,6 @@ export class NoteDeleteService {
// この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき
if (note.renoteId && (await this.noteEntityService.countSameRenotes(user.id, note.renoteId, note.id)) === 0) {
this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1);
if (!user.isBot) this.notesRepository.decrement({ id: note.renoteId }, 'score', 1);
}
if (note.replyId) {

18
packages/backend/src/core/ReactionService.ts
View File

@ -4,6 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
@ -26,6 +27,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
import { RoleService } from '@/core/RoleService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
const FALLBACK = '❤';
@ -66,6 +68,9 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
@Injectable()
export class ReactionService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -86,6 +91,7 @@ export class ReactionService {
private noteEntityService: NoteEntityService,
private userBlockingService: UserBlockingService,
private idService: IdService,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
private apRendererService: ApRendererService,
private apDeliverManagerService: ApDeliverManagerService,
@ -182,11 +188,19 @@ export class ReactionService {
await this.notesRepository.createQueryBuilder().update()
.set({
reactions: () => sql,
... (!user.isBot ? { score: () => '"score" + 1' } : {}),
})
.where('id = :id', { id: note.id })
.execute();
// 30%の確率でハイライト用ランキング更新
if (Math.random() < 0.3) {
if (note.channelId != null) {
this.featuredService.updateInChannelNotesRanking(note.id, note.channelId, 1);
} else if (note.visibility === 'public' && note.userHost == null) {
this.featuredService.updateGlobalNotesRanking(note.id, 1);
}
}
const meta = await this.metaService.fetch();
if (meta.enableChartsForRemoteUser || (user.host == null)) {
@ -275,8 +289,6 @@ export class ReactionService {
.where('id = :id', { id: note.id })
.execute();
if (!user.isBot) this.notesRepository.decrement({ id: note.id }, 'score', 1);
this.globalEventService.publishNoteStream(note.id, 'unreacted', {
reaction: this.decodeReaction(exist.reaction).reaction,
userId: user.id,

5
packages/backend/src/models/Note.ts
View File

@ -138,11 +138,6 @@ export class MiNote {
})
public url: string | null;
@Column('integer', {
default: 0, select: false,
})
public score: number;
@Index()
@Column({
...id(),

1
packages/backend/src/models/NoteReaction.ts
View File

@ -14,7 +14,6 @@ export class MiNoteReaction {
@PrimaryColumn(id())
public id: string;
@Index()
@Column('timestamp with time zone', {
comment: 'The created date of the NoteReaction.',
})

51
packages/backend/src/server/api/endpoints/notes/featured.ts
View File

@ -6,9 +6,9 @@
import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueryService } from '@/core/QueryService.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { DI } from '@/di-symbols.js';
import { FeaturedService } from '@/core/FeaturedService.js';
export const meta = {
tags: ['notes'],
@ -40,41 +40,50 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
private globalNotesRankingCache: string[] = [];
private globalNotesRankingCacheLastFetchedAt = 0;
constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private noteEntityService: NoteEntityService,
private queryService: QueryService,
private featuredService: FeaturedService,
) {
super(meta, paramDef, async (ps, me) => {
const day = 1000 * 60 * 60 * 24 * 3; // 3日前まで
let noteIds: string[];
if (ps.channelId) {
noteIds = await this.featuredService.getInChannelNotesRanking(ps.channelId, 50);
} else {
if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (Date.now() - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) {
noteIds = this.globalNotesRankingCache;
} else {
noteIds = await this.featuredService.getGlobalNotesRanking(100);
this.globalNotesRankingCache = noteIds;
this.globalNotesRankingCacheLastFetchedAt = Date.now();
}
}
if (noteIds.length === 0) {
return [];
}
noteIds.sort((a, b) => a > b ? -1 : 1);
noteIds.slice(ps.offset, ps.offset + ps.limit);
const query = this.notesRepository.createQueryBuilder('note')
.addSelect('note.score')
.where('note.userHost IS NULL')
.andWhere('note.score > 0')
.andWhere('note.createdAt > :date', { date: new Date(Date.now() - day) })
.andWhere('note.visibility = \'public\'')
.where('note.id IN (:...noteIds)', { noteIds: noteIds })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
if (ps.channelId) query.andWhere('note.channelId = :channelId', { channelId: ps.channelId });
const notes = await query.getMany();
notes.sort((a, b) => a.id > b.id ? -1 : 1);
if (me) this.queryService.generateMutedUserQuery(query, me);
if (me) this.queryService.generateBlockedUserQuery(query, me);
let notes = await query
.orderBy('note.score', 'DESC')
.limit(100)
.getMany();
notes.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
notes = notes.slice(ps.offset, ps.offset + ps.limit);
// TODO: ミュート等考慮
return await this.noteEntityService.packMany(notes, me);
});