Compare commits

...

27 commits

Author SHA1 Message Date
tamaina
a926e84b14 Merge branch 'develop' into fetch-outbox 2023-09-16 08:33:23 +00:00
tamaina
df636659e2 Merge branch 'develop' into fetch-outbox 2023-09-11 06:32:04 +00:00
tamaina
888cd2eb9f Merge branch 'develop' into fetch-outbox 2023-08-24 06:23:05 +00:00
Kagami Sascha Rosylight
ff4b3d2d9e Merge branch 'develop' into fetch-outbox 2023-08-06 00:04:26 +02:00
tamaina
60fd848182 configable 2023-07-15 14:24:05 +00:00
tamaina
76def0032e ? 2023-07-15 14:06:36 +00:00
tamaina
da0804eb17 fix type error 2023-07-15 14:05:05 +00:00
tamaina
6b26ce3768 Merge branch 'develop' into fetch-outbox 2023-07-15 14:04:54 +00:00
tamaina
5a0d7d41e6
Merge branch 'develop' into fetch-outbox 2023-07-14 13:36:04 +09:00
Kagami Sascha Rosylight
08e2b6ee32 perform only create activities 2023-07-09 23:34:07 +02:00
Kagami Sascha Rosylight
ca0c673b44 Update activitypub.ts 2023-07-09 22:39:02 +02:00
Kagami Sascha Rosylight
70bb9a4d1f Update activitypub.ts 2023-07-09 22:36:41 +02:00
Kagami Sascha Rosylight
b93046c071 Update activitypub.ts 2023-07-09 22:36:27 +02:00
Kagami Sascha Rosylight
f34f0dfcb6 Merge remote-tracking branch 'origin/develop' into fetch-outbox 2023-07-09 22:36:14 +02:00
Kagami Sascha Rosylight
26040c2bb0 Update activitypub.ts 2023-07-09 22:36:07 +02:00
Kagami Sascha Rosylight
bdbad4605b
Merge branch 'develop' into fetch-outbox 2023-07-09 15:04:12 +02:00
Kagami Sascha Rosylight
ec62fe02b1
Merge branch 'develop' into fetch-outbox 2023-07-09 00:08:54 +02:00
Kagami Sascha Rosylight
a74af07992 reformat 2023-07-08 15:48:10 +02:00
Kagami Sascha Rosylight
aa78c29e8c Merge remote-tracking branch 'origin/develop' into fetch-outbox 2023-07-08 15:47:00 +02:00
Kagami Sascha Rosylight
45d0b46e7a typo 2023-07-08 15:44:27 +02:00
Kagami Sascha Rosylight
6087d02047 adjust tests 2023-07-08 15:40:48 +02:00
Kagami Sascha Rosylight
7bf318ae98 remove extra resolving in tests 2023-07-08 15:13:23 +02:00
tamaina
71d74676f0
Merge branch 'develop' into fetch-outbox 2023-07-07 14:54:28 +09:00
tamaina
5077df2973 100 -> 15 2023-07-07 05:02:28 +00:00
Kagami Sascha Rosylight
a1388a8444 Update CHANGELOG.md 2023-07-06 03:39:16 +02:00
Kagami Sascha Rosylight
630e97bd06 Merge remote-tracking branch 'origin/develop' into fetch-outbox 2023-07-06 03:38:13 +02:00
Kagami Sascha Rosylight
2e1de4fca9 feat(backend): fetch the first page of outbox when resolving Person 2023-07-06 02:56:17 +02:00
9 changed files with 251 additions and 21 deletions

View file

@ -8,7 +8,7 @@
-
### Server
-
- 最初照会したユーザーの最新ノートを受け取るように
-->

View file

@ -187,6 +187,10 @@ id: "aidx"
# Sign to ActivityPub GET request (default: true)
signToActivityPubGet: true
# Limit of notes to fetch from outbox with remote user first fetched (default: 5)
# https://github.com/misskey-dev/misskey/pull/11130
outboxNotesFetchLimit: 5
#allowedPrivateNetworks: [
# '127.0.0.1/32'
#]

View file

@ -85,6 +85,7 @@ type Source = {
videoThumbnailGenerator?: string;
signToActivityPubGet?: boolean;
outboxNotesFetchLimit?: number;
perChannelMaxNoteCacheCount?: number;
perUserNotificationsMaxCount?: number;

View file

@ -27,7 +27,7 @@ import { QueueService } from '@/core/QueueService.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, AbuseUserReportsRepository, FollowRequestsRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import type { MiRemoteUser } from '@/models/entities/User.js';
import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isOrderedCollectionPage, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -87,11 +87,19 @@ export class ApInboxService {
}
@bindThis
public async performActivity(actor: MiRemoteUser, activity: IObject): Promise<void> {
if (isCollectionOrOrderedCollection(activity)) {
public async performActivity(actor: MiRemoteUser, activity: IObject, {
limit = Infinity,
allow = null as (string[] | null) } = {},
): Promise<void> {
if (isCollectionOrOrderedCollection(activity) || isOrderedCollectionPage(activity)) {
const resolver = this.apResolverService.createResolver();
for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems)) {
for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems).slice(0, limit)) {
const act = await resolver.resolve(item);
const type = getApType(act);
if (allow && !allow.includes(type)) {
this.logger.info(`skipping activity type: ${type}`);
continue;
}
try {
await this.performOneActivity(actor, act);
} catch (err) {
@ -367,7 +375,7 @@ export class ApInboxService {
});
if (isPost(object)) {
this.createNote(resolver, actor, object, false, activity);
await this.createNote(resolver, actor, object, false, activity);
} else {
this.logger.warn(`Unknown type: ${getApType(object)}`);
}

View file

@ -15,11 +15,11 @@ import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { isCollectionOrOrderedCollection } from './type.js';
import { isCollectionOrOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js';
import type { IObject, ICollection, IOrderedCollection, IOrderedCollectionPage } from './type.js';
export class Resolver {
private history: Set<string>;
@ -64,6 +64,18 @@ export class Resolver {
}
}
public async resolveOrderedCollectionPage(value: string | IObject): Promise<IOrderedCollectionPage> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
if (isOrderedCollectionPage(collection)) {
return collection;
} else {
throw new Error(`unrecognized collection type: ${collection.type}`);
}
}
@bindThis
public async resolve(value: string | IObject): Promise<IObject> {
if (typeof value !== 'string') {

View file

@ -38,7 +38,8 @@ import { MetaService } from '@/core/MetaService.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import type { AccountMoveService } from '@/core/AccountMoveService.js';
import { checkHttps } from '@/misc/check-https.js';
import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage, isPropertyValue } from '../type.js';
import { ApInboxService } from '../ApInboxService.js';
import { extractApHashtags } from './tag.js';
import type { OnModuleInit } from '@nestjs/common';
import type { ApNoteService } from './ApNoteService.js';
@ -68,6 +69,7 @@ export class ApPersonService implements OnModuleInit {
private apResolverService: ApResolverService;
private apNoteService: ApNoteService;
private apImageService: ApImageService;
private apInboxService: ApInboxService;
private apMfmService: ApMfmService;
private mfmService: MfmService;
private hashtagService: HashtagService;
@ -116,6 +118,7 @@ export class ApPersonService implements OnModuleInit {
this.apResolverService = this.moduleRef.get('ApResolverService');
this.apNoteService = this.moduleRef.get('ApNoteService');
this.apImageService = this.moduleRef.get('ApImageService');
this.apInboxService = this.moduleRef.get('ApInboxService');
this.apMfmService = this.moduleRef.get('ApMfmService');
this.mfmService = this.moduleRef.get('MfmService');
this.hashtagService = this.moduleRef.get('HashtagService');
@ -384,7 +387,10 @@ export class ApPersonService implements OnModuleInit {
}
//#endregion
await this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err));
await Promise.allSettled([
this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err)),
this.updateOutboxFirstPage(user, person.outbox, resolver).catch(err => this.logger.error(err)),
]);
return user;
}
@ -589,7 +595,7 @@ export class ApPersonService implements OnModuleInit {
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x)));
// Resolve and regist Notes
// Resolve and register Notes
const limit = promiseLimit<MiNote | null>(2);
const featuredNotes = await Promise.all(items
.filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも
@ -616,6 +622,35 @@ export class ApPersonService implements OnModuleInit {
});
}
/**
* Retrieve outbox from an actor object.
*
* This only retrieves the first page for now.
*/
public async updateOutboxFirstPage(user: RemoteUser, outbox: IActor['outbox'], resolver: Resolver): Promise<void> {
if (!this.config.outboxNotesFetchLimit) return;
// https://www.w3.org/TR/activitypub/#actor-objects
// Outbox is a required property for all actors
if (!outbox) {
throw new Error('No outbox property');
}
this.logger.info(`Fetching the outbox for ${user.uri}: ${outbox}`);
const collection = await resolver.resolveCollection(outbox);
if (!isOrderedCollection(collection)) {
throw new Error('Outbox must be an ordered collection');
}
const firstPage = collection.first ?
await resolver.resolveOrderedCollectionPage(collection.first) :
collection;
// Perform activity but only the first outboxNotesFetchLimit ones with `type: Create`
await this.apInboxService.performActivity(user, firstPage, { limit: this.config.outboxNotesFetchLimit, allow: ['Create'] });
}
/**
*
* @param src updatePerson後である必要があるupdatePersonで呼ばれる前提

View file

@ -92,16 +92,37 @@ export interface IActivity extends IObject {
};
}
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection
export interface ICollection extends IObject {
type: 'Collection';
totalItems: number;
current?: ICollectionPage | string;
first?: ICollectionPage | string;
last?: ICollectionPage | string;
items: ApObject;
}
export interface IOrderedCollection extends IObject {
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection
export interface IOrderedCollection extends Omit<ICollection, 'type' | 'items'> {
type: 'OrderedCollection';
totalItems: number;
orderedItems: ApObject;
// orderedItems is not defined well
// https://github.com/w3c/activitystreams/issues/494
orderedItems?: ApObject;
}
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage
export interface ICollectionPage extends Omit<ICollection, 'type'> {
type: 'CollectionPage';
partOf?: ICollection | string;
next?: ICollectionPage | string;
prev?: ICollectionPage | string;
}
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollectionpage
export interface IOrderedCollectionPage extends Omit<IOrderedCollection, 'type'>, Omit<ICollectionPage, 'type' | 'items'> {
type: 'OrderedCollectionPage';
startIndex?: number,
}
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
@ -188,6 +209,9 @@ export const isCollection = (object: IObject): object is ICollection =>
export const isOrderedCollection = (object: IObject): object is IOrderedCollection =>
getApType(object) === 'OrderedCollection';
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
isCollection(object) || isOrderedCollection(object);

View file

@ -68,7 +68,7 @@ export class MockResolver extends Resolver {
const r = this.#responseMap.get(value);
if (!r) {
throw new Error('Not registed for mock');
throw new Error('Not registered for mock');
}
const object = JSON.parse(r.content);

View file

@ -17,7 +17,7 @@ import { GlobalModule } from '@/GlobalModule.js';
import { CoreModule } from '@/core/CoreModule.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js';
import type { IActivity, IApDocument, IActor, ICollection, IObject, IOrderedCollection, IOrderedCollectionPage, IPost } from '@/core/activitypub/type.js';
import { MiMeta, MiNote } from '@/models/_.js';
import { secureRndstr } from '@/misc/secure-rndstr.js';
import { DownloadService } from '@/core/DownloadService.js';
@ -29,6 +29,16 @@ const host = 'https://host1.test';
type NonTransientIActor = IActor & { id: string };
type NonTransientIPost = IPost & { id: string };
type NonTransientICollection = ICollection & { id: string };
type NonTransientIOrderedCollection = IOrderedCollection & { id: string };
type NonTransientIOrderedCollectionPage = IOrderedCollectionPage & { id: string };
/**
* Use when the order of the array is not definitive
*/
function deepSortedEqual<T extends unknown[]>(array1: unknown[], array2: T): asserts array1 is T {
return assert.deepStrictEqual(array1.sort(), array2.sort());
}
function createRandomActor({ actorHost = host } = {}): NonTransientIActor {
const preferredUsername = secureRndstr(8);
@ -60,7 +70,7 @@ function createRandomNotes(actor: NonTransientIActor, length: number): NonTransi
return new Array(length).fill(null).map(() => createRandomNote(actor));
}
function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): ICollection {
function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): NonTransientICollection {
const items = createRandomNotes(actor, length);
return {
@ -72,6 +82,53 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe
};
}
function createRandomActivities(actor: NonTransientIActor, type: string, length: number): IActivity[] {
return new Array(length).fill(null).map((): IActivity => {
const note = createRandomNote(actor);
return {
type,
id: `${note.id}/activity`,
actor,
object: note,
};
});
}
function createRandomNonPagedOutbox(actor: NonTransientIActor, length: number): NonTransientIOrderedCollection {
const orderedItems = createRandomActivities(actor, 'Create', length);
return {
'@context': 'https://www.w3.org/ns/activitystreams',
type: 'OrderedCollection',
id: actor.outbox as string,
totalItems: orderedItems.length,
orderedItems,
};
}
function createRandomOutboxPage(actor: NonTransientIActor, id: string, length: number): NonTransientIOrderedCollectionPage {
const orderedItems = createRandomActivities(actor, 'Create', length);
return {
'@context': 'https://www.w3.org/ns/activitystreams',
type: 'OrderedCollectionPage',
id,
totalItems: orderedItems.length,
orderedItems,
};
}
function createRandomPagedOutbox(actor: NonTransientIActor): NonTransientIOrderedCollection {
return {
'@context': 'https://www.w3.org/ns/activitystreams',
type: 'OrderedCollection',
id: actor.outbox as string,
totalItems: 10,
first: `${actor.outbox}?first`,
};
}
async function createRandomRemoteUser(
resolver: MockResolver,
personService: ApPersonService,
@ -196,7 +253,7 @@ describe('ActivityPub', () => {
describe('Renderer', () => {
test('Render an announce with visibility: followers', () => {
rendererService.renderAnnounce(null, {
rendererService.renderAnnounce('hoge', {
createdAt: new Date(0),
visibility: 'followers',
} as MiNote);
@ -216,7 +273,7 @@ describe('ActivityPub', () => {
await personService.createPerson(actor.id, resolver);
// All notes in `featured` are same-origin, no need to fetch notes again
assert.deepStrictEqual(resolver.remoteGetTrials(), [actor.id, actor.featured]);
deepSortedEqual(resolver.remoteGetTrials(), [actor.id, actor.featured, actor.outbox]);
// Created notes without resolving anything
for (const item of featured.items as IPost[]) {
@ -247,9 +304,9 @@ describe('ActivityPub', () => {
await personService.createPerson(actor1.id, resolver);
// actor2Note is from a different server and needs to be fetched again
assert.deepStrictEqual(
deepSortedEqual(
resolver.remoteGetTrials(),
[actor1.id, actor1.featured, actor2Note.id, actor2.id],
[actor1.id, actor1.featured, actor1.outbox, actor2Note.id, actor2.id, actor2.outbox],
);
const note = await noteService.fetchNote(actor2Note.id);
@ -276,6 +333,95 @@ describe('ActivityPub', () => {
});
});
describe('Outbox', () => {
test('Fetch non-paged outbox from IActor', async () => {
const actor = createRandomActor();
const outbox = createRandomNonPagedOutbox(actor, 10);
resolver.register(actor.id, actor);
resolver.register(actor.outbox as string, outbox);
await personService.createPerson(actor.id, resolver);
deepSortedEqual(
resolver.remoteGetTrials(),
[actor.id, actor.outbox],
);
for (const item of outbox.orderedItems as IActivity[]) {
const note = await noteService.fetchNote(item.object);
assert.ok(note);
assert.strictEqual(note.text, 'test test foo');
assert.strictEqual(note.uri, (item.object as IObject).id);
}
});
test('Fetch paged outbox from IActor', async () => {
const actor = createRandomActor();
const outbox = createRandomPagedOutbox(actor);
const page = createRandomOutboxPage(actor, outbox.id, 10);
resolver.register(actor.id, actor);
resolver.register(actor.outbox as string, outbox);
resolver.register(outbox.first as string, page);
await personService.createPerson(actor.id, resolver);
deepSortedEqual(
resolver.remoteGetTrials(),
[actor.id, actor.outbox, outbox.first],
);
for (const item of page.orderedItems as IActivity[]) {
const note = await noteService.fetchNote(item.object);
assert.ok(note);
assert.strictEqual(note.text, 'test test foo');
assert.strictEqual(note.uri, (item.object as IObject).id);
}
});
test('Fetch only the first 20 items', async () => {
const actor = createRandomActor();
const outbox = createRandomNonPagedOutbox(actor, 200);
resolver.register(actor.id, actor);
resolver.register(actor.outbox as string, outbox);
await personService.createPerson(actor.id, resolver);
const items = outbox.orderedItems as IActivity[];
deepSortedEqual(
resolver.remoteGetTrials(),
[actor.id, actor.outbox],
);
assert.ok(await noteService.fetchNote(items[19].object));
assert.ok(!await noteService.fetchNote(items[20].object));
});
test('Perform only Create activities', async () => {
const actor = createRandomActor();
const outbox = createRandomNonPagedOutbox(actor, 0);
outbox.orderedItems = createRandomActivities(actor, 'Announce', 10);
resolver.register(actor.id, actor);
resolver.register(actor.outbox as string, outbox);
await personService.createPerson(actor.id, resolver);
deepSortedEqual(
resolver.remoteGetTrials(),
[actor.id, actor.outbox],
);
for (const item of outbox.orderedItems as IActivity[]) {
const note = await noteService.fetchNote(item.object);
assert.ok(!note);
}
});
});
describe('Images', () => {
test('Create images', async () => {
const imageObject: IApDocument = {