feat(backend): fetch the first page of outbox when resolving Person
This commit is contained in:
Kagami Sascha Rosylight
parent
92d9946f59
commit
2e1de4fca9
5 changed files with 217 additions and 25 deletions
|
|
@ -24,7 +24,7 @@ import { QueueService } from '@/core/QueueService.js';
|
|||
import type { UsersRepository, NotesRepository, FollowingsRepository, AbuseUserReportsRepository, FollowRequestsRepository, } from '@/models/index.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import type { RemoteUser } from '@/models/entities/User.js';
|
||||
import { getApHrefNullable, getApId, getApIds, getApType, getOneApHrefNullable, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
|
||||
import { getApHrefNullable, getApId, getApIds, getApType, getOneApHrefNullable, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isOrderedCollectionPage, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
|
||||
import { ApNoteService } from './models/ApNoteService.js';
|
||||
import { ApLoggerService } from './ApLoggerService.js';
|
||||
import { ApDbResolverService } from './ApDbResolverService.js';
|
||||
|
|
@ -86,10 +86,10 @@ export class ApInboxService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
public async performActivity(actor: RemoteUser, activity: IObject) {
|
||||
if (isCollectionOrOrderedCollection(activity)) {
|
||||
public async performActivity(actor: RemoteUser, activity: IObject, limit = Infinity) {
|
||||
if (isCollectionOrOrderedCollection(activity) || isOrderedCollectionPage(activity)) {
|
||||
const resolver = this.apResolverService.createResolver();
|
||||
for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems)) {
|
||||
for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems).slice(0, limit)) {
|
||||
const act = await resolver.resolve(item);
|
||||
try {
|
||||
await this.performOneActivity(actor, act);
|
||||
|
|
@ -366,7 +366,7 @@ export class ApInboxService {
|
|||
});
|
||||
|
||||
if (isPost(object)) {
|
||||
this.createNote(resolver, actor, object, false, activity);
|
||||
await this.createNote(resolver, actor, object, false, activity);
|
||||
} else {
|
||||
this.logger.warn(`Unknown type: ${getApType(object)}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import { UtilityService } from '@/core/UtilityService.js';
|
|||
import { bindThis } from '@/decorators.js';
|
||||
import { LoggerService } from '@/core/LoggerService.js';
|
||||
import type Logger from '@/logger.js';
|
||||
import { isCollectionOrOrderedCollection } from './type.js';
|
||||
import { isCollectionOrOrderedCollection, isOrderedCollectionPage } from './type.js';
|
||||
import { ApDbResolverService } from './ApDbResolverService.js';
|
||||
import { ApRendererService } from './ApRendererService.js';
|
||||
import { ApRequestService } from './ApRequestService.js';
|
||||
import type { IObject, ICollection, IOrderedCollection } from './type.js';
|
||||
import type { IObject, ICollection, IOrderedCollection, IOrderedCollectionPage } from './type.js';
|
||||
|
||||
export class Resolver {
|
||||
private history: Set<string>;
|
||||
|
|
@ -59,6 +59,18 @@ export class Resolver {
|
|||
}
|
||||
}
|
||||
|
||||
public async resolveOrderedCollectionPage(value: string | IObject): Promise<IOrderedCollectionPage> {
|
||||
const collection = typeof value === 'string'
|
||||
? await this.resolve(value)
|
||||
: value;
|
||||
|
||||
if (isOrderedCollectionPage(collection)) {
|
||||
return collection;
|
||||
} else {
|
||||
throw new Error(`unrecognized collection type: ${collection.type}`);
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async resolve(value: string | IObject): Promise<IObject> {
|
||||
if (value == null) {
|
||||
|
|
|
|||
|
|
@ -34,7 +34,8 @@ import { MetaService } from '@/core/MetaService.js';
|
|||
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
|
||||
import type { AccountMoveService } from '@/core/AccountMoveService.js';
|
||||
import { checkHttps } from '@/misc/check-https.js';
|
||||
import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
|
||||
import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage, isPropertyValue } from '../type.js';
|
||||
import { ApInboxService } from '../ApInboxService.js';
|
||||
import { extractApHashtags } from './tag.js';
|
||||
import type { OnModuleInit } from '@nestjs/common';
|
||||
import type { ApNoteService } from './ApNoteService.js';
|
||||
|
|
@ -62,6 +63,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
private apResolverService: ApResolverService;
|
||||
private apNoteService: ApNoteService;
|
||||
private apImageService: ApImageService;
|
||||
private apInboxService: ApInboxService;
|
||||
private apMfmService: ApMfmService;
|
||||
private mfmService: MfmService;
|
||||
private hashtagService: HashtagService;
|
||||
|
|
@ -128,6 +130,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
this.apResolverService = this.moduleRef.get('ApResolverService');
|
||||
this.apNoteService = this.moduleRef.get('ApNoteService');
|
||||
this.apImageService = this.moduleRef.get('ApImageService');
|
||||
this.apInboxService = this.moduleRef.get('ApInboxService');
|
||||
this.apMfmService = this.moduleRef.get('ApMfmService');
|
||||
this.mfmService = this.moduleRef.get('MfmService');
|
||||
this.hashtagService = this.moduleRef.get('HashtagService');
|
||||
|
|
@ -281,7 +284,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
// Create user
|
||||
let user: RemoteUser;
|
||||
try {
|
||||
// Start transaction
|
||||
// Start transaction
|
||||
await this.db.transaction(async transactionalEntityManager => {
|
||||
user = await transactionalEntityManager.save(new User({
|
||||
id: this.idService.genId(),
|
||||
|
|
@ -327,9 +330,9 @@ export class ApPersonService implements OnModuleInit {
|
|||
}
|
||||
});
|
||||
} catch (e) {
|
||||
// duplicate key error
|
||||
// duplicate key error
|
||||
if (isDuplicateKeyValueError(e)) {
|
||||
// /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応
|
||||
// /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応
|
||||
const u = await this.usersRepository.findOneBy({
|
||||
uri: person.id,
|
||||
});
|
||||
|
|
@ -406,7 +409,10 @@ export class ApPersonService implements OnModuleInit {
|
|||
});
|
||||
//#endregion
|
||||
|
||||
await this.updateFeatured(user!.id, resolver).catch(err => this.logger.error(err));
|
||||
await Promise.all([
|
||||
this.updateFeatured(user!.id, resolver),
|
||||
this.updateOutboxFirstPage(user!, person.outbox, resolver),
|
||||
]).catch(err => this.logger.error(err));
|
||||
|
||||
return user!;
|
||||
}
|
||||
|
|
@ -415,7 +421,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
* Personの情報を更新します。
|
||||
* Misskeyに対象のPersonが登録されていなければ無視します。
|
||||
* もしアカウントの移行が確認された場合、アカウント移行処理を行います。
|
||||
*
|
||||
*
|
||||
* @param uri URI of Person
|
||||
* @param resolver Resolver
|
||||
* @param hint Hint of Person object (この値が正当なPersonの場合、Remote resolveをせずに更新に利用します)
|
||||
|
|
@ -498,7 +504,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
(!exist.movedToUri && updates.movedToUri) ||
|
||||
// 移行先がある→別のもの
|
||||
(exist.movedToUri !== updates.movedToUri && exist.movedToUri && updates.movedToUri);
|
||||
// 移行先がある→ない、ない→ないは無視
|
||||
// 移行先がある→ない、ない→ないは無視
|
||||
|
||||
if (moving) updates.movedAt = new Date();
|
||||
|
||||
|
|
@ -598,9 +604,9 @@ export class ApPersonService implements OnModuleInit {
|
|||
@bindThis
|
||||
public analyzeAttachments(attachments: IObject | IObject[] | undefined) {
|
||||
const fields: {
|
||||
name: string,
|
||||
value: string
|
||||
}[] = [];
|
||||
name: string,
|
||||
value: string
|
||||
}[] = [];
|
||||
if (Array.isArray(attachments)) {
|
||||
for (const attachment of attachments.filter(isPropertyValue)) {
|
||||
fields.push({
|
||||
|
|
@ -613,8 +619,35 @@ export class ApPersonService implements OnModuleInit {
|
|||
return { fields };
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve outbox from an actor object.
|
||||
*
|
||||
* This only retrieves the first page for now.
|
||||
*/
|
||||
public async updateOutboxFirstPage(user: RemoteUser, outbox: IActor['outbox'], resolver: Resolver): Promise<void> {
|
||||
// https://www.w3.org/TR/activitypub/#actor-objects
|
||||
// Outbox is a required property for all actors
|
||||
if (!outbox) {
|
||||
throw new Error('No outbox property');
|
||||
}
|
||||
|
||||
this.logger.info(`Fetching the outbox for ${user.uri}: ${outbox}`);
|
||||
|
||||
const collection = await resolver.resolveCollection(outbox);
|
||||
if (!isOrderedCollection(collection)) {
|
||||
throw new Error('Outbox must be an ordered collection');
|
||||
}
|
||||
|
||||
const firstPage = collection.first ?
|
||||
await resolver.resolveOrderedCollectionPage(collection.first) :
|
||||
collection;
|
||||
|
||||
// Perform activity but only the first 100 ones
|
||||
await this.apInboxService.performActivity(user, firstPage, 100);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async updateFeatured(userId: User['id'], resolver?: Resolver) {
|
||||
public async updateFeatured(userId: User['id'], resolver?: Resolver): Promise<void> {
|
||||
const user = await this.usersRepository.findOneByOrFail({ id: userId });
|
||||
if (!this.userEntityService.isRemoteUser(user)) return;
|
||||
if (!user.featured) return;
|
||||
|
|
@ -631,7 +664,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
|
||||
const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x)));
|
||||
|
||||
// Resolve and regist Notes
|
||||
// Resolve and register Notes
|
||||
const limit = promiseLimit<Note | null>(2);
|
||||
const featuredNotes = await Promise.all(items
|
||||
.filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも
|
||||
|
|
@ -688,7 +721,7 @@ export class ApPersonService implements OnModuleInit {
|
|||
// (uriが存在しなかったり応答がなかったりする場合resolvePersonはthrow Errorする)
|
||||
dst = await this.resolvePerson(src.movedToUri);
|
||||
}
|
||||
|
||||
|
||||
if (dst.movedToUri === dst.uri) return 'skip: movedTo itself (dst)'; // ？？？
|
||||
if (src.movedToUri !== dst.uri) return 'skip: missmatch uri'; // ？？？
|
||||
if (dst.movedToUri === src.uri) return 'skip: dst.movedToUri === src.uri';
|
||||
|
|
|
|||
|
|
@ -87,16 +87,37 @@ export interface IActivity extends IObject {
|
|||
};
|
||||
}
|
||||
|
||||
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection
|
||||
export interface ICollection extends IObject {
|
||||
type: 'Collection';
|
||||
totalItems: number;
|
||||
current?: ICollectionPage | string;
|
||||
first?: ICollectionPage | string;
|
||||
last?: ICollectionPage | string;
|
||||
items: ApObject;
|
||||
}
|
||||
|
||||
export interface IOrderedCollection extends IObject {
|
||||
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection
|
||||
export interface IOrderedCollection extends Omit<ICollection, 'type' | 'items'> {
|
||||
type: 'OrderedCollection';
|
||||
totalItems: number;
|
||||
orderedItems: ApObject;
|
||||
|
||||
// orderedItems is not defined well
|
||||
// https://github.com/w3c/activitystreams/issues/494
|
||||
orderedItems?: ApObject;
|
||||
}
|
||||
|
||||
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage
|
||||
export interface ICollectionPage extends Omit<ICollection, 'type'> {
|
||||
type: 'CollectionPage';
|
||||
partOf?: ICollection | string;
|
||||
next?: ICollectionPage | string;
|
||||
prev?: ICollectionPage | string;
|
||||
}
|
||||
|
||||
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollectionpage
|
||||
export interface IOrderedCollectionPage extends Omit<IOrderedCollection, 'type'>, Omit<ICollectionPage, 'type' | 'items'> {
|
||||
type: 'OrderedCollectionPage';
|
||||
startIndex?: number,
|
||||
}
|
||||
|
||||
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
|
||||
|
|
@ -183,6 +204,9 @@ export const isCollection = (object: IObject): object is ICollection =>
|
|||
export const isOrderedCollection = (object: IObject): object is IOrderedCollection =>
|
||||
getApType(object) === 'OrderedCollection';
|
||||
|
||||
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
|
||||
getApType(object) === 'OrderedCollectionPage';
|
||||
|
||||
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
|
||||
isCollection(object) || isOrderedCollection(object);
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js';
|
|||
import { CoreModule } from '@/core/CoreModule.js';
|
||||
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
|
||||
import { LoggerService } from '@/core/LoggerService.js';
|
||||
import type { IActor } from '@/core/activitypub/type.js';
|
||||
import type { IActor, ICreate, IObject, IOrderedCollection, IOrderedCollectionPage, IPost } from '@/core/activitypub/type.js';
|
||||
import { Note } from '@/models/index.js';
|
||||
import { secureRndstr } from '@/misc/secure-rndstr.js';
|
||||
import { MockResolver } from '../misc/mock-resolver.js';
|
||||
|
|
@ -32,6 +32,59 @@ function createRandomActor(): IActor & { id: string } {
|
|||
};
|
||||
}
|
||||
|
||||
function createRandomCreateActivity(actor: IActor, length: number): ICreate[] {
|
||||
return new Array(length).fill(null).map((): ICreate => {
|
||||
const id = secureRndstr(8);
|
||||
const noteId = `${host}/notes/${id}`;
|
||||
|
||||
return {
|
||||
type: 'Create',
|
||||
id: `${noteId}/activity`,
|
||||
actor,
|
||||
object: {
|
||||
id: noteId,
|
||||
type: 'Note',
|
||||
attributedTo: actor.id,
|
||||
content: 'test test foo',
|
||||
} satisfies IPost,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function createRandomNonPagedOutbox(actor: IActor, length: number): IOrderedCollection {
|
||||
const orderedItems = createRandomCreateActivity(actor, length);
|
||||
|
||||
return {
|
||||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||||
type: 'OrderedCollection',
|
||||
id: actor.outbox as string,
|
||||
totalItems: orderedItems.length,
|
||||
orderedItems,
|
||||
};
|
||||
}
|
||||
|
||||
function createRandomOutboxPage(actor: IActor, id: string, length: number): IOrderedCollectionPage {
|
||||
const orderedItems = createRandomCreateActivity(actor, length);
|
||||
|
||||
return {
|
||||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||||
type: 'OrderedCollectionPage',
|
||||
id,
|
||||
totalItems: orderedItems.length,
|
||||
orderedItems,
|
||||
};
|
||||
}
|
||||
|
||||
function createRandomPagedOutbox(actor: IActor): IOrderedCollection {
|
||||
return {
|
||||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||||
type: 'OrderedCollection',
|
||||
id: actor.outbox as string,
|
||||
totalItems: 10,
|
||||
first: `${actor.outbox}?first`,
|
||||
};
|
||||
}
|
||||
|
||||
describe('ActivityPub', () => {
|
||||
let noteService: ApNoteService;
|
||||
let personService: ApPersonService;
|
||||
|
|
@ -53,7 +106,7 @@ describe('ActivityPub', () => {
|
|||
|
||||
// Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error
|
||||
const federatedInstanceService = app.get<FederatedInstanceService>(FederatedInstanceService);
|
||||
jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => {}));
|
||||
jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => { }));
|
||||
});
|
||||
|
||||
describe('Parse minimum object', () => {
|
||||
|
|
@ -126,4 +179,74 @@ describe('ActivityPub', () => {
|
|||
} as Note);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Outbox', () => {
|
||||
test('Fetch non-paged outbox from IActor', async () => {
|
||||
const actor = createRandomActor();
|
||||
const outbox = createRandomNonPagedOutbox(actor, 10);
|
||||
|
||||
resolver._register(actor.id, actor);
|
||||
resolver._register(actor.outbox as string, outbox);
|
||||
|
||||
// XXX: This shouldn't be needed as the collection already has the full information
|
||||
// But somehow the resolver currently doesn't use it at all and always fetches again
|
||||
for (const item of outbox.orderedItems as IObject[]) {
|
||||
resolver._register(item.id!, item);
|
||||
}
|
||||
|
||||
await personService.createPerson(actor.id, resolver);
|
||||
|
||||
for (const item of outbox.orderedItems as ICreate[]) {
|
||||
const note = await noteService.fetchNote(item.object);
|
||||
assert.ok(note);
|
||||
assert.strictEqual(note.text, 'test test foo');
|
||||
assert.strictEqual(note.uri, (item.object as IObject).id);
|
||||
}
|
||||
});
|
||||
|
||||
test('Fetch paged outbox from IActor', async () => {
|
||||
const actor = createRandomActor();
|
||||
const outbox = createRandomPagedOutbox(actor);
|
||||
const page = createRandomOutboxPage(actor, outbox.id!, 10);
|
||||
|
||||
resolver._register(actor.id, actor);
|
||||
resolver._register(actor.outbox as string, outbox);
|
||||
resolver._register(outbox.first as string, page);
|
||||
|
||||
// XXX: This shouldn't be needed as the collection already has the full information
|
||||
// But somehow the resolver currently doesn't use it at all and always fetches again
|
||||
for (const item of page.orderedItems as IObject[]) {
|
||||
resolver._register(item.id!, item);
|
||||
}
|
||||
|
||||
await personService.createPerson(actor.id, resolver);
|
||||
|
||||
for (const item of page.orderedItems as ICreate[]) {
|
||||
const note = await noteService.fetchNote(item.object);
|
||||
assert.ok(note);
|
||||
assert.strictEqual(note.text, 'test test foo');
|
||||
assert.strictEqual(note.uri, (item.object as IObject).id);
|
||||
}
|
||||
});
|
||||
|
||||
test('Fetch only the first 100 items', async () => {
|
||||
const actor = createRandomActor();
|
||||
const outbox = createRandomNonPagedOutbox(actor, 200);
|
||||
|
||||
resolver._register(actor.id, actor);
|
||||
resolver._register(actor.outbox as string, outbox);
|
||||
|
||||
// XXX: This shouldn't be needed as the collection already has the full information
|
||||
// But somehow the resolver currently doesn't use it at all and always fetches again
|
||||
for (const item of outbox.orderedItems as IObject[]) {
|
||||
resolver._register(item.id!, item);
|
||||
}
|
||||
|
||||
await personService.createPerson(actor.id, resolver);
|
||||
|
||||
const items = outbox.orderedItems as ICreate[];
|
||||
assert.ok(await noteService.fetchNote(items[99].object));
|
||||
assert.ok(!await noteService.fetchNote(items[100].object));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue