monkee.ch/monkeeShark
1
0
Fork 0
Code Pull requests Activity

Merge branch 'develop'

Browse source
This commit is contained in:
syuilo 2023-01-25 15:16:07 +09:00
commit 9ffecf25dc
23 changed files with 309 additions and 465 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
CHANGELOG.md
View file

@ -9,6 +9,14 @@
You should also include the user name that made the change.
-->
## 13.2.2 (2023/01/25)
### Improvements
- サーバーのパフォーマンスを改善
### Bugfixes
- サインイン時に誤ったレートリミットがかかることがある問題を修正
- MFMのposition、rotate、scaleで小数が使えない問題を修正
## 13.2.1 (2023/01/24)
### Improvements
- デザインの調整

43
locales/uk-UA.yml
View file

@ -901,6 +901,7 @@ show: "Відображення"
color: "Колір"
achievements: "Досягнення"
_achievements:
earnedAt: "Відкрито"
_types:
_notes1:
title: "налаштовую свій msky"
@ -954,30 +955,72 @@ _achievements:
flavor: "Так багато потрібно сказати?"
_login3:
title: "Новачок I"
description: "3 дні користування загально"
_login7:
title: "Новачок II"
description: "7 днів користування загально"
_login15:
title: "Новачок III"
description: "15 днів користування загально"
_login30:
title: "Міскієць I"
description: "30 днів користування загально"
_login60:
title: "Міскієць II"
description: "60 днів користування загально"
_login100:
title: "Міскієць III"
description: "100 днів користування загально"
_login200:
title: "Завсідник I"
description: "200 днів користування загально"
_login300:
title: "Завсідник II"
description: "300 днів користування загально"
_login400:
title: "Завсідник III"
description: "400 днів користування загально"
_login500:
title: "Ветеран I"
description: "500 днів користування загально"
_login600:
title: "Ветеран II"
description: "600 днів користування загально"
_login700:
title: "Ветеран III"
description: "700 днів користування загально"
_login800:
description: "800 днів користування загально"
_login900:
description: "900 днів користування загально"
_login1000:
description: "1000 днів користування загально"
flavor: "Дякуємо, що користуєтеся Misskey!"
_following50:
description: "Кількість підписок сягнула 50"
_following100:
title: "100 друзів"
description: "Кількість підписок сягнула 100"
_following300:
title: "Надлишок друзів"
description: "Кількість підписок сягнула 300"
_followers1:
title: "Перший підписник"
description: "З'явився перший підписник"
_followers10:
title: "Follow me!"
description: "Кількість підписників досягла 10"
_followers50:
description: "Кількість підписників досягла 50"
_followers100:
description: "Кількість підписників досягла 100"
_followers300:
description: "Кількість підписників досягла 300"
_followers500:
description: "Кількість підписників досягла 500"
_followers1000:
title: "Інфлюенсер"
description: "Кількість підписників досягла 1000"
_passedSinceAccountCreated3:
description: "Минуло 3 роки з моменту створення акаунта"
_loggedInOnBirthday:

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "misskey",
"version": "13.2.1",
"version": "13.2.2",
"codename": "nasubi",
"repository": {
"type": "git",

9
packages/backend/package.json
View file

@ -63,7 +63,7 @@
"file-type": "18.2.0",
"fluent-ffmpeg": "2.1.2",
"form-data": "^4.0.0",
"got": "12.5.3",
"got": "^12.5.3",
"hpagent": "1.2.0",
"ioredis": "4.28.5",
"ip-cidr": "3.0.11",
@ -119,14 +119,14 @@
"typeorm": "0.3.11",
"typescript": "4.9.4",
"ulid": "2.3.0",
"undici": "^5.16.0",
"unzipper": "0.10.11",
"uuid": "9.0.0",
"vary": "1.1.2",
"web-push": "3.5.0",
"websocket": "1.0.34",
"ws": "8.12.0",
"xev": "3.0.2"
"xev": "3.0.2",
"node-fetch": "3.3.0"
},
"devDependencies": {
"@redocly/openapi-core": "1.0.0-beta.120",
@ -182,7 +182,6 @@
"eslint-plugin-import": "2.27.5",
"execa": "6.1.0",
"jest": "29.3.1",
"jest-mock": "^29.3.1",
"node-fetch": "3.3.0"
"jest-mock": "^29.3.1"
}
}

10
packages/backend/src/core/AntennaService.ts
View file

@ -77,10 +77,16 @@ export class AntennaService implements OnApplicationShutdown {
const { type, body } = obj.message as StreamMessages['internal']['payload'];
switch (type) {
case 'antennaCreated':
this.antennas.push(body);
this.antennas.push({
...body,
createdAt: new Date(body.createdAt),
});
break;
case 'antennaUpdated':
this.antennas[this.antennas.findIndex(a => a.id === body.id)] = body;
this.antennas[this.antennas.findIndex(a => a.id === body.id)] = {
...body,
createdAt: new Date(body.createdAt),
};
break;
case 'antennaDeleted':
this.antennas = this.antennas.filter(a => a.id !== body.id);

17
packages/backend/src/core/CaptchaService.ts
View file

@ -21,18 +21,13 @@ export class CaptchaService {
response,
});
const res = await this.httpRequestService.fetch(
url,
{
method: 'POST',
body: params,
const res = await this.httpRequestService.send(url, {
method: 'POST',
body: JSON.stringify(params),
headers: {
'Content-Type': 'application/json',
},
{
noOkError: true,
}
).catch(err => {
throw `${err.message ?? err}`;
});
}, { throwErrorWhenResponseNotOk: false });
if (!res.ok) {
throw `${res.status}`;

76
packages/backend/src/core/DownloadService.ts
View file

@ -5,10 +5,10 @@ import { Inject, Injectable } from '@nestjs/common';
import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip';
import chalk from 'chalk';
import { buildConnector } from 'undici';
import got, * as Got from 'got';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { createTemp } from '@/misc/create-temp.js';
import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js';
@ -20,7 +20,6 @@ import { bindThis } from '@/decorators.js';
@Injectable()
export class DownloadService {
private logger: Logger;
private undiciFetcher: UndiciFetcher;
constructor(
@Inject(DI.config)
@ -30,21 +29,6 @@ export class DownloadService {
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('download');
this.undiciFetcher = this.httpRequestService.createFetcher({
connect: process.env.NODE_ENV === 'development' ?
this.httpRequestService.clientDefaults.connect
:
this.httpRequestService.getConnectorWithIpCheck(
buildConnector({
...this.httpRequestService.clientDefaults.connect,
}),
(ip) => !this.isPrivateIp(ip),
),
bodyTimeout: 30 * 1000,
}, {
connect: this.httpRequestService.clientDefaults.connect,
}, this.logger);
}
@bindThis
@ -55,14 +39,60 @@ export class DownloadService {
const operationTimeout = 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000;
const response = await this.undiciFetcher.fetch(url);
const req = got.stream(url, {
headers: {
'User-Agent': this.config.userAgent,
},
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent: {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
},
http2: false, // default
retry: {
limit: 0,
},
}).on('response', (res: Got.Response) => {
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
if (this.isPrivateIp(res.ip)) {
this.logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
if (response.body === null) {
throw new StatusError('No body', 400, 'No body');
const contentLength = res.headers['content-length'];
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
req.destroy();
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
req.destroy();
}
});
try {
await pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw e;
}
}
await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
}

5
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -2,6 +2,7 @@ import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { JSDOM } from 'jsdom';
import tinycolor from 'tinycolor2';
import fetch from 'node-fetch';
import type { Instance } from '@/models/entities/Instance.js';
import type { InstancesRepository } from '@/models/index.js';
import { AppLockService } from '@/core/AppLockService.js';
@ -190,7 +191,9 @@ export class FetchInstanceMetadataService {
const faviconUrl = url + '/favicon.ico';
const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true });
const favicon = await this.httpRequestService.send(faviconUrl, {
method: 'HEAD',
}, { throwErrorWhenResponseNotOk: false });
if (favicon.ok) {
return faviconUrl;

381
packages/backend/src/core/HttpRequestService.ts
View file

@ -1,284 +1,67 @@
import * as http from 'node:http';
import * as https from 'node:https';
import { LookupFunction } from 'node:net';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import * as undici from 'undici';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
// true to allow, false to deny
export type IpChecker = (ip: string) => boolean;
/*
* Child class to create and save Agent for fetch.
* You should construct this when you want
* to change timeout, size limit, socket connect function, etc.
*/
export class UndiciFetcher {
/**
* Get http non-proxy agent (undici)
*/
public nonProxiedAgent: undici.Agent;
/**
* Get http proxy or non-proxy agent (undici)
*/
public agent: undici.ProxyAgent | undici.Agent;
private proxyBypassHosts: string[];
private userAgent: string | undefined;
private logger: Logger | undefined;
constructor(
args: {
agentOptions: undici.Agent.Options;
proxy?: {
uri: string;
options?: undici.Agent.Options; // Override of agentOptions
},
proxyBypassHosts?: string[];
userAgent?: string;
},
logger?: Logger,
) {
this.logger = logger;
this.logger?.debug('UndiciFetcher constructor', args);
this.proxyBypassHosts = args.proxyBypassHosts ?? [];
this.userAgent = args.userAgent;
this.nonProxiedAgent = new undici.Agent({
...args.agentOptions,
connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called', socket);
if (err) {
this.logger?.debug('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : args.agentOptions.connect,
});
this.agent = args.proxy
? new undici.ProxyAgent({
...args.agentOptions,
...args.proxy.options,
uri: args.proxy.uri,
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy.options?.connect ?? args.agentOptions.connect) !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called (secure)', socket);
if (err) {
this.logger?.debug('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : (args.proxy.options?.connect ?? args.agentOptions.connect),
})
: this.nonProxiedAgent;
}
/**
* Get agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
@bindThis
public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
return this.nonProxiedAgent;
} else {
return this.agent;
}
}
@bindThis
public async fetch(
url: string | URL,
options: undici.RequestInit = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
): Promise<undici.Response> {
const res = await undici.fetch(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
...options,
headers: {
'User-Agent': this.userAgent ?? '',
...(options.headers ?? {}),
},
}).catch((err) => {
this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err);
throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
});
if (!res.ok && !privateOptions.noOkError) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
}
return res;
}
@bindThis
public async request(
url: string | URL,
options: { dispatcher?: undici.Dispatcher } & Omit<undici.Dispatcher.RequestOptions, 'origin' | 'path' | 'method'> & Partial<Pick<undici.Dispatcher.RequestOptions, 'method'>> = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
): Promise<undici.Dispatcher.ResponseData> {
const res = await undici.request(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
...options,
headers: {
'user-agent': this.userAgent ?? '',
...(options.headers ?? {}),
},
}).catch((err) => {
this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err);
throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
});
if (res.statusCode >= 400) {
throw new StatusError(`${res.statusCode}`, res.statusCode, '');
}
return res;
}
@bindThis
public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
const { body } = await this.request(
url,
{
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
},
);
return await body.json() as T;
}
@bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
const { body } = await this.request(
url,
{
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
},
);
return await body.text();
}
}
import type { Response } from 'node-fetch';
import type { URL } from 'node:url';
@Injectable()
export class HttpRequestService {
public defaultFetcher: UndiciFetcher;
public fetch: UndiciFetcher['fetch'];
public request: UndiciFetcher['request'];
public getHtml: UndiciFetcher['getHtml'];
public defaultJsonFetcher: UndiciFetcher;
public getJson: UndiciFetcher['getJson'];
//#region for old http/https, only used in S3Service
// http non-proxy agent
/**
* Get http non-proxy agent
*/
private http: http.Agent;
// https non-proxy agent
/**
* Get https non-proxy agent
*/
private https: https.Agent;
// http proxy or non-proxy agent
/**
* Get http proxy or non-proxy agent
*/
public httpAgent: http.Agent;
// https proxy or non-proxy agent
/**
* Get https proxy or non-proxy agent
*/
public httpsAgent: https.Agent;
//#endregion
public readonly dnsCache: CacheableLookup;
public readonly clientDefaults: undici.Agent.Options;
private maxSockets: number;
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('http-request');
this.dnsCache = new CacheableLookup({
const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
this.clientDefaults = {
keepAliveTimeout: 30 * 1000,
keepAliveMaxTimeout: 10 * 60 * 1000,
keepAliveTimeoutThreshold: 1 * 1000,
strictContentLength: true,
headersTimeout: 10 * 1000,
bodyTimeout: 10 * 1000,
maxHeaderSize: 16364, // default
maxResponseSize: 10 * 1024 * 1024,
maxRedirections: 3,
connect: {
timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
},
};
this.maxSockets = Math.max(64, ((this.config.deliverJobConcurrency ?? 128) / (this.config.clusterLimit ?? 1)));
this.defaultFetcher = this.createFetcher({}, {}, this.logger);
this.fetch = this.defaultFetcher.fetch;
this.request = this.defaultFetcher.request;
this.getHtml = this.defaultFetcher.getHtml;
this.defaultJsonFetcher = this.createFetcher({
maxResponseSize: 1024 * 256,
}, {}, this.logger);
this.getJson = this.defaultJsonFetcher.getJson;
//#region for old http/https, only used in S3Service
this.http = new http.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: this.dnsCache.lookup,
lookup: cache.lookup,
} as http.AgentOptions);
this.https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: this.dnsCache.lookup,
lookup: cache.lookup,
} as https.AgentOptions);
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
this.httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets: this.maxSockets,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
@ -289,47 +72,21 @@ export class HttpRequestService {
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets: this.maxSockets,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
: this.https;
//#endregion
}
@bindThis
private getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
return {
agentOptions: {
...this.clientDefaults,
...opts,
},
...(this.config.proxy ? {
proxy: {
uri: this.config.proxy,
options: {
connections: this.maxSockets,
...proxyOpts,
},
},
} : {}),
userAgent: this.config.userAgent,
};
}
@bindThis
public createFetcher(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}, logger: Logger) {
return new UndiciFetcher(this.getStandardUndiciFetcherOption(opts, proxyOpts), logger);
}
/**
* Get http agent by URL
* Get agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
@bindThis
public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
return url.protocol === 'http:' ? this.http : this.https;
} else {
@ -337,37 +94,67 @@ export class HttpRequestService {
}
}
/**
* check ip
*/
@bindThis
public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
return (options, cb) => {
connector(options, (err, socket) => {
this.logger.debug('Socket connector (with ip checker) called', socket);
if (err) {
this.logger.error('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
public async getJson(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<unknown> {
const res = await this.send(url, {
method: 'GET',
headers: Object.assign({
'User-Agent': this.config.userAgent,
Accept: accept,
}, headers ?? {}),
timeout: 5000,
size: 1024 * 256,
});
if (socket.remoteAddress == undefined) {
this.logger.error('Socket error: remoteAddress is undefined');
cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
return;
}
return await res.json();
}
// allow
if (checkIp(socket.remoteAddress)) {
this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
cb(null, socket);
return;
}
@bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
const res = await this.send(url, {
method: 'GET',
headers: Object.assign({
'User-Agent': this.config.userAgent,
Accept: accept,
}, headers ?? {}),
timeout: 5000,
});
this.logger.error('IP is not allowed', socket);
cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
socket.destroy();
});
};
return await res.text();
}
@bindThis
public async send(url: string, args: {
method?: string,
body?: string,
headers?: Record<string, string>,
timeout?: number,
size?: number,
} = {}, extra: {
throwErrorWhenResponseNotOk: boolean;
} = {
throwErrorWhenResponseNotOk: true,
}): Promise<Response> {
const timeout = args.timeout ?? 5000;
const controller = new AbortController();
setTimeout(() => {
controller.abort();
}, timeout);
const res = await fetch(url, {
method: args.method ?? 'GET',
headers: args.headers,
body: args.body,
size: args.size ?? 10 * 1024 * 1024,
agent: (url) => this.getAgentByUrl(url),
signal: controller.signal,
});
if (!res.ok && extra.throwErrorWhenResponseNotOk) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
}
return res;
}
}

26
packages/backend/src/core/RoleService.ts
View file

@ -91,10 +91,12 @@ export class RoleService implements OnApplicationShutdown {
case 'roleCreated': {
const cached = this.rolesCache.get(null);
if (cached) {
body.createdAt = new Date(body.createdAt);
body.updatedAt = new Date(body.updatedAt);
body.lastUsedAt = new Date(body.lastUsedAt);
cached.push(body);
cached.push({
...body,
createdAt: new Date(body.createdAt),
updatedAt: new Date(body.updatedAt),
lastUsedAt: new Date(body.lastUsedAt),
});
}
break;
}
@ -103,10 +105,12 @@ export class RoleService implements OnApplicationShutdown {
if (cached) {
const i = cached.findIndex(x => x.id === body.id);
if (i > -1) {
body.createdAt = new Date(body.createdAt);
body.updatedAt = new Date(body.updatedAt);
body.lastUsedAt = new Date(body.lastUsedAt);
cached[i] = body;
cached[i] = {
...body,
createdAt: new Date(body.createdAt),
updatedAt: new Date(body.updatedAt),
lastUsedAt: new Date(body.lastUsedAt),
};
}
}
break;
@ -121,8 +125,10 @@ export class RoleService implements OnApplicationShutdown {
case 'userRoleAssigned': {
const cached = this.roleAssignmentByUserIdCache.get(body.userId);
if (cached) {
body.createdAt = new Date(body.createdAt);
cached.push(body);
cached.push({
...body,
createdAt: new Date(body.createdAt),
});
}
break;
}

2
packages/backend/src/core/S3Service.ts
View file

@ -33,7 +33,7 @@ export class S3Service {
? false
: meta.objectStorageS3ForcePathStyle,
httpOptions: {
agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
},
});
}

2
packages/backend/src/core/activitypub/ApRendererService.ts
View file

@ -21,11 +21,11 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import type { UserKeypair } from '@/models/entities/UserKeypair.js';
import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, EmojisRepository, PollsRepository } from '@/models/index.js';
import { bindThis } from '@/decorators.js';
import { LdSignatureService } from './LdSignatureService.js';
import { ApMfmService } from './ApMfmService.js';
import type { IActivity, IObject } from './type.js';
import type { IIdentifier } from './models/identifier.js';
import { bindThis } from '@/decorators.js';
@Injectable()
export class ApRendererService {

38
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -5,16 +5,14 @@ import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { User } from '@/models/entities/User.js';
import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
import type { Dispatcher } from 'undici';
import { DevNull } from '@/misc/dev-null.js';
type Request = {
url: string;
method: Dispatcher.HttpMethod;
method: string;
headers: Record<string, string>;
};
@ -32,7 +30,6 @@ type PrivateKey = {
@Injectable()
export class ApRequestService {
private undiciFetcher: UndiciFetcher;
private logger: Logger;
constructor(
@ -43,10 +40,8 @@ export class ApRequestService {
private httpRequestService: HttpRequestService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = this.httpRequestService.createFetcher({
maxRedirections: 0,
}, {}, this.logger);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
}
@bindThis
@ -165,15 +160,11 @@ export class ApRequestService {
},
});
const response = await this.undiciFetcher.request(
url,
{
method: req.request.method,
headers: req.request.headers,
body,
},
);
response.body.pipe(new DevNull());
await this.httpRequestService.send(url, {
method: req.request.method,
headers: req.request.headers,
body,
});
}
/**
@ -195,13 +186,10 @@ export class ApRequestService {
},
});
const res = await this.httpRequestService.fetch(
url,
{
method: req.request.method,
headers: req.request.headers,
},
);
const res = await this.httpRequestService.send(url, {
method: req.request.method,
headers: req.request.headers,
});
return await res.json();
}

9
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js';
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
@ -19,7 +19,6 @@ import type { IObject, ICollection, IOrderedCollection } from './type.js';
export class Resolver {
private history: Set<string>;
private user?: ILocalUser;
private undiciFetcher: UndiciFetcher;
private logger: Logger;
constructor(
@ -39,10 +38,8 @@ export class Resolver {
private recursionLimit = 100,
) {
this.history = new Set();
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = this.httpRequestService.createFetcher({
maxRedirections: 0,
}, {}, this.logger);
}
@bindThis
@ -106,7 +103,7 @@ export class Resolver {
const object = (this.user
? await this.apRequestService.signedGet(value, this.user) as IObject
: await this.undiciFetcher.getJson<IObject>(value, 'application/activity+json, application/ld+json'));
: await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject;
if (object == null || (
Array.isArray(object['@context']) ?

19
packages/backend/src/core/activitypub/LdSignatureService.ts
View file

@ -9,7 +9,7 @@ import { CONTEXTS } from './misc/contexts.js';
class LdSignature {
public debug = false;
public preLoad = true;
public loderTimeout = 10 * 1000;
public loderTimeout = 5000;
constructor(
private httpRequestService: HttpRequestService,
@ -115,19 +115,12 @@ class LdSignature {
@bindThis
private async fetchDocument(url: string) {
const json = await this.httpRequestService.fetch(
url,
{
headers: {
Accept: 'application/ld+json, application/json',
},
// TODO
//timeout: this.loderTimeout,
const json = await this.httpRequestService.send(url, {
headers: {
Accept: 'application/ld+json, application/json',
},
{
noOkError: true,
}
).then(res => {
timeout: this.loderTimeout,
}, { throwErrorWhenResponseNotOk: false }).then(res => {
if (!res.ok) {
throw `${res.status} ${res.statusText}`;
} else {

8
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -566,22 +566,22 @@ export class ApPersonService implements OnModuleInit {
this.logger.info(`Updating the featured: ${user.uri}`);
if (resolver == null) resolver = this.apResolverService.createResolver();
const _resolver = resolver ?? this.apResolverService.createResolver();
// Resolve to (Ordered)Collection Object
const collection = await resolver.resolveCollection(user.featured);
const collection = await _resolver.resolveCollection(user.featured);
if (!isCollectionOrOrderedCollection(collection)) throw new Error('Object is not Collection or OrderedCollection');
// Resolve to Object(may be Note) arrays
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
const items = await Promise.all(toArray(unresolvedItems).map(x => resolver.resolve(x)));
const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x)));
// Resolve and regist Notes
const limit = promiseLimit<Note | null>(2);
const featuredNotes = await Promise.all(items
.filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも
.slice(0, 5)
.map(item => limit(() => this.apNoteService.resolveNote(item, resolver))));
.map(item => limit(() => this.apNoteService.resolveNote(item, _resolver))));
await this.db.transaction(async transactionalEntityManager => {
await transactionalEntityManager.delete(UserNotePining, { userId: user.id });

15
packages/backend/src/misc/get-ip-hash.ts
View file

@ -1,9 +1,14 @@
import IPCIDR from 'ip-cidr';
export function getIpHash(ip: string) {
// because a single person may control many IPv6 addresses,
// only a /64 subnet prefix of any IP will be taken into account.
// (this means for IPv4 the entire address is used)
const prefix = IPCIDR.createAddress(ip).mask(64);
return 'ip-' + BigInt('0b' + prefix).toString(36);
try {
// because a single person may control many IPv6 addresses,
// only a /64 subnet prefix of any IP will be taken into account.
// (this means for IPv4 the entire address is used)
const prefix = IPCIDR.createAddress(ip).mask(64);
return 'ip-' + BigInt('0b' + prefix).toString(36);
} catch (e) {
const prefix = IPCIDR.createAddress(ip.replace(/:[0-9]+$/, '')).mask(64);
return 'ip-' + BigInt('0b' + prefix).toString(36);
}
}

39
packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts
View file

@ -6,10 +6,10 @@ import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { WebhookDeliverJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
@Injectable()
export class WebhookDeliverProcessorService {
@ -33,26 +33,23 @@ export class WebhookDeliverProcessorService {
try {
this.logger.debug(`delivering ${job.data.webhookId}`);
const res = await this.httpRequestService.fetch(
job.data.to,
{
method: 'POST',
headers: {
'User-Agent': 'Misskey-Hooks',
'X-Misskey-Host': this.config.host,
'X-Misskey-Hook-Id': job.data.webhookId,
'X-Misskey-Hook-Secret': job.data.secret,
},
body: JSON.stringify({
hookId: job.data.webhookId,
userId: job.data.userId,
eventId: job.data.eventId,
createdAt: job.data.createdAt,
type: job.data.type,
body: job.data.content,
}),
}
);
const res = await this.httpRequestService.send(job.data.to, {
method: 'POST',
headers: {
'User-Agent': 'Misskey-Hooks',
'X-Misskey-Host': this.config.host,
'X-Misskey-Hook-Id': job.data.webhookId,
'X-Misskey-Hook-Secret': job.data.secret,
},
body: JSON.stringify({
hookId: job.data.webhookId,
userId: job.data.userId,
eventId: job.data.eventId,
createdAt: job.data.createdAt,
type: job.data.type,
body: job.data.content,
}),
});
this.webhooksRepository.update({ id: job.data.webhookId }, {
latestSentAt: new Date(),

17
packages/backend/src/server/api/endpoints/fetch-rss.ts
View file

@ -33,16 +33,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
private httpRequestService: HttpRequestService,
) {
super(meta, paramDef, async (ps, me) => {
const res = await this.httpRequestService.fetch(
ps.url,
{
method: 'GET',
headers: {
Accept: 'application/rss+xml, */*',
},
// timeout: 5000,
}
);
const res = await this.httpRequestService.send(ps.url, {
method: 'GET',
headers: {
Accept: 'application/rss+xml, */*',
},
timeout: 5000,
});
const text = await res.text();

22
packages/backend/src/server/api/endpoints/notes/translate.ts
View file

@ -7,8 +7,8 @@ import { DI } from '@/di-symbols.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { MetaService } from '@/core/MetaService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { ApiError } from '../../error.js';
import { GetterService } from '@/server/api/GetterService.js';
import { ApiError } from '../../error.js';
export const meta = {
tags: ['notes'],
@ -83,20 +83,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
const res = await this.httpRequestService.fetch(
endpoint,
{
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/json, */*',
},
body: params.toString(),
const res = await this.httpRequestService.send(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/json, */*',
},
{
noOkError: false,
}
);
body: params.toString(),
});
const json = (await res.json()) as {
translations: {

2
packages/backend/src/server/api/stream/types.ts
View file

@ -30,7 +30,7 @@ export interface InternalStreamTypes {
remoteUserUpdated: Serialized<{ id: User['id']; }>;
follow: Serialized<{ followerId: User['id']; followeeId: User['id']; }>;
unfollow: Serialized<{ followerId: User['id']; followeeId: User['id']; }>;
policiesUpdated: Serialized<Role['options']>;
policiesUpdated: Serialized<Role['policies']>;
roleCreated: Serialized<Role>;
roleDeleted: Serialized<Role>;
roleUpdated: Serialized<Role>;

10
packages/frontend/src/components/mfm.ts
View file

@ -190,19 +190,19 @@ export default defineComponent({
return h(MkSparkle, {}, genEl(token.children));
}
case 'rotate': {
const degrees = parseInt(token.props.args.deg) ?? '90';
const degrees = parseFloat(token.props.args.deg) ?? '90';
style = `transform: rotate(${degrees}deg); transform-origin: center center;`;
break;
}
case 'position': {
const x = parseInt(token.props.args.x ?? '0');
const y = parseInt(token.props.args.y ?? '0');
const x = parseFloat(token.props.args.x ?? '0');
const y = parseFloat(token.props.args.y ?? '0');
style = `transform: translateX(${x}em) translateY(${y}em);`;
break;
}
case 'scale': {
const x = Math.min(parseInt(token.props.args.x ?? '1'), 5);
const y = Math.min(parseInt(token.props.args.y ?? '1'), 5);
const x = Math.min(parseFloat(token.props.args.x ?? '1'), 5);
const y = Math.min(parseFloat(token.props.args.y ?? '1'), 5);
style = `transform: scale(${x}, ${y});`;
break;
}

14
pnpm-lock.yaml
View file

@ -142,7 +142,7 @@ importers:
file-type: 18.2.0
fluent-ffmpeg: 2.1.2
form-data: ^4.0.0
got: 12.5.3
got: ^12.5.3
hpagent: 1.2.0
ioredis: 4.28.5
ip-cidr: 3.0.11
@ -201,7 +201,6 @@ importers:
typeorm: 0.3.11
typescript: 4.9.4
ulid: 2.3.0
undici: ^5.16.0
unzipper: 0.10.11
uuid: 9.0.0
vary: 1.1.2
@ -265,6 +264,7 @@ importers:
misskey-js: 0.0.14
ms: 3.0.0-canary.1
nested-property: 4.0.0
node-fetch: 3.3.0
nodemailer: 6.9.0
nsfwjs: 2.4.2_@tensorflow+tfjs@4.2.0
oauth: 0.10.0
@ -306,7 +306,6 @@ importers:
typeorm: 0.3.11_ioredis@4.28.5+pg@8.8.0
typescript: 4.9.4
ulid: 2.3.0
undici: 5.16.0
unzipper: 0.10.11
uuid: 9.0.0
vary: 1.1.2
@ -372,7 +371,6 @@ importers:
execa: 6.1.0
jest: 29.3.1_@types+node@18.11.18
jest-mock: 29.3.1
node-fetch: 3.3.0
packages/frontend:
specifiers:
@ -3801,7 +3799,7 @@ packages:
/axios/0.24.0:
resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==}
dependencies:
follow-redirects: 1.15.2_debug@4.3.4
follow-redirects: 1.15.2
transitivePeerDependencies:
- debug
dev: false
@ -3809,7 +3807,7 @@ packages:
/axios/0.27.2_debug@4.3.4:
resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==}
dependencies:
follow-redirects: 1.15.2_debug@4.3.4
follow-redirects: 1.15.2
form-data: 4.0.0
transitivePeerDependencies:
- debug
@ -6973,7 +6971,7 @@ packages:
readable-stream: 2.3.7
dev: false
/follow-redirects/1.15.2_debug@4.3.4:
/follow-redirects/1.15.2:
resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
engines: {node: '>=4.0'}
peerDependencies:
@ -6981,8 +6979,6 @@ packages:
peerDependenciesMeta:
debug:
optional: true
dependencies:
debug: 4.3.4
/for-each/0.3.3:
resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}