allow setting separate timeout / max size for imports - fixes #479
This commit is contained in:
parent
55fc2879f3
commit
082e1d1afb
|
@ -287,5 +287,10 @@ checkActivityPubGetSignature: false
|
||||||
# Upload or download file size limits (bytes)
|
# Upload or download file size limits (bytes)
|
||||||
#maxFileSize: 262144000
|
#maxFileSize: 262144000
|
||||||
|
|
||||||
|
# timeout and maximum size for imports (e.g. note imports)
|
||||||
|
#import:
|
||||||
|
# downloadTimeout: 30
|
||||||
|
# maxFileSize: 262144000
|
||||||
|
|
||||||
# PID File of master process
|
# PID File of master process
|
||||||
#pidFile: /tmp/misskey.pid
|
#pidFile: /tmp/misskey.pid
|
||||||
|
|
|
@ -97,6 +97,12 @@ type Source = {
|
||||||
perChannelMaxNoteCacheCount?: number;
|
perChannelMaxNoteCacheCount?: number;
|
||||||
perUserNotificationsMaxCount?: number;
|
perUserNotificationsMaxCount?: number;
|
||||||
deactivateAntennaThreshold?: number;
|
deactivateAntennaThreshold?: number;
|
||||||
|
|
||||||
|
import?: {
|
||||||
|
downloadTimeout: number;
|
||||||
|
maxFileSize: number;
|
||||||
|
};
|
||||||
|
|
||||||
pidFile: string;
|
pidFile: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -177,6 +183,12 @@ export type Config = {
|
||||||
perChannelMaxNoteCacheCount: number;
|
perChannelMaxNoteCacheCount: number;
|
||||||
perUserNotificationsMaxCount: number;
|
perUserNotificationsMaxCount: number;
|
||||||
deactivateAntennaThreshold: number;
|
deactivateAntennaThreshold: number;
|
||||||
|
|
||||||
|
import: {
|
||||||
|
downloadTimeout: number;
|
||||||
|
maxFileSize: number;
|
||||||
|
} | undefined;
|
||||||
|
|
||||||
pidFile: string;
|
pidFile: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -284,6 +296,7 @@ export function loadConfig(): Config {
|
||||||
perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
|
perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
|
||||||
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
|
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
|
||||||
deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
|
deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
|
||||||
|
import: config.import,
|
||||||
pidFile: config.pidFile,
|
pidFile: config.pidFile,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -425,4 +438,5 @@ function applyEnvOverrides(config: Source) {
|
||||||
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
|
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
|
||||||
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
|
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
|
||||||
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
|
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
|
||||||
|
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,14 +35,14 @@ export class DownloadService {
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public async downloadUrl(url: string, path: string): Promise<{
|
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
|
||||||
filename: string;
|
filename: string;
|
||||||
}> {
|
}> {
|
||||||
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
|
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
|
||||||
|
|
||||||
const timeout = 30 * 1000;
|
const timeout = options.timeout ?? 30 * 1000;
|
||||||
const operationTimeout = 60 * 1000;
|
const operationTimeout = options.operationTimeout ?? 60 * 1000;
|
||||||
const maxSize = this.config.maxFileSize ?? 262144000;
|
const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000;
|
||||||
|
|
||||||
const urlObj = new URL(url);
|
const urlObj = new URL(url);
|
||||||
let filename = urlObj.pathname.split('/').pop() ?? 'untitled';
|
let filename = urlObj.pathname.split('/').pop() ?? 'untitled';
|
||||||
|
|
|
@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js';
|
||||||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||||
import type * as Bull from 'bullmq';
|
import type * as Bull from 'bullmq';
|
||||||
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
|
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
|
||||||
|
import type { Config } from '@/config.js';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ImportNotesProcessorService {
|
export class ImportNotesProcessorService {
|
||||||
private logger: Logger;
|
private logger: Logger;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
|
@Inject(DI.config)
|
||||||
|
private config: Config,
|
||||||
|
|
||||||
@Inject(DI.usersRepository)
|
@Inject(DI.usersRepository)
|
||||||
private usersRepository: UsersRepository,
|
private usersRepository: UsersRepository,
|
||||||
|
|
||||||
|
@ -73,6 +77,11 @@ export class ImportNotesProcessorService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
private downloadUrl(url: string, path:string): Promise<{filename: string}> {
|
||||||
|
return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
|
||||||
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
|
private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
|
||||||
type NotesMap = {
|
type NotesMap = {
|
||||||
|
@ -176,7 +185,7 @@ export class ImportNotesProcessorService {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fsp.writeFile(destPath, '', 'binary');
|
await fsp.writeFile(destPath, '', 'binary');
|
||||||
await this.downloadService.downloadUrl(file.url, destPath);
|
await this.downloadUrl(file.url, destPath);
|
||||||
} catch (e) { // TODO: 何度か再試行
|
} catch (e) { // TODO: 何度か再試行
|
||||||
if (e instanceof Error || typeof e === 'string') {
|
if (e instanceof Error || typeof e === 'string') {
|
||||||
this.logger.error(e);
|
this.logger.error(e);
|
||||||
|
@ -206,7 +215,7 @@ export class ImportNotesProcessorService {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fsp.writeFile(destPath, '', 'binary');
|
await fsp.writeFile(destPath, '', 'binary');
|
||||||
await this.downloadService.downloadUrl(file.url, destPath);
|
await this.downloadUrl(file.url, destPath);
|
||||||
} catch (e) { // TODO: 何度か再試行
|
} catch (e) { // TODO: 何度か再試行
|
||||||
if (e instanceof Error || typeof e === 'string') {
|
if (e instanceof Error || typeof e === 'string') {
|
||||||
this.logger.error(e);
|
this.logger.error(e);
|
||||||
|
@ -239,7 +248,7 @@ export class ImportNotesProcessorService {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fsp.writeFile(destPath, '', 'binary');
|
await fsp.writeFile(destPath, '', 'binary');
|
||||||
await this.downloadService.downloadUrl(file.url, destPath);
|
await this.downloadUrl(file.url, destPath);
|
||||||
} catch (e) { // TODO: 何度か再試行
|
} catch (e) { // TODO: 何度か再試行
|
||||||
if (e instanceof Error || typeof e === 'string') {
|
if (e instanceof Error || typeof e === 'string') {
|
||||||
this.logger.error(e);
|
this.logger.error(e);
|
||||||
|
@ -297,7 +306,7 @@ export class ImportNotesProcessorService {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fsp.writeFile(path, '', 'utf-8');
|
await fsp.writeFile(path, '', 'utf-8');
|
||||||
await this.downloadService.downloadUrl(file.url, path);
|
await this.downloadUrl(file.url, path);
|
||||||
} catch (e) { // TODO: 何度か再試行
|
} catch (e) { // TODO: 何度か再試行
|
||||||
if (e instanceof Error || typeof e === 'string') {
|
if (e instanceof Error || typeof e === 'string') {
|
||||||
this.logger.error(e);
|
this.logger.error(e);
|
||||||
|
@ -349,7 +358,7 @@ export class ImportNotesProcessorService {
|
||||||
|
|
||||||
if (!exists) {
|
if (!exists) {
|
||||||
try {
|
try {
|
||||||
await this.downloadService.downloadUrl(file.url, filePath);
|
await this.downloadUrl(file.url, filePath);
|
||||||
} catch (e) { // TODO: 何度か再試行
|
} catch (e) { // TODO: 何度か再試行
|
||||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||||
}
|
}
|
||||||
|
@ -488,7 +497,7 @@ export class ImportNotesProcessorService {
|
||||||
|
|
||||||
if (!exists) {
|
if (!exists) {
|
||||||
try {
|
try {
|
||||||
await this.downloadService.downloadUrl(file.url, filePath);
|
await this.downloadUrl(file.url, filePath);
|
||||||
} catch (e) { // TODO: 何度か再試行
|
} catch (e) { // TODO: 何度か再試行
|
||||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue