merge: allow setting separate timeout / max size for imports - fixes #479 (!519)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/519

Closes #479

Approved-by: Marie <marie@kaifa.ch>
Approved-by: Amelia Yukii <amelia.yukii@shourai.de>

.config/example.yml

@@ -287,5 +287,10 @@ checkActivityPubGetSignature: false
 # Upload or download file size limits (bytes)
 #maxFileSize: 262144000
 
+# timeout and maximum size for imports (e.g. note imports)
+#import:
+#  downloadTimeout: 30
+#  maxFileSize: 262144000
+
 # PID File of master process
 #pidFile: /tmp/misskey.pid

packages/backend/src/config.ts

@@ -97,6 +97,12 @@ type Source = {
 	perChannelMaxNoteCacheCount?: number;
 	perUserNotificationsMaxCount?: number;
 	deactivateAntennaThreshold?: number;
+
+	import?: {
+		downloadTimeout: number;
+		maxFileSize: number;
+	};
+
 	pidFile: string;
 };
 
@@ -177,6 +183,12 @@ export type Config = {
 	perChannelMaxNoteCacheCount: number;
 	perUserNotificationsMaxCount: number;
 	deactivateAntennaThreshold: number;
+
+	import: {
+		downloadTimeout: number;
+		maxFileSize: number;
+	} | undefined;
+
 	pidFile: string;
 };
 
@@ -284,6 +296,7 @@ export function loadConfig(): Config {
 		perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
 		perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
 		deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
+		import: config.import,
 		pidFile: config.pidFile,
 	};
 }
@@ -427,4 +440,5 @@ function applyEnvOverrides(config: Source) {
 	_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
 	_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
 	_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
+	_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
 }

packages/backend/src/core/DownloadService.ts

@@ -35,14 +35,14 @@ export class DownloadService {
 	}
 
 	@bindThis
-	public async downloadUrl(url: string, path: string): Promise<{
+	public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
 		filename: string;
 	}> {
 		this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
 
-		const timeout = 30 * 1000;
+		const timeout = options.timeout ?? 30 * 1000;
-		const operationTimeout = 60 * 1000;
+		const operationTimeout = options.operationTimeout ?? 60 * 1000;
-		const maxSize = this.config.maxFileSize ?? 262144000;
+		const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000;
 
 		const urlObj = new URL(url);
 		let filename = urlObj.pathname.split('/').pop() ?? 'untitled';

packages/backend/src/queue/processors/ImportNotesProcessorService.ts

@@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js';
 import { QueueLoggerService } from '../QueueLoggerService.js';
 import type * as Bull from 'bullmq';
 import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
+import type { Config } from '@/config.js';
 
 @Injectable()
 export class ImportNotesProcessorService {
 	private logger: Logger;
 
 	constructor(
+		@Inject(DI.config)
+		private config: Config,
+
 		@Inject(DI.usersRepository)
 		private usersRepository: UsersRepository,
 
@@ -73,6 +77,11 @@ export class ImportNotesProcessorService {
 		}
 	}
 
+	@bindThis
+	private downloadUrl(url: string, path:string): Promise<{filename: string}> {
+		return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
+	}
+
 	@bindThis
 	private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
 		type NotesMap = {
@@ -176,7 +185,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(destPath, '', 'binary');
-				await this.downloadService.downloadUrl(file.url, destPath);
+				await this.downloadUrl(file.url, destPath);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -206,7 +215,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(destPath, '', 'binary');
-				await this.downloadService.downloadUrl(file.url, destPath);
+				await this.downloadUrl(file.url, destPath);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -239,7 +248,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(destPath, '', 'binary');
-				await this.downloadService.downloadUrl(file.url, destPath);
+				await this.downloadUrl(file.url, destPath);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -297,7 +306,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(path, '', 'utf-8');
-				await this.downloadService.downloadUrl(file.url, path);
+				await this.downloadUrl(file.url, path);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -349,7 +358,7 @@ export class ImportNotesProcessorService {
 
 				if (!exists) {
 					try {
-						await this.downloadService.downloadUrl(file.url, filePath);
+						await this.downloadUrl(file.url, filePath);
 					} catch (e) { // TODO: 何度か再試行
 						this.logger.error(e instanceof Error ? e : new Error(e as string));
 					}
@@ -488,7 +497,7 @@ export class ImportNotesProcessorService {
 
 				if (!exists) {
 					try {
-						await this.downloadService.downloadUrl(file.url, filePath);
+						await this.downloadUrl(file.url, filePath);
 					} catch (e) { // TODO: 何度か再試行
 						this.logger.error(e instanceof Error ? e : new Error(e as string));
 					}