From 25e6409cc91fc7ad733f8c8154aca3d55124c5c1 Mon Sep 17 00:00:00 2001 From: dakkar Date: Wed, 20 Mar 2024 15:38:20 +0000 Subject: [PATCH 1/4] allow overriding all string config values via env - fixes #465 will need end-user documentation! --- packages/backend/src/config.ts | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index c99bc7ae03..1e08a0f6a8 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -212,6 +212,8 @@ export function loadConfig(): Config { {} as Source, ) as Source; + applyEnvOverrides(config); + const url = tryCreateUrl(config.url); const version = meta.version; const host = url.host; @@ -304,3 +306,102 @@ function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOp db: options.db ?? 0, }; } + +/* + this function allows overriding any string-valued config option with + a sensible-named environment variable + + e.g. `MK_CONFIG_MEILISEARCH_APIKEY` overrides `config.meilisearch.apikey` + + the option's containing object must be present in the config *file*, + so in the example above, `config.meilisearch` must be set to + something in the file, it can't be completely commented out. + + you can also override a single `dbSlave` value, + e.g. `MK_CONFIG_DBSLAVES_1_PASS` sets the password for the 2nd + database replica (the first one would be + `MK_CONFIG_DBSLAVES_0_PASS`); again, `config.dbSlaves` must be set + to an array of the right size already in the file + + values can be read from files, too: setting `MK_DB_PASS_FILE` to + `/some/file` would set the main database password to the contents of + `/some/file` (trimmed of whitespaces) + */ +function applyEnvOverrides(config: Source) { + // these inner functions recurse through the config structure, using + // the given steps, building the env variable name + + function _apply_top(steps: (string | number)[]) { + _apply_inner(config, '', steps); + } + + function _apply_inner(thisConfig: any, name: string, steps: (string | number)[]) { + // are there more steps after this one? recurse + if (steps.length > 1) { + const thisStep = steps.shift(); + if (thisStep === null || thisStep === undefined) return; + + // if a step is not a simple value, iterate through it + if (typeof thisStep === 'object') { + for (const thisOneStep of thisStep) { + _descend(thisConfig, name, thisOneStep, steps); + } + } else { + _descend(thisConfig, name, thisStep, steps); + } + + // the actual override has happened at the bottom of the + // recursion, we're done + return; + } + + // this is the last step, same thing as above + const lastStep = steps[0]; + + if (typeof lastStep === 'object') { + for (const lastOneStep of lastStep) { + _lastBit(thisConfig, name, lastOneStep); + } + } else { + _lastBit(thisConfig, name, lastStep); + } + } + + // this recurses down, bailing out if there's no config to override + function _descend(thisConfig: any, name: string, thisStep: string | number, steps: (string | number)[]) { + name = `${name}${thisStep.toString().toUpperCase()}_`; + thisConfig = thisConfig[thisStep]; + if (!thisConfig) return; + _apply_inner(thisConfig, name, steps); + } + + // this is the bottom of the recursion: look at the environment and + // set the value + function _lastBit(thisConfig: any, name: string, lastStep: string | number) { + name = `${name}${lastStep.toString().toUpperCase()}`; + + const val = process.env[`MK_CONFIG_${name}`]; + if (val != null && val != undefined) { + thisConfig[lastStep] = val; + } + + const file = process.env[`MK_CONFIG_${name}_FILE`]; + if (file) { + thisConfig[lastStep] = fs.readFileSync(file, 'utf-8').trim(); + } + } + + // these are all the settings that can be overridden + + _apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts']]); + _apply_top(['db', ['host', 'port', 'db', 'user', 'pass']]); + _apply_top(['dbSlaves', config.dbSlaves?.keys(), ['host', 'port', 'db', 'user', 'pass']]); + _apply_top([ + ['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines'], + ['host','port','username','pass','db','prefix'], + ]); + _apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]); + _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); + _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]); + _apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]); +} From 435cab01c8cf7f29e873eb7d9711dd0ad2ead816 Mon Sep 17 00:00:00 2001 From: dakkar Date: Thu, 21 Mar 2024 10:00:16 +0000 Subject: [PATCH 2/4] deal with (possible, future) non-alnum config keys --- packages/backend/src/config.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 1e08a0f6a8..8f814e4520 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -367,9 +367,13 @@ function applyEnvOverrides(config: Source) { } } + function _step2name(step: string|number): string { + return step.toString().replaceAll(/[^a-z0-9]+/gi,'').toUpperCase(); + } + // this recurses down, bailing out if there's no config to override function _descend(thisConfig: any, name: string, thisStep: string | number, steps: (string | number)[]) { - name = `${name}${thisStep.toString().toUpperCase()}_`; + name = `${name}${_step2name(thisStep)}_`; thisConfig = thisConfig[thisStep]; if (!thisConfig) return; _apply_inner(thisConfig, name, steps); @@ -378,7 +382,7 @@ function applyEnvOverrides(config: Source) { // this is the bottom of the recursion: look at the environment and // set the value function _lastBit(thisConfig: any, name: string, lastStep: string | number) { - name = `${name}${lastStep.toString().toUpperCase()}`; + name = `${name}${_step2name(lastStep)}`; const val = process.env[`MK_CONFIG_${name}`]; if (val != null && val != undefined) { From 0e8cdb30b719a1cc0d9fc3e9c14eacb54444caeb Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 24 Mar 2024 11:12:17 +0000 Subject: [PATCH 3/4] allow setting values not present in the config file replicas and arrays in general, are more complicated :/ --- packages/backend/src/config.ts | 53 +++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 8f814e4520..8d4c5464a6 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -311,17 +311,13 @@ function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOp this function allows overriding any string-valued config option with a sensible-named environment variable - e.g. `MK_CONFIG_MEILISEARCH_APIKEY` overrides `config.meilisearch.apikey` - - the option's containing object must be present in the config *file*, - so in the example above, `config.meilisearch` must be set to - something in the file, it can't be completely commented out. + e.g. `MK_CONFIG_MEILISEARCH_APIKEY` sets `config.meilisearch.apikey` you can also override a single `dbSlave` value, e.g. `MK_CONFIG_DBSLAVES_1_PASS` sets the password for the 2nd database replica (the first one would be - `MK_CONFIG_DBSLAVES_0_PASS`); again, `config.dbSlaves` must be set - to an array of the right size already in the file + `MK_CONFIG_DBSLAVES_0_PASS`); in this case, `config.dbSlaves` must + be set to an array of the right size already in the file values can be read from files, too: setting `MK_DB_PASS_FILE` to `/some/file` would set the main database password to the contents of @@ -332,10 +328,10 @@ function applyEnvOverrides(config: Source) { // the given steps, building the env variable name function _apply_top(steps: (string | number)[]) { - _apply_inner(config, '', steps); + _walk('', [], steps); } - function _apply_inner(thisConfig: any, name: string, steps: (string | number)[]) { + function _walk(name: string, path: (string | number)[], steps: (string | number)[]) { // are there more steps after this one? recurse if (steps.length > 1) { const thisStep = steps.shift(); @@ -344,10 +340,10 @@ function applyEnvOverrides(config: Source) { // if a step is not a simple value, iterate through it if (typeof thisStep === 'object') { for (const thisOneStep of thisStep) { - _descend(thisConfig, name, thisOneStep, steps); + _descend(name, path, thisOneStep, steps); } } else { - _descend(thisConfig, name, thisStep, steps); + _descend(name, path, thisStep, steps); } // the actual override has happened at the bottom of the @@ -360,10 +356,10 @@ function applyEnvOverrides(config: Source) { if (typeof lastStep === 'object') { for (const lastOneStep of lastStep) { - _lastBit(thisConfig, name, lastOneStep); + _lastBit(name, path, lastOneStep); } } else { - _lastBit(thisConfig, name, lastStep); + _lastBit(name, path, lastStep); } } @@ -372,29 +368,40 @@ function applyEnvOverrides(config: Source) { } // this recurses down, bailing out if there's no config to override - function _descend(thisConfig: any, name: string, thisStep: string | number, steps: (string | number)[]) { + function _descend(name: string, path: (string | number)[], thisStep: string | number, steps: (string | number)[]) { name = `${name}${_step2name(thisStep)}_`; - thisConfig = thisConfig[thisStep]; - if (!thisConfig) return; - _apply_inner(thisConfig, name, steps); + path = [ ...path, thisStep ]; + _walk(name, path, steps); } // this is the bottom of the recursion: look at the environment and // set the value - function _lastBit(thisConfig: any, name: string, lastStep: string | number) { - name = `${name}${_step2name(lastStep)}`; + function _lastBit(name: string, path: (string | number)[], lastStep: string | number) { + name = `MK_CONFIG_${name}${_step2name(lastStep)}`; - const val = process.env[`MK_CONFIG_${name}`]; + const val = process.env[name]; if (val != null && val != undefined) { - thisConfig[lastStep] = val; + _assign(path, lastStep, val); } - const file = process.env[`MK_CONFIG_${name}_FILE`]; + const file = process.env[`${name}_FILE`]; if (file) { - thisConfig[lastStep] = fs.readFileSync(file, 'utf-8').trim(); + _assign(path, lastStep, fs.readFileSync(file, 'utf-8').trim()); } } + function _assign(path: (string | number)[], lastStep: string | number, value: string) { + let thisConfig = config; + for (const step of path) { + if (!thisConfig[step]) { + thisConfig[step] = {}; + } + thisConfig = thisConfig[step]; + } + + thisConfig[lastStep] = value; + } + // these are all the settings that can be overridden _apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts']]); From 4271402e0d0e1840791158de288a3e7617227ec4 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 24 Mar 2024 11:17:55 +0000 Subject: [PATCH 4/4] recognise numbers and boolean values --- packages/backend/src/config.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 8d4c5464a6..f6ce9b3cdf 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -390,6 +390,8 @@ function applyEnvOverrides(config: Source) { } } + const alwaysStrings = { 'chmodSocket': 1 }; + function _assign(path: (string | number)[], lastStep: string | number, value: string) { let thisConfig = config; for (const step of path) { @@ -399,6 +401,14 @@ function applyEnvOverrides(config: Source) { thisConfig = thisConfig[step]; } + if (!alwaysStrings[lastStep]) { + if (value.match(/^[0-9]+$/)) { + value = parseInt(value); + } else if (value.match(/^(true|false)$/i)) { + value = !!value.match(/^true$/i); + } + } + thisConfig[lastStep] = value; }