merge: allow overriding all string config values via env - fixes #465 (!476)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/476

Closes #465

Approved-by: Marie <marie@kaifa.ch>

.gitlab-ci.yml

@@ -11,7 +11,7 @@ testCommit:
   variables:
     POSTGRES_PASSWORD: ci
   script:
-    - apt-get update && apt-get install -y git wget curl build-essential python3 
+    - apt-get update && apt-get install -y git wget curl build-essential python3
     - cp .config/ci.yml .config/default.yml
     - corepack enable
     - corepack prepare pnpm@latest --activate
@@ -55,6 +55,8 @@ getImageTag:
   only:
     - stable
     - develop
+    - tags
+
 buildDocker:
   stage: deploy
   needs:
@@ -78,6 +80,8 @@ buildDocker:
   only:
     - stable
     - develop
+    - tags
+
 mergeManifests:
   stage: deploy
   needs:
@@ -103,3 +107,4 @@ mergeManifests:
   only:
     - stable
     - develop
+    - tags

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "sharkey",
-	"version": "2024.3.1",
+	"version": "2024.3.2-devel",
 	"codename": "shonk",
 	"repository": {
 		"type": "git",

packages/backend/package.json

@@ -172,7 +172,7 @@
 		"stringz": "2.1.0",
 		"systeminformation": "5.22.0",
 		"tinycolor2": "1.6.0",
-		"tmp": "0.2.2",
+		"tmp": "0.2.3",
 		"tsc-alias": "1.8.8",
 		"tsconfig-paths": "4.2.0",
 		"typeorm": "0.3.20",

packages/backend/src/config.ts

@@ -212,6 +212,8 @@ export function loadConfig(): Config {
 			{} as Source,
 		) as Source;
 
+	applyEnvOverrides(config);
+
 	const url = tryCreateUrl(config.url);
 	const version = meta.version;
 	const host = url.host;
@@ -304,3 +306,123 @@ function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOp
 		db: options.db ?? 0,
 	};
 }
+
+/*
+	this function allows overriding any string-valued config option with
+	a sensible-named environment variable
+
+	e.g. `MK_CONFIG_MEILISEARCH_APIKEY` sets `config.meilisearch.apikey`
+
+	you can also override a single `dbSlave` value,
+	e.g. `MK_CONFIG_DBSLAVES_1_PASS` sets the password for the 2nd
+	database replica (the first one would be
+	`MK_CONFIG_DBSLAVES_0_PASS`); in this case, `config.dbSlaves` must
+	be set to an array of the right size already in the file
+
+	values can be read from files, too: setting `MK_DB_PASS_FILE` to
+	`/some/file` would set the main database password to the contents of
+	`/some/file` (trimmed of whitespaces)
+ */
+function applyEnvOverrides(config: Source) {
+	// these inner functions recurse through the config structure, using
+	// the given steps, building the env variable name
+
+	function _apply_top(steps: (string | number)[]) {
+		_walk('', [], steps);
+	}
+
+	function _walk(name: string, path: (string | number)[], steps: (string | number)[]) {
+		// are there more steps after this one? recurse
+		if (steps.length > 1) {
+			const thisStep = steps.shift();
+			if (thisStep === null || thisStep === undefined) return;
+
+			// if a step is not a simple value, iterate through it
+			if (typeof thisStep === 'object') {
+				for (const thisOneStep of thisStep) {
+					_descend(name, path, thisOneStep, steps);
+				}
+			} else {
+				_descend(name, path, thisStep, steps);
+			}
+
+			// the actual override has happened at the bottom of the
+			// recursion, we're done
+			return;
+		}
+
+		// this is the last step, same thing as above
+		const lastStep = steps[0];
+
+		if (typeof lastStep === 'object') {
+			for (const lastOneStep of lastStep) {
+				_lastBit(name, path, lastOneStep);
+			}
+		} else {
+			_lastBit(name, path, lastStep);
+		}
+	}
+
+	function _step2name(step: string|number): string {
+		return step.toString().replaceAll(/[^a-z0-9]+/gi,'').toUpperCase();
+	}
+
+	// this recurses down, bailing out if there's no config to override
+	function _descend(name: string, path: (string | number)[], thisStep: string | number, steps: (string | number)[]) {
+		name = `${name}${_step2name(thisStep)}_`;
+		path = [ ...path, thisStep ];
+		_walk(name, path, steps);
+	}
+
+	// this is the bottom of the recursion: look at the environment and
+	// set the value
+	function _lastBit(name: string, path: (string | number)[], lastStep: string | number) {
+		name = `MK_CONFIG_${name}${_step2name(lastStep)}`;
+
+		const val = process.env[name];
+		if (val != null && val != undefined) {
+			_assign(path, lastStep, val);
+		}
+
+		const file = process.env[`${name}_FILE`];
+		if (file) {
+			_assign(path, lastStep, fs.readFileSync(file, 'utf-8').trim());
+		}
+	}
+
+	const alwaysStrings = { 'chmodSocket': 1 };
+
+	function _assign(path: (string | number)[], lastStep: string | number, value: string) {
+		let thisConfig = config;
+		for (const step of path) {
+			if (!thisConfig[step]) {
+				thisConfig[step] = {};
+			}
+			thisConfig = thisConfig[step];
+		}
+
+		if (!alwaysStrings[lastStep]) {
+			if (value.match(/^[0-9]+$/)) {
+				value = parseInt(value);
+			} else if (value.match(/^(true|false)$/i)) {
+				value = !!value.match(/^true$/i);
+			}
+		}
+
+		thisConfig[lastStep] = value;
+	}
+
+	// these are all the settings that can be overridden
+
+	_apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts']]);
+	_apply_top(['db', ['host', 'port', 'db', 'user', 'pass']]);
+	_apply_top(['dbSlaves', config.dbSlaves?.keys(), ['host', 'port', 'db', 'user', 'pass']]);
+	_apply_top([
+		['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines'],
+		['host','port','username','pass','db','prefix'],
+	]);
+	_apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]);
+	_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
+	_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
+	_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
+}

packages/backend/src/queue/processors/ImportNotesProcessorService.ts

@@ -421,7 +421,7 @@ export class ImportNotesProcessorService {
 					if (file.name) {
 						this.driveService.updateFile(exists, { comment: file.name }, user);
 					}
-					
+
 					files.push(exists);
 				}
 			}

packages/backend/src/server/FileServerService.ts

@@ -192,6 +192,7 @@ export class FileServerService {
 						reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
 						reply.header('Accept-Ranges', 'bytes');
 						reply.header('Content-Length', chunksize);
+						reply.code(206);
 					} else {
 						image = {
 							data: fs.createReadStream(file.path),
@@ -261,7 +262,6 @@ export class FileServerService {
 					const parts = range.replace(/bytes=/, '').split('-');
 					const start = parseInt(parts[0], 10);
 					let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
-					console.log(end);
 					if (end > file.file.size) {
 						end = file.file.size - 1;
 					}
@@ -431,6 +431,7 @@ export class FileServerService {
 					reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
 					reply.header('Accept-Ranges', 'bytes');
 					reply.header('Content-Length', chunksize);
+					reply.code(206);
 				} else {
 					image = {
 						data: fs.createReadStream(file.path),
@@ -527,6 +528,9 @@ export class FileServerService {
 		if (!file.storedInternal) {
 			if (!(file.isLink && file.uri)) return '204';
 			const result = await this.downloadAndDetectTypeFromUrl(file.uri);
+			if (!file.size) {
+				file.size = (await fs.promises.stat(result.path)).size;
+			}
 			return {
 				...result,
 				url: file.uri,

packages/backend/src/server/web/views/note.pug

@@ -5,8 +5,8 @@ block vars
 	- const title = user.name ? `${user.name} (@${user.username})` : `@${user.username}`;
 	- const url = `${config.url}/notes/${note.id}`;
 	- const isRenote = note.renote && note.text == null && note.fileIds.length == 0 && note.poll == null;
-	- const images = (note.files || []).filter(file => file.type.startsWith('image/') && !file.isSensitive)
-	- const videos = (note.files || []).filter(file => file.type.startsWith('video/') && !file.isSensitive)
+	- const images = note.cw ? [] : (note.files || []).filter(file => file.type.startsWith('image/') && !file.isSensitive)
+	- const videos = note.cw ? [] : (note.files || []).filter(file => file.type.startsWith('video/') && !file.isSensitive)
 
 block title
 	= `${title} | ${instanceName}`

pnpm-lock.yaml

@@ -392,8 +392,8 @@ importers:
         specifier: 1.6.0
         version: 1.6.0
       tmp:
-        specifier: 0.2.2
-        version: 0.2.2
+        specifier: 0.2.3
+        version: 0.2.3
       tsc-alias:
         specifier: 1.8.8
         version: 1.8.8
@@ -18813,6 +18813,12 @@ packages:
     engines: {node: '>=14'}
     dependencies:
       rimraf: 5.0.5
+    dev: true
+
+  /tmp@0.2.3:
+    resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==}
+    engines: {node: '>=14.14'}
+    dev: false
 
   /tmpl@1.0.5:
     resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==}