xz: Add --memlimit-mt-decompress along with a default limit value.

--memlimit-mt-decompress allows specifying the limit for
multithreaded decompression. This matches memlimit_threading in
liblzma. This limit can only affect the number of threads being
used; it will never prevent xz from decompressing a file. The
old --memlimit-decompress option is still used at the same time.

If the value of --memlimit-decompress (the default value or
one specified by the user) is less than the value of
--memlimit-mt-decompress , then --memlimit-mt-decompress is
reduced to match --memlimit-decompress.

Man page wasn't updated yet.

src/xz/args.c

@@ -29,10 +29,11 @@ bool opt_ignore_check = false;
 const char stdin_filename[] = "(stdin)";
 
 
-/// Parse and set the memory usage limit for compression and/or decompression.
+/// Parse and set the memory usage limit for compression, decompression,
+/// and/or multithreaded decompression.
 static void
 parse_memlimit(const char *name, const char *name_percentage, char *str,
-		bool set_compress, bool set_decompress)
+		bool set_compress, bool set_decompress, bool set_mtdec)
 {
 	bool is_percentage = false;
 	uint64_t value;
@@ -49,8 +50,8 @@ parse_memlimit(const char *name, const char *name_percentage, char *str,
 		value = str_to_uint64(name, str, 0, UINT64_MAX);
 	}
 
-	hardware_memlimit_set(
+	hardware_memlimit_set(value, set_compress, set_decompress, set_mtdec,
-			value, set_compress, set_decompress, is_percentage);
+			is_percentage);
 	return;
 }
 
@@ -138,6 +139,7 @@ parse_real(args_info *args, int argc, char **argv)
 		OPT_BLOCK_LIST,
 		OPT_MEM_COMPRESS,
 		OPT_MEM_DECOMPRESS,
+		OPT_MEM_MT_DECOMPRESS,
 		OPT_NO_ADJUST,
 		OPT_INFO_MEMORY,
 		OPT_ROBOT,
@@ -176,6 +178,7 @@ parse_real(args_info *args, int argc, char **argv)
 		{ "block-list",  required_argument, NULL,  OPT_BLOCK_LIST },
 		{ "memlimit-compress",   required_argument, NULL, OPT_MEM_COMPRESS },
 		{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
+		{ "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS },
 		{ "memlimit",     required_argument, NULL,  'M' },
 		{ "memory",       required_argument, NULL,  'M' }, // Old alias
 		{ "no-adjust",    no_argument,       NULL,  OPT_NO_ADJUST },
@@ -225,20 +228,27 @@ parse_real(args_info *args, int argc, char **argv)
 		case OPT_MEM_COMPRESS:
 			parse_memlimit("memlimit-compress",
 					"memlimit-compress%", optarg,
-					true, false);
+					true, false, false);
 			break;
 
 		// --memlimit-decompress
 		case OPT_MEM_DECOMPRESS:
 			parse_memlimit("memlimit-decompress",
 					"memlimit-decompress%", optarg,
-					false, true);
+					false, true, false);
+			break;
+
+		// --memlimit-mt-decompress
+		case OPT_MEM_MT_DECOMPRESS:
+			parse_memlimit("memlimit-mt-decompress",
+					"memlimit-mt-decompress%", optarg,
+					false, false, true);
 			break;
 
 		// --memlimit
 		case 'M':
 			parse_memlimit("memlimit", "memlimit%", optarg,
-					true, true);
+					true, true, true);
 			break;
 
 		// --suffix

src/xz/coder.c

@@ -524,32 +524,20 @@ coder_init(file_pair *pair)
 			mt_options.flags = flags;
 
 			mt_options.threads = hardware_threads_get();
-
-			// TODO: Support --memlimit-threading=LIMIT.
 			mt_options.memlimit_stop
 				= hardware_memlimit_get(MODE_DECOMPRESS);
-			mt_options.memlimit_threading
-					= mt_options.memlimit_stop;
 
-			if (mt_options.threads == 1) {
+			// If single-threaded mode was requested, set the
-				// Single-threaded mode was requested. Force
+			// memlimit for threading to zero. This forces the
-				// the decoder to use minimal memory, matching
+			// decoder to use single-threaded mode which matches
 			// the behavior of lzma_stream_decoder().
-				mt_options.memlimit_threading = 0;
-
-			} else if (mt_options.memlimit_threading
-					== UINT64_MAX) {
-				// TODO: Support --memlimit-threading=LIMIT.
 			//
-				// If lzma_physmem() fails, it returns 0 and
+			// Otherwise use the limit for threaded decompression
-				// we end up with a single thread.
+			// which has a sane default (users are still free to
-				//
+			// make it insanely high though).
-				// NOTE: It is assential that we never end up
-				// with an effectively infinite value in
-				// memlimit_threading!
 			mt_options.memlimit_threading
-						= lzma_physmem() / 4;
+					= mt_options.threads == 1
-			}
+					? 0 : hardware_memlimit_mtdec_get();
 
 			ret = lzma_stream_decoder_mt(&strm, &mt_options);
 #	else

src/xz/hardware.c

@@ -18,10 +18,26 @@
 static uint32_t threads_max = 1;
 
 /// Memory usage limit for compression
-static uint64_t memlimit_compress;
+static uint64_t memlimit_compress = 0;
 
 /// Memory usage limit for decompression
-static uint64_t memlimit_decompress;
+static uint64_t memlimit_decompress = 0;
+
+/// Default memory usage for multithreaded modes:
+///
+///   - Default value for --memlimit-mt-decompress
+///
+/// This value is caluclated in hardware_init() and cannot be changed later.
+static uint64_t memlimit_mt_default;
+
+/// Memory usage limit for multithreaded decompression. This is a soft limit:
+/// if reducing the number of threads to one isn't enough to keep memory
+/// usage below this limit, then one thread is used and this limit is ignored.
+/// memlimit_decompress is still obeyed.
+///
+/// This can be set with --memlimit-mt-decompress. The default value for
+/// this is memlimit_mt_default.
+static uint64_t memlimit_mtdec;
 
 /// Total amount of physical RAM
 static uint64_t total_ram;
@@ -60,7 +76,8 @@ hardware_threads_get(void)
 
 extern void
 hardware_memlimit_set(uint64_t new_memlimit,
-		bool set_compress, bool set_decompress, bool is_percentage)
+		bool set_compress, bool set_decompress, bool set_mtdec,
+		bool is_percentage)
 {
 	if (is_percentage) {
 		assert(new_memlimit > 0);
@@ -110,6 +127,9 @@ hardware_memlimit_set(uint64_t new_memlimit,
 	if (set_decompress)
 		memlimit_decompress = new_memlimit;
 
+	if (set_mtdec)
+		memlimit_mtdec = new_memlimit;
+
 	return;
 }
 
@@ -132,6 +152,23 @@ hardware_memlimit_get(enum operation_mode mode)
 }
 
 
+extern uint64_t
+hardware_memlimit_mtdec_get(void)
+{
+	uint64_t m = memlimit_mtdec != 0
+			? memlimit_mtdec
+			: memlimit_mt_default;
+
+	// Cap the value to memlimit_decompress if it has been specified.
+	// This is nice for --info-memory. It wouldn't be needed for liblzma
+	// since it does this anyway.
+	if (memlimit_decompress != 0 && m > memlimit_decompress)
+		m = memlimit_decompress;
+
+	return m;
+}
+
+
 /// Helper for hardware_memlimit_show() to print one human-readable info line.
 static void
 memlimit_show(const char *str, size_t str_columns, uint64_t value)
@@ -203,7 +240,20 @@ hardware_init(void)
 	if (total_ram == 0)
 		total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
 
-	// Set the defaults.
+	// FIXME? There may be better methods to determine the default value.
-	hardware_memlimit_set(0, true, true, false);
+	// One Linux-specific suggestion is to use MemAvailable from
+	// /proc/meminfo as the starting point.
+	memlimit_mt_default = total_ram / 4;
+
+	// A too high value may cause 32-bit xz to run out of address space.
+	// Use a conservative maximum value here. A few typical address space
+	// sizes with Linux:
+	//   - x86-64 with 32-bit xz: 4 GiB
+	//   - x86: 3 GiB
+	//   - MIPS32: 2 GiB
+	const size_t mem_ceiling = SIZE_MAX / 3; // About 1365 GiB on 32-bit
+	if (memlimit_mt_default > mem_ceiling)
+		memlimit_mt_default = mem_ceiling;
+
 	return;
 }

src/xz/hardware.h

@@ -22,16 +22,21 @@ extern void hardware_threads_set(uint32_t threadlimit);
 extern uint32_t hardware_threads_get(void);
 
 
-/// Set the memory usage limit. There are separate limits for compression
+/// Set the memory usage limit. There are separate limits for compression,
-/// and decompression (the latter includes also --list), one or both can
+/// decompression (also includes --list), and multithreaded decompression.
-/// be set with a single call to this function. Zero indicates resetting
+/// Any combination of these can be set with a single call to this function.
-/// the limit back to the defaults. The limit can also be set as a percentage
+/// Zero indicates resetting the limit back to the defaults.
-/// of installed RAM; the percentage must be in the range [1, 100].
+/// The limit can also be set as a percentage of installed RAM; the
+/// percentage must be in the range [1, 100].
 extern void hardware_memlimit_set(uint64_t new_memlimit,
-		bool set_compress, bool set_decompress, bool is_percentage);
+		bool set_compress, bool set_decompress, bool set_mtdec,
+		bool is_percentage);
 
 /// Get the current memory usage limit for compression or decompression.
 extern uint64_t hardware_memlimit_get(enum operation_mode mode);
 
+/// Get the current memory usage limit for multithreaded decompression.
+extern uint64_t hardware_memlimit_mtdec_get(void);
+
 /// Display the amount of RAM and memory usage limits and exit.
 extern void hardware_memlimit_show(void) lzma_attribute((__noreturn__));

src/xz/message.c

@@ -1180,9 +1180,11 @@ message_help(bool long_help)
 		puts(_( // xgettext:no-c-format
 "      --memlimit-compress=LIMIT\n"
 "      --memlimit-decompress=LIMIT\n"
+"      --memlimit-mt-decompress=LIMIT\n"
 "  -M, --memlimit=LIMIT\n"
 "                      set memory usage limit for compression, decompression,\n"
-"                      or both; LIMIT is in bytes, % of RAM, or 0 for defaults"));
+"                      threaded decompression, or all of these; LIMIT is in\n"
+"                      bytes, % of RAM, or 0 for defaults"));
 
 		puts(_(
 "      --no-adjust     if compression settings exceed the memory usage limit,\n"