Make the default memory usage limit 40 % of RAM for both
compressing and decompressing. This should be OK now that xz automatically scales down the compression settings if they would exceed the memory usage limit (earlier, the limit for compression was increased to 90 % because low limit broke scripts that used "xz -9" on systems with low RAM). Support spcifying the memory usage limit as a percentage of RAM (e.g. --memory=50%). Support --threads=0 to reset the thread limit to the default value (number of available CPU cores). Use UINT32_MAX instead of SIZE_MAX as the maximum in args.c. hardware.c was already expecting uint32_t value. Cleaned up the output of --help and --long-help.
This commit is contained in:
Lasse Collin
parent
071b825b23
commit
b0063023f8
|
|
@ -114,13 +114,27 @@ parse_real(args_info *args, int argc, char **argv)
|
|||
break;
|
||||
|
||||
// --memory
|
||||
case 'M':
|
||||
// On 32-bit systems, SIZE_MAX would make more sense
|
||||
// than UINT64_MAX. But use UINT64_MAX still so that
|
||||
// scripts that assume > 4 GiB values don't break.
|
||||
case 'M': {
|
||||
// Support specifying the limit as a percentage of
|
||||
// installed physical RAM.
|
||||
size_t len = strlen(optarg);
|
||||
if (len > 0 && optarg[len - 1] == '%') {
|
||||
optarg[len - 1] = '\0';
|
||||
hardware_memlimit_set_percentage(
|
||||
str_to_uint64(
|
||||
"memory%", optarg, 1, 100));
|
||||
} else {
|
||||
// On 32-bit systems, SIZE_MAX would make more
|
||||
// sense than UINT64_MAX. But use UINT64_MAX
|
||||
// still so that scripts that assume > 4 GiB
|
||||
// values don't break.
|
||||
hardware_memlimit_set(str_to_uint64(
|
||||
"memory", optarg, 0, UINT64_MAX));
|
||||
"memory", optarg,
|
||||
0, UINT64_MAX));
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
// --suffix
|
||||
case 'S':
|
||||
|
|
@ -129,7 +143,7 @@ parse_real(args_info *args, int argc, char **argv)
|
|||
|
||||
case 'T':
|
||||
hardware_threadlimit_set(str_to_uint64(
|
||||
"threads", optarg, 1, SIZE_MAX));
|
||||
"threads", optarg, 0, UINT32_MAX));
|
||||
break;
|
||||
|
||||
// --version
|
||||
|
|
|
|||
|
|
@ -17,37 +17,24 @@
|
|||
|
||||
/// Maximum number of free *coder* threads. This can be set with
|
||||
/// the --threads=NUM command line option.
|
||||
static uint32_t threads_max;
|
||||
static uint32_t threadlimit;
|
||||
|
||||
|
||||
/// Memory usage limit for encoding
|
||||
static uint64_t memlimit_encoder;
|
||||
|
||||
/// Memory usage limit for decoding
|
||||
static uint64_t memlimit_decoder;
|
||||
|
||||
/// Memory usage limit given on the command line or environment variable.
|
||||
/// Zero indicates the default (memlimit_encoder or memlimit_decoder).
|
||||
static uint64_t memlimit_custom = 0;
|
||||
|
||||
|
||||
/// Get the number of CPU cores, and set opt_threads to default to that value.
|
||||
/// User can then override this with --threads command line option.
|
||||
static void
|
||||
hardware_threadlimit_init(void)
|
||||
{
|
||||
threads_max = cpucores();
|
||||
if (threads_max == 0)
|
||||
threads_max = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
/// Memory usage limit
|
||||
static uint64_t memlimit;
|
||||
|
||||
|
||||
extern void
|
||||
hardware_threadlimit_set(uint32_t threadlimit)
|
||||
hardware_threadlimit_set(uint32_t new_threadlimit)
|
||||
{
|
||||
threads_max = threadlimit;
|
||||
if (new_threadlimit == 0) {
|
||||
// The default is the number of available CPU cores.
|
||||
threadlimit = cpucores();
|
||||
if (threadlimit == 0)
|
||||
threadlimit = 1;
|
||||
} else {
|
||||
threadlimit = new_threadlimit;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -55,13 +42,30 @@ hardware_threadlimit_set(uint32_t threadlimit)
|
|||
extern uint32_t
|
||||
hardware_threadlimit_get(void)
|
||||
{
|
||||
return threads_max;
|
||||
return threadlimit;
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
hardware_memlimit_init(void)
|
||||
extern void
|
||||
hardware_memlimit_set(uint64_t new_memlimit)
|
||||
{
|
||||
if (new_memlimit == 0) {
|
||||
// The default is 40 % of total installed physical RAM.
|
||||
hardware_memlimit_set_percentage(40);
|
||||
} else {
|
||||
memlimit = new_memlimit;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
extern void
|
||||
hardware_memlimit_set_percentage(uint32_t percentage)
|
||||
{
|
||||
assert(percentage > 0);
|
||||
assert(percentage <= 100);
|
||||
|
||||
uint64_t mem = physmem();
|
||||
|
||||
// If we cannot determine the amount of RAM, assume 32 MiB. Maybe
|
||||
|
|
@ -70,40 +74,22 @@ hardware_memlimit_init(void)
|
|||
if (mem == 0)
|
||||
mem = UINT64_C(32) * 1024 * 1024;
|
||||
|
||||
// Use at maximum of 90 % of RAM when encoding and 33 % when decoding.
|
||||
memlimit_encoder = mem - mem / 10;
|
||||
memlimit_decoder = mem / 3;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
extern void
|
||||
hardware_memlimit_set(uint64_t memlimit)
|
||||
{
|
||||
memlimit_custom = memlimit;
|
||||
memlimit = percentage * mem / 100;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
extern uint64_t
|
||||
hardware_memlimit_encoder(void)
|
||||
hardware_memlimit_get(void)
|
||||
{
|
||||
return memlimit_custom != 0 ? memlimit_custom : memlimit_encoder;
|
||||
}
|
||||
|
||||
|
||||
extern uint64_t
|
||||
hardware_memlimit_decoder(void)
|
||||
{
|
||||
return memlimit_custom != 0 ? memlimit_custom : memlimit_decoder;
|
||||
return memlimit;
|
||||
}
|
||||
|
||||
|
||||
extern void
|
||||
hardware_init(void)
|
||||
{
|
||||
hardware_memlimit_init();
|
||||
hardware_threadlimit_init();
|
||||
hardware_memlimit_set(0);
|
||||
hardware_threadlimit_set(0);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@ extern uint32_t hardware_threadlimit_get(void);
|
|||
/// decoding. Zero indicates resetting the limit back to defaults.
|
||||
extern void hardware_memlimit_set(uint64_t memlimit);
|
||||
|
||||
/// Get the memory usage limit for encoding. By default this is 90 % of RAM.
|
||||
extern uint64_t hardware_memlimit_encoder(void);
|
||||
/// Set custom memory usage limit as a percentage of installed RAM.
|
||||
/// The percentage must be in the range [1, 100].
|
||||
extern void hardware_memlimit_set_percentage(uint32_t percentage);
|
||||
|
||||
|
||||
/// Get the memory usage limit for decoding. By default this is 30 % of RAM.
|
||||
extern uint64_t hardware_memlimit_decoder(void);
|
||||
/// Get the current memory usage limit.
|
||||
extern uint64_t hardware_memlimit_get(void);
|
||||
|
|
|
|||
|
|
@ -1072,6 +1072,7 @@ message_help(bool long_help)
|
|||
" -e, --extreme use more CPU time when encoding to increase compression\n"
|
||||
" ratio without increasing memory usage of the decoder"));
|
||||
|
||||
if (long_help)
|
||||
puts(_(
|
||||
" -M, --memory=NUM use roughly NUM bytes of memory at maximum; 0 indicates\n"
|
||||
" the default setting, which depends on the operation mode\n"
|
||||
|
|
@ -1085,9 +1086,9 @@ message_help(bool long_help)
|
|||
|| defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2)
|
||||
puts(_(
|
||||
"\n"
|
||||
" --lzma1=[OPTS] LZMA1 or LZMA2; OPTS is a comma-separated list of zero or\n"
|
||||
" --lzma2=[OPTS] more of the following options (valid values; default):\n"
|
||||
" preset=NUM reset options to preset number NUM (1-9)\n"
|
||||
" --lzma1[=OPTS] LZMA1 or LZMA2; OPTS is a comma-separated list of zero or\n"
|
||||
" --lzma2[=OPTS] more of the following options (valid values; default):\n"
|
||||
" preset=NUM reset options to preset number NUM (0-9)\n"
|
||||
" dict=NUM dictionary size (4KiB - 1536MiB; 8MiB)\n"
|
||||
" lc=NUM number of literal context bits (0-4; 3)\n"
|
||||
" lp=NUM number of literal position bits (0-4; 0)\n"
|
||||
|
|
@ -1110,7 +1111,7 @@ message_help(bool long_help)
|
|||
#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA)
|
||||
puts(_(
|
||||
"\n"
|
||||
" --delta=[OPTS] Delta filter; valid OPTS (valid values; default):\n"
|
||||
" --delta[=OPTS] Delta filter; valid OPTS (valid values; default):\n"
|
||||
" dist=NUM distance between bytes being subtracted\n"
|
||||
" from each other (1-256; 1)"));
|
||||
#endif
|
||||
|
|
@ -1118,7 +1119,7 @@ message_help(bool long_help)
|
|||
#if defined(HAVE_ENCODER_SUBBLOCK) || defined(HAVE_DECODER_SUBBLOCK)
|
||||
puts(_(
|
||||
"\n"
|
||||
" --subblock=[OPTS] Subblock filter; valid OPTS (valid values; default):\n"
|
||||
" --subblock[=OPTS] Subblock filter; valid OPTS (valid values; default):\n"
|
||||
" size=NUM number of bytes of data per subblock\n"
|
||||
" (1 - 256Mi; 4Ki)\n"
|
||||
" rle=NUM run-length encoder chunk size (0-256; 0)"));
|
||||
|
|
@ -1149,19 +1150,16 @@ message_help(bool long_help)
|
|||
|
||||
if (long_help) {
|
||||
printf(_(
|
||||
"On this system and configuration, the tool will use at maximum of\n"
|
||||
" * roughly %'" PRIu64 " MiB RAM for compression;\n"
|
||||
" * roughly %'" PRIu64 " MiB RAM for decompression; and\n"),
|
||||
hardware_memlimit_encoder() / (1024 * 1024),
|
||||
hardware_memlimit_decoder() / (1024 * 1024));
|
||||
printf(N_(" * one thread for (de)compression.\n\n",
|
||||
" * %'" PRIu32 " threads for (de)compression.\n\n",
|
||||
"On this system and configuration, this program will use at maximum of roughly\n"
|
||||
"%'" PRIu64 " MiB RAM and "), hardware_memlimit_get() / (1024 * 1024));
|
||||
printf(N_("one thread.\n\n", "%'" PRIu32 " threads.\n\n",
|
||||
hardware_threadlimit_get()),
|
||||
hardware_threadlimit_get());
|
||||
}
|
||||
|
||||
printf(_("Report bugs to <%s> (in English or Finnish).\n"),
|
||||
PACKAGE_BUGREPORT);
|
||||
printf(_("XZ Utils home page: <http://tukaani.org/xz/>\n"));
|
||||
|
||||
my_exit(E_SUCCESS);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -152,15 +152,12 @@ coder_set_compression_settings(void)
|
|||
// If using --format=raw, we can be decoding. The memusage function
|
||||
// also validates the filter chain and the options used for the
|
||||
// filters.
|
||||
const uint64_t memory_limit = hardware_memlimit_get();
|
||||
uint64_t memory_usage;
|
||||
uint64_t memory_limit;
|
||||
if (opt_mode == MODE_COMPRESS) {
|
||||
if (opt_mode == MODE_COMPRESS)
|
||||
memory_usage = lzma_raw_encoder_memusage(filters);
|
||||
memory_limit = hardware_memlimit_encoder();
|
||||
} else {
|
||||
else
|
||||
memory_usage = lzma_raw_decoder_memusage(filters);
|
||||
memory_limit = hardware_memlimit_decoder();
|
||||
}
|
||||
|
||||
if (memory_usage == UINT64_MAX)
|
||||
message_fatal("Unsupported filter chain or filter options");
|
||||
|
|
@ -286,17 +283,17 @@ coder_init(void)
|
|||
switch (opt_format) {
|
||||
case FORMAT_AUTO:
|
||||
ret = lzma_auto_decoder(&strm,
|
||||
hardware_memlimit_decoder(), flags);
|
||||
hardware_memlimit_get(), flags);
|
||||
break;
|
||||
|
||||
case FORMAT_XZ:
|
||||
ret = lzma_stream_decoder(&strm,
|
||||
hardware_memlimit_decoder(), flags);
|
||||
hardware_memlimit_get(), flags);
|
||||
break;
|
||||
|
||||
case FORMAT_LZMA:
|
||||
ret = lzma_alone_decoder(&strm,
|
||||
hardware_memlimit_decoder());
|
||||
hardware_memlimit_get());
|
||||
break;
|
||||
|
||||
case FORMAT_RAW:
|
||||
|
|
@ -436,8 +433,7 @@ coder_run(file_pair *pair)
|
|||
// Figure out how much memory it would have
|
||||
// actually needed.
|
||||
uint64_t memusage = lzma_memusage(&strm);
|
||||
uint64_t memlimit
|
||||
= hardware_memlimit_decoder();
|
||||
uint64_t memlimit = hardware_memlimit_get();
|
||||
|
||||
// Round the memory limit down and usage up.
|
||||
// This way we don't display a ridiculous
|
||||
|
|
|
|||
Loading…
Reference in New Issue