xz: Create command line options for filters[1-9].
The new command line options are meant to be combined with --block-list. They work as an optional extension to --block-list to specify a custom filter chain for each block listed. The new options allow the creation of up to 9 reusable filter chains. For instance: xz --block-list=1:10MiB,3:5MiB,,2:5MiB,1:0 --filters1=delta--lzma2 \ --filters2=x86--lzma2 --filters3=arm64--lzma2 Will create the following blocks: 1. A block of size 10 MiB with filter chain delta, lzma2. 2. A block of size 5 MiB with filter chain arm64, lzma2. 3. A block of size 5 MiB with filter chain arm64, lzma2. 4. A block of size 5 MiB with filter chain x86, lzma2. 5. A block containing the rest of the file contents with filter chain delta, lzma2.
This commit is contained in:
parent
072d292501
commit
d6af7f3470
|
@ -83,14 +83,14 @@ parse_block_list(const char *str_const)
|
||||||
++count;
|
++count;
|
||||||
|
|
||||||
// Prevent an unlikely integer overflow.
|
// Prevent an unlikely integer overflow.
|
||||||
if (count > SIZE_MAX / sizeof(uint64_t) - 1)
|
if (count > SIZE_MAX / sizeof(block_list_entry) - 1)
|
||||||
message_fatal(_("%s: Too many arguments to --block-list"),
|
message_fatal(_("%s: Too many arguments to --block-list"),
|
||||||
str);
|
str);
|
||||||
|
|
||||||
// Allocate memory to hold all the sizes specified.
|
// Allocate memory to hold all the sizes specified.
|
||||||
// If --block-list was specified already, its value is forgotten.
|
// If --block-list was specified already, its value is forgotten.
|
||||||
free(opt_block_list);
|
free(opt_block_list);
|
||||||
opt_block_list = xmalloc((count + 1) * sizeof(uint64_t));
|
opt_block_list = xmalloc((count + 1) * sizeof(block_list_entry));
|
||||||
|
|
||||||
for (size_t i = 0; i < count; ++i) {
|
for (size_t i = 0; i < count; ++i) {
|
||||||
// Locate the next comma and replace it with \0.
|
// Locate the next comma and replace it with \0.
|
||||||
|
@ -98,6 +98,40 @@ parse_block_list(const char *str_const)
|
||||||
if (p != NULL)
|
if (p != NULL)
|
||||||
*p = '\0';
|
*p = '\0';
|
||||||
|
|
||||||
|
// Use the default filter chain unless overridden.
|
||||||
|
opt_block_list[i].filters_index = 0;
|
||||||
|
|
||||||
|
// To specify a filter chain, the block list entry may be
|
||||||
|
// prepended with "[filter-chain-number]:". The size is
|
||||||
|
// still required for every block.
|
||||||
|
// For instance:
|
||||||
|
// --block-list=2:10MiB,1:5MiB,,8MiB,0:0
|
||||||
|
//
|
||||||
|
// Translates to:
|
||||||
|
// 1. Block of 10 MiB using filter chain 2
|
||||||
|
// 2. Block of 5 MiB using filter chain 1
|
||||||
|
// 3. Block of 5 MiB using filter chain 1
|
||||||
|
// 4. Block of 8 MiB using the default filter chain
|
||||||
|
// 5. The last block uses the default filter chain
|
||||||
|
//
|
||||||
|
// The block list:
|
||||||
|
// --block-list=2:MiB,1:,0
|
||||||
|
//
|
||||||
|
// Is not allowed because the second block does not specify
|
||||||
|
// the block size, only the filter chain.
|
||||||
|
if (str[0] >= '0' && str[0] <= '9' && str[1] == ':') {
|
||||||
|
if (str[2] == '\0')
|
||||||
|
message_fatal(_("In --block-list, block "
|
||||||
|
"size is missing after "
|
||||||
|
"filter chain number `%c:'"),
|
||||||
|
str[0]);
|
||||||
|
|
||||||
|
int filter_num = str[0] - '0';
|
||||||
|
opt_block_list[i].filters_index =
|
||||||
|
(uint32_t)filter_num;
|
||||||
|
str += 2;
|
||||||
|
}
|
||||||
|
|
||||||
if (str[0] == '\0') {
|
if (str[0] == '\0') {
|
||||||
// There is no string, that is, a comma follows
|
// There is no string, that is, a comma follows
|
||||||
// another comma. Use the previous value.
|
// another comma. Use the previous value.
|
||||||
|
@ -107,17 +141,17 @@ parse_block_list(const char *str_const)
|
||||||
assert(i > 0);
|
assert(i > 0);
|
||||||
opt_block_list[i] = opt_block_list[i - 1];
|
opt_block_list[i] = opt_block_list[i - 1];
|
||||||
} else {
|
} else {
|
||||||
opt_block_list[i] = str_to_uint64("block-list", str,
|
opt_block_list[i].size = str_to_uint64("block-list",
|
||||||
0, UINT64_MAX);
|
str, 0, UINT64_MAX);
|
||||||
|
|
||||||
// Zero indicates no more new Blocks.
|
// Zero indicates no more new Blocks.
|
||||||
if (opt_block_list[i] == 0) {
|
if (opt_block_list[i].size == 0) {
|
||||||
if (i + 1 != count)
|
if (i + 1 != count)
|
||||||
message_fatal(_("0 can only be used "
|
message_fatal(_("0 can only be used "
|
||||||
"as the last element "
|
"as the last element "
|
||||||
"in --block-list"));
|
"in --block-list"));
|
||||||
|
|
||||||
opt_block_list[i] = UINT64_MAX;
|
opt_block_list[i].size = UINT64_MAX;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,7 +159,7 @@ parse_block_list(const char *str_const)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Terminate the array.
|
// Terminate the array.
|
||||||
opt_block_list[count] = 0;
|
opt_block_list[count].size = 0;
|
||||||
|
|
||||||
free(str_start);
|
free(str_start);
|
||||||
return;
|
return;
|
||||||
|
@ -137,6 +171,16 @@ parse_real(args_info *args, int argc, char **argv)
|
||||||
{
|
{
|
||||||
enum {
|
enum {
|
||||||
OPT_FILTERS = INT_MIN,
|
OPT_FILTERS = INT_MIN,
|
||||||
|
OPT_FILTERS1,
|
||||||
|
OPT_FILTERS2,
|
||||||
|
OPT_FILTERS3,
|
||||||
|
OPT_FILTERS4,
|
||||||
|
OPT_FILTERS5,
|
||||||
|
OPT_FILTERS6,
|
||||||
|
OPT_FILTERS7,
|
||||||
|
OPT_FILTERS8,
|
||||||
|
OPT_FILTERS9,
|
||||||
|
|
||||||
OPT_X86,
|
OPT_X86,
|
||||||
OPT_POWERPC,
|
OPT_POWERPC,
|
||||||
OPT_IA64,
|
OPT_IA64,
|
||||||
|
@ -208,6 +252,16 @@ parse_real(args_info *args, int argc, char **argv)
|
||||||
|
|
||||||
// Filters
|
// Filters
|
||||||
{ "filters", optional_argument, NULL, OPT_FILTERS},
|
{ "filters", optional_argument, NULL, OPT_FILTERS},
|
||||||
|
{ "filters1", optional_argument, NULL, OPT_FILTERS1},
|
||||||
|
{ "filters2", optional_argument, NULL, OPT_FILTERS2},
|
||||||
|
{ "filters3", optional_argument, NULL, OPT_FILTERS3},
|
||||||
|
{ "filters4", optional_argument, NULL, OPT_FILTERS4},
|
||||||
|
{ "filters5", optional_argument, NULL, OPT_FILTERS5},
|
||||||
|
{ "filters6", optional_argument, NULL, OPT_FILTERS6},
|
||||||
|
{ "filters7", optional_argument, NULL, OPT_FILTERS7},
|
||||||
|
{ "filters8", optional_argument, NULL, OPT_FILTERS8},
|
||||||
|
{ "filters9", optional_argument, NULL, OPT_FILTERS9},
|
||||||
|
|
||||||
{ "lzma1", optional_argument, NULL, OPT_LZMA1 },
|
{ "lzma1", optional_argument, NULL, OPT_LZMA1 },
|
||||||
{ "lzma2", optional_argument, NULL, OPT_LZMA2 },
|
{ "lzma2", optional_argument, NULL, OPT_LZMA2 },
|
||||||
{ "x86", optional_argument, NULL, OPT_X86 },
|
{ "x86", optional_argument, NULL, OPT_X86 },
|
||||||
|
@ -379,6 +433,20 @@ parse_real(args_info *args, int argc, char **argv)
|
||||||
coder_add_filters_from_str(optarg);
|
coder_add_filters_from_str(optarg);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
// --filters1...--filters9
|
||||||
|
case OPT_FILTERS1:
|
||||||
|
case OPT_FILTERS2:
|
||||||
|
case OPT_FILTERS3:
|
||||||
|
case OPT_FILTERS4:
|
||||||
|
case OPT_FILTERS5:
|
||||||
|
case OPT_FILTERS6:
|
||||||
|
case OPT_FILTERS7:
|
||||||
|
case OPT_FILTERS8:
|
||||||
|
case OPT_FILTERS9:
|
||||||
|
coder_add_block_filters(optarg,
|
||||||
|
(size_t)(c - OPT_FILTERS));
|
||||||
|
break;
|
||||||
|
|
||||||
case OPT_X86:
|
case OPT_X86:
|
||||||
coder_add_filter(LZMA_FILTER_X86,
|
coder_add_filter(LZMA_FILTER_X86,
|
||||||
options_bcj(optarg));
|
options_bcj(optarg));
|
||||||
|
|
188
src/xz/coder.c
188
src/xz/coder.c
|
@ -26,28 +26,40 @@ enum format_type opt_format = FORMAT_AUTO;
|
||||||
bool opt_auto_adjust = true;
|
bool opt_auto_adjust = true;
|
||||||
bool opt_single_stream = false;
|
bool opt_single_stream = false;
|
||||||
uint64_t opt_block_size = 0;
|
uint64_t opt_block_size = 0;
|
||||||
uint64_t *opt_block_list = NULL;
|
block_list_entry *opt_block_list = NULL;
|
||||||
|
|
||||||
|
|
||||||
/// Stream used to communicate with liblzma
|
/// Stream used to communicate with liblzma
|
||||||
static lzma_stream strm = LZMA_STREAM_INIT;
|
static lzma_stream strm = LZMA_STREAM_INIT;
|
||||||
|
|
||||||
/// Filters needed for all encoding all formats, and also decoding in raw data
|
/// Maximum number of filter chains. The first filter chain is the default,
|
||||||
static lzma_filter filters[LZMA_FILTERS_MAX + 1];
|
/// and 9 other filter chains can be specified with --filtersX.
|
||||||
|
#define NUM_FILTER_CHAIN_MAX 10
|
||||||
|
|
||||||
|
/// The default filter chain is in filters[0]. It is used for encoding
|
||||||
|
/// in all supported formats and also for decdoing raw streams. The other
|
||||||
|
/// filter chains are set by --filtersX to support changing filters with
|
||||||
|
/// the --block-list option.
|
||||||
|
static lzma_filter filters[NUM_FILTER_CHAIN_MAX][LZMA_FILTERS_MAX + 1];
|
||||||
|
|
||||||
|
/// Bit mask representing the filters specified through --filtersX. This
|
||||||
|
/// is needed to verify that an entry in the --block-list option does not
|
||||||
|
/// try to reference a filter chain that was not initialized.
|
||||||
|
static uint32_t filters_init_mask = 1;
|
||||||
|
|
||||||
/// Input and output buffers
|
/// Input and output buffers
|
||||||
static io_buf in_buf;
|
static io_buf in_buf;
|
||||||
static io_buf out_buf;
|
static io_buf out_buf;
|
||||||
|
|
||||||
/// Number of filters. Zero indicates that we are using a preset.
|
/// Number of filters in the default filter chain. Zero indicates that
|
||||||
|
/// we are using a preset.
|
||||||
static uint32_t filters_count = 0;
|
static uint32_t filters_count = 0;
|
||||||
|
|
||||||
/// Number of the preset (0-9)
|
/// Number of the preset (0-9)
|
||||||
static uint32_t preset_number = LZMA_PRESET_DEFAULT;
|
static uint32_t preset_number = LZMA_PRESET_DEFAULT;
|
||||||
|
|
||||||
/// True if the current filter chain was set using the --filters option.
|
/// True if the current default filter chain was set using the --filters
|
||||||
/// The filter chain is reset if a preset option (like -9) or an old-style
|
/// option. The filter chain is reset if a preset option (like -9) or an
|
||||||
/// filter option (like --lzma2) is used after a --filters option.
|
/// old-style filter option (like --lzma2) is used after a --filters option.
|
||||||
static bool string_to_filter_used = false;
|
static bool string_to_filter_used = false;
|
||||||
|
|
||||||
/// Integrity check type
|
/// Integrity check type
|
||||||
|
@ -65,7 +77,6 @@ static bool allow_trailing_input;
|
||||||
static lzma_mt mt_options = {
|
static lzma_mt mt_options = {
|
||||||
.flags = 0,
|
.flags = 0,
|
||||||
.timeout = 300,
|
.timeout = 300,
|
||||||
.filters = filters,
|
|
||||||
};
|
};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
@ -85,7 +96,7 @@ forget_filter_chain(void)
|
||||||
// Setting a preset or using --filters makes us forget
|
// Setting a preset or using --filters makes us forget
|
||||||
// the earlier custom filter chain (if any).
|
// the earlier custom filter chain (if any).
|
||||||
if (filters_count > 0) {
|
if (filters_count > 0) {
|
||||||
lzma_filters_free(filters, NULL);
|
lzma_filters_free(filters[0], NULL);
|
||||||
filters_count = 0;
|
filters_count = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -122,11 +133,11 @@ coder_add_filter(lzma_vli id, void *options)
|
||||||
if (string_to_filter_used)
|
if (string_to_filter_used)
|
||||||
forget_filter_chain();
|
forget_filter_chain();
|
||||||
|
|
||||||
filters[filters_count].id = id;
|
filters[0][filters_count].id = id;
|
||||||
filters[filters_count].options = options;
|
filters[0][filters_count].options = options;
|
||||||
// Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify
|
// Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify
|
||||||
// implementation of forget_filter_chain().
|
// implementation of forget_filter_chain().
|
||||||
filters[++filters_count].id = LZMA_VLI_UNKNOWN;
|
filters[0][++filters_count].id = LZMA_VLI_UNKNOWN;
|
||||||
|
|
||||||
// Setting a custom filter chain makes us forget the preset options.
|
// Setting a custom filter chain makes us forget the preset options.
|
||||||
// This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
|
// This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
|
||||||
|
@ -139,19 +150,24 @@ coder_add_filter(lzma_vli id, void *options)
|
||||||
|
|
||||||
|
|
||||||
static void
|
static void
|
||||||
str_to_filter(const char *str, lzma_filter *filter, uint32_t flags)
|
str_to_filters(const char *str, uint32_t index, uint32_t flags)
|
||||||
{
|
{
|
||||||
int error_pos;
|
int error_pos;
|
||||||
const char *err = lzma_str_to_filters(str, &error_pos, filter,
|
const char *err = lzma_str_to_filters(str, &error_pos,
|
||||||
flags, NULL);
|
filters[index], flags, NULL);
|
||||||
|
|
||||||
if (err != NULL) {
|
if (err != NULL) {
|
||||||
|
char filter_num[2] = "";
|
||||||
|
if (index > 0)
|
||||||
|
filter_num[0] = '0' + index;
|
||||||
|
|
||||||
// FIXME? The message in err isn't translated.
|
// FIXME? The message in err isn't translated.
|
||||||
// Including the translations in the xz translations is
|
// Including the translations in the xz translations is
|
||||||
// slightly ugly but possible. Creating a new domain for
|
// slightly ugly but possible. Creating a new domain for
|
||||||
// liblzma might not be worth it especially since on some
|
// liblzma might not be worth it especially since on some
|
||||||
// OSes it adds extra dependencies to translation libraries.
|
// OSes it adds extra dependencies to translation libraries.
|
||||||
message(V_ERROR, _("Error in --filters=FILTERS option:"));
|
message(V_ERROR, _("Error in --filters%s=FILTERS option:"),
|
||||||
|
filter_num);
|
||||||
message(V_ERROR, "%s", str);
|
message(V_ERROR, "%s", str);
|
||||||
message(V_ERROR, "%*s^", error_pos, "");
|
message(V_ERROR, "%*s^", error_pos, "");
|
||||||
message_fatal("%s", err);
|
message_fatal("%s", err);
|
||||||
|
@ -170,11 +186,12 @@ coder_add_filters_from_str(const char *filter_str)
|
||||||
string_to_filter_used = true;
|
string_to_filter_used = true;
|
||||||
|
|
||||||
// Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw.
|
// Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw.
|
||||||
str_to_filter(filter_str, filters, LZMA_STR_ALL_FILTERS);
|
str_to_filters(filter_str, 0, LZMA_STR_ALL_FILTERS);
|
||||||
|
|
||||||
// Set the filters_count to be the number of filters converted from
|
// Set the filters_count to be the number of filters converted from
|
||||||
// the string.
|
// the string.
|
||||||
for (filters_count = 0; filters[filters_count].id != LZMA_VLI_UNKNOWN;
|
for (filters_count = 0; filters[0][filters_count].id
|
||||||
|
!= LZMA_VLI_UNKNOWN;
|
||||||
++filters_count) ;
|
++filters_count) ;
|
||||||
|
|
||||||
assert(filters_count > 0);
|
assert(filters_count > 0);
|
||||||
|
@ -182,6 +199,19 @@ coder_add_filters_from_str(const char *filter_str)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
extern void
|
||||||
|
coder_add_block_filters(const char *str, size_t slot)
|
||||||
|
{
|
||||||
|
// Free old filters first, if they were previously allocated.
|
||||||
|
if (filters_init_mask & (1 << slot))
|
||||||
|
lzma_filters_free(filters[slot], NULL);
|
||||||
|
|
||||||
|
str_to_filters(str, slot, 0);
|
||||||
|
|
||||||
|
filters_init_mask |= 1 << slot;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static void lzma_attribute((__noreturn__))
|
static void lzma_attribute((__noreturn__))
|
||||||
memlimit_too_small(uint64_t memory_usage)
|
memlimit_too_small(uint64_t memory_usage)
|
||||||
{
|
{
|
||||||
|
@ -192,6 +222,17 @@ memlimit_too_small(uint64_t memory_usage)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// For a given opt_block_list index, validate that the filter has been
|
||||||
|
// set. If it has not been set, we must exit with error to avoid using
|
||||||
|
// an uninitialized filter chain.
|
||||||
|
static void
|
||||||
|
validate_block_list_filter(const uint32_t filter_num)
|
||||||
|
{
|
||||||
|
if (!(filters_init_mask & (1 << filter_num)))
|
||||||
|
message_fatal(_("filter chain %u used by --block-list, but "
|
||||||
|
"not specified with --filters%u="),
|
||||||
|
(unsigned)filter_num, (unsigned)filter_num);
|
||||||
|
}
|
||||||
extern void
|
extern void
|
||||||
coder_set_compression_settings(void)
|
coder_set_compression_settings(void)
|
||||||
{
|
{
|
||||||
|
@ -200,6 +241,11 @@ coder_set_compression_settings(void)
|
||||||
assert(opt_format != FORMAT_LZIP);
|
assert(opt_format != FORMAT_LZIP);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
if (opt_block_list != NULL)
|
||||||
|
for (uint32_t i = 0; opt_block_list[i].size != 0; i++)
|
||||||
|
validate_block_list_filter(
|
||||||
|
opt_block_list[i].filters_index);
|
||||||
|
|
||||||
// The default check type is CRC64, but fallback to CRC32
|
// The default check type is CRC64, but fallback to CRC32
|
||||||
// if CRC64 isn't supported by the copy of liblzma we are
|
// if CRC64 isn't supported by the copy of liblzma we are
|
||||||
// using. CRC32 is always supported.
|
// using. CRC32 is always supported.
|
||||||
|
@ -212,6 +258,10 @@ coder_set_compression_settings(void)
|
||||||
// Options for LZMA1 or LZMA2 in case we are using a preset.
|
// Options for LZMA1 or LZMA2 in case we are using a preset.
|
||||||
static lzma_options_lzma opt_lzma;
|
static lzma_options_lzma opt_lzma;
|
||||||
|
|
||||||
|
// The first filter in the filters[] array is for the default
|
||||||
|
// filter chain.
|
||||||
|
lzma_filter *default_filters = filters[0];
|
||||||
|
|
||||||
if (filters_count == 0) {
|
if (filters_count == 0) {
|
||||||
// We are using a preset. This is not a good idea in raw mode
|
// We are using a preset. This is not a good idea in raw mode
|
||||||
// except when playing around with things. Different versions
|
// except when playing around with things. Different versions
|
||||||
|
@ -232,20 +282,20 @@ coder_set_compression_settings(void)
|
||||||
message_bug();
|
message_bug();
|
||||||
|
|
||||||
// Use LZMA2 except with --format=lzma we use LZMA1.
|
// Use LZMA2 except with --format=lzma we use LZMA1.
|
||||||
filters[0].id = opt_format == FORMAT_LZMA
|
default_filters[0].id = opt_format == FORMAT_LZMA
|
||||||
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
|
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
|
||||||
filters[0].options = &opt_lzma;
|
default_filters[0].options = &opt_lzma;
|
||||||
|
|
||||||
filters_count = 1;
|
filters_count = 1;
|
||||||
|
|
||||||
// Terminate the filter options array.
|
// Terminate the filter options array.
|
||||||
filters[1].id = LZMA_VLI_UNKNOWN;
|
default_filters[1].id = LZMA_VLI_UNKNOWN;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we are using the .lzma format, allow exactly one filter
|
// If we are using the .lzma format, allow exactly one filter
|
||||||
// which has to be LZMA1.
|
// which has to be LZMA1.
|
||||||
if (opt_format == FORMAT_LZMA && (filters_count != 1
|
if (opt_format == FORMAT_LZMA && (filters_count != 1
|
||||||
|| filters[0].id != LZMA_FILTER_LZMA1))
|
|| default_filters[0].id != LZMA_FILTER_LZMA1))
|
||||||
message_fatal(_("The .lzma format supports only "
|
message_fatal(_("The .lzma format supports only "
|
||||||
"the LZMA1 filter"));
|
"the LZMA1 filter"));
|
||||||
|
|
||||||
|
@ -253,19 +303,19 @@ coder_set_compression_settings(void)
|
||||||
// filter to prevent LZMA_PROG_ERROR.
|
// filter to prevent LZMA_PROG_ERROR.
|
||||||
if (opt_format == FORMAT_XZ)
|
if (opt_format == FORMAT_XZ)
|
||||||
for (size_t i = 0; i < filters_count; ++i)
|
for (size_t i = 0; i < filters_count; ++i)
|
||||||
if (filters[i].id == LZMA_FILTER_LZMA1)
|
if (default_filters[i].id == LZMA_FILTER_LZMA1)
|
||||||
message_fatal(_("LZMA1 cannot be used "
|
message_fatal(_("LZMA1 cannot be used "
|
||||||
"with the .xz format"));
|
"with the .xz format"));
|
||||||
|
|
||||||
// Print the selected filter chain.
|
// Print the selected default filter chain.
|
||||||
message_filters_show(V_DEBUG, filters);
|
message_filters_show(V_DEBUG, default_filters);
|
||||||
|
|
||||||
// The --flush-timeout option requires LZMA_SYNC_FLUSH support
|
// The --flush-timeout option requires LZMA_SYNC_FLUSH support
|
||||||
// from the filter chain. Currently threaded encoder doesn't support
|
// from the filter chain. Currently the threaded encoder doesn't
|
||||||
// LZMA_SYNC_FLUSH so single-threaded mode must be used.
|
// support LZMA_SYNC_FLUSH so single-threaded mode must be used.
|
||||||
if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
|
if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
|
||||||
for (size_t i = 0; i < filters_count; ++i) {
|
for (size_t i = 0; i < filters_count; ++i) {
|
||||||
switch (filters[i].id) {
|
switch (default_filters[i].id) {
|
||||||
case LZMA_FILTER_LZMA2:
|
case LZMA_FILTER_LZMA2:
|
||||||
case LZMA_FILTER_DELTA:
|
case LZMA_FILTER_DELTA:
|
||||||
break;
|
break;
|
||||||
|
@ -307,12 +357,12 @@ coder_set_compression_settings(void)
|
||||||
} else
|
} else
|
||||||
# endif
|
# endif
|
||||||
{
|
{
|
||||||
memory_usage = lzma_raw_encoder_memusage(filters);
|
memory_usage = lzma_raw_encoder_memusage(default_filters);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
} else {
|
} else {
|
||||||
#ifdef HAVE_DECODERS
|
#ifdef HAVE_DECODERS
|
||||||
memory_usage = lzma_raw_decoder_memusage(filters);
|
memory_usage = lzma_raw_decoder_memusage(default_filters);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -327,7 +377,7 @@ coder_set_compression_settings(void)
|
||||||
message_mem_needed(V_DEBUG, memory_usage);
|
message_mem_needed(V_DEBUG, memory_usage);
|
||||||
#ifdef HAVE_DECODERS
|
#ifdef HAVE_DECODERS
|
||||||
if (opt_mode == MODE_COMPRESS) {
|
if (opt_mode == MODE_COMPRESS) {
|
||||||
const uint64_t decmem = lzma_raw_decoder_memusage(filters);
|
const uint64_t decmem = lzma_raw_decoder_memusage(default_filters);
|
||||||
if (decmem != UINT64_MAX)
|
if (decmem != UINT64_MAX)
|
||||||
message(V_DEBUG, _("Decompression will need "
|
message(V_DEBUG, _("Decompression will need "
|
||||||
"%s MiB of memory."), uint64_to_str(
|
"%s MiB of memory."), uint64_to_str(
|
||||||
|
@ -407,7 +457,7 @@ coder_set_compression_settings(void)
|
||||||
// the multithreaded mode but the output
|
// the multithreaded mode but the output
|
||||||
// is also different.
|
// is also different.
|
||||||
hardware_threads_set(1);
|
hardware_threads_set(1);
|
||||||
memory_usage = lzma_raw_encoder_memusage(filters);
|
memory_usage = lzma_raw_encoder_memusage(default_filters);
|
||||||
message(V_WARNING, _("Switching to single-threaded mode "
|
message(V_WARNING, _("Switching to single-threaded mode "
|
||||||
"to not exceed the memory usage limit of %s MiB"),
|
"to not exceed the memory usage limit of %s MiB"),
|
||||||
uint64_to_str(round_up_to_mib(memory_limit), 0));
|
uint64_to_str(round_up_to_mib(memory_limit), 0));
|
||||||
|
@ -425,9 +475,9 @@ coder_set_compression_settings(void)
|
||||||
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
|
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
|
||||||
// it use less RAM. With other filters we don't know what to do.
|
// it use less RAM. With other filters we don't know what to do.
|
||||||
size_t i = 0;
|
size_t i = 0;
|
||||||
while (filters[i].id != LZMA_FILTER_LZMA2
|
while (default_filters[i].id != LZMA_FILTER_LZMA2
|
||||||
&& filters[i].id != LZMA_FILTER_LZMA1) {
|
&& default_filters[i].id != LZMA_FILTER_LZMA1) {
|
||||||
if (filters[i].id == LZMA_VLI_UNKNOWN)
|
if (default_filters[i].id == LZMA_VLI_UNKNOWN)
|
||||||
memlimit_too_small(memory_usage);
|
memlimit_too_small(memory_usage);
|
||||||
|
|
||||||
++i;
|
++i;
|
||||||
|
@ -435,7 +485,7 @@ coder_set_compression_settings(void)
|
||||||
|
|
||||||
// Decrease the dictionary size until we meet the memory
|
// Decrease the dictionary size until we meet the memory
|
||||||
// usage limit. First round down to full mebibytes.
|
// usage limit. First round down to full mebibytes.
|
||||||
lzma_options_lzma *opt = filters[i].options;
|
lzma_options_lzma *opt = default_filters[i].options;
|
||||||
const uint32_t orig_dict_size = opt->dict_size;
|
const uint32_t orig_dict_size = opt->dict_size;
|
||||||
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
|
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
|
||||||
while (true) {
|
while (true) {
|
||||||
|
@ -448,7 +498,7 @@ coder_set_compression_settings(void)
|
||||||
if (opt->dict_size < (UINT32_C(1) << 20))
|
if (opt->dict_size < (UINT32_C(1) << 20))
|
||||||
memlimit_too_small(memory_usage);
|
memlimit_too_small(memory_usage);
|
||||||
|
|
||||||
memory_usage = lzma_raw_encoder_memusage(filters);
|
memory_usage = lzma_raw_encoder_memusage(default_filters);
|
||||||
if (memory_usage == UINT64_MAX)
|
if (memory_usage == UINT64_MAX)
|
||||||
message_bug();
|
message_bug();
|
||||||
|
|
||||||
|
@ -466,7 +516,7 @@ coder_set_compression_settings(void)
|
||||||
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
|
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
|
||||||
"from %s MiB to %s MiB to not exceed "
|
"from %s MiB to %s MiB to not exceed "
|
||||||
"the memory usage limit of %s MiB"),
|
"the memory usage limit of %s MiB"),
|
||||||
filters[i].id == LZMA_FILTER_LZMA2
|
default_filters[i].id == LZMA_FILTER_LZMA2
|
||||||
? '2' : '1',
|
? '2' : '1',
|
||||||
uint64_to_str(orig_dict_size >> 20, 0),
|
uint64_to_str(orig_dict_size >> 20, 0),
|
||||||
uint64_to_str(opt->dict_size >> 20, 1),
|
uint64_to_str(opt->dict_size >> 20, 1),
|
||||||
|
@ -566,6 +616,13 @@ coder_init(file_pair *pair)
|
||||||
// These will be handled later in this function.
|
// These will be handled later in this function.
|
||||||
allow_trailing_input = false;
|
allow_trailing_input = false;
|
||||||
|
|
||||||
|
// Set the first filter chain. If the --block-list option is not
|
||||||
|
// used then use the default filter chain (filters[0]).
|
||||||
|
// Otherwise, use first filter chain from the block list.
|
||||||
|
lzma_filter *active_filters = opt_block_list == NULL
|
||||||
|
? filters[0]
|
||||||
|
: filters[opt_block_list[0].filters_index];
|
||||||
|
|
||||||
if (opt_mode == MODE_COMPRESS) {
|
if (opt_mode == MODE_COMPRESS) {
|
||||||
#ifdef HAVE_ENCODERS
|
#ifdef HAVE_ENCODERS
|
||||||
switch (opt_format) {
|
switch (opt_format) {
|
||||||
|
@ -576,17 +633,19 @@ coder_init(file_pair *pair)
|
||||||
|
|
||||||
case FORMAT_XZ:
|
case FORMAT_XZ:
|
||||||
# ifdef MYTHREAD_ENABLED
|
# ifdef MYTHREAD_ENABLED
|
||||||
|
mt_options.filters = active_filters;
|
||||||
if (hardware_threads_is_mt())
|
if (hardware_threads_is_mt())
|
||||||
ret = lzma_stream_encoder_mt(
|
ret = lzma_stream_encoder_mt(
|
||||||
&strm, &mt_options);
|
&strm, &mt_options);
|
||||||
else
|
else
|
||||||
# endif
|
# endif
|
||||||
ret = lzma_stream_encoder(
|
ret = lzma_stream_encoder(
|
||||||
&strm, filters, check);
|
&strm, active_filters, check);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case FORMAT_LZMA:
|
case FORMAT_LZMA:
|
||||||
ret = lzma_alone_encoder(&strm, filters[0].options);
|
ret = lzma_alone_encoder(&strm,
|
||||||
|
active_filters[0].options);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
# ifdef HAVE_LZIP_DECODER
|
# ifdef HAVE_LZIP_DECODER
|
||||||
|
@ -598,7 +657,7 @@ coder_init(file_pair *pair)
|
||||||
# endif
|
# endif
|
||||||
|
|
||||||
case FORMAT_RAW:
|
case FORMAT_RAW:
|
||||||
ret = lzma_raw_encoder(&strm, filters);
|
ret = lzma_raw_encoder(&strm, active_filters);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -722,7 +781,7 @@ coder_init(file_pair *pair)
|
||||||
case FORMAT_RAW:
|
case FORMAT_RAW:
|
||||||
// Memory usage has already been checked in
|
// Memory usage has already been checked in
|
||||||
// coder_set_compression_settings().
|
// coder_set_compression_settings().
|
||||||
ret = lzma_raw_decoder(&strm, filters);
|
ret = lzma_raw_decoder(&strm, active_filters);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -800,12 +859,39 @@ split_block(uint64_t *block_remaining,
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// The Block at *list_pos has been finished. Go to the next
|
// The Block at *list_pos has been finished. Go to the next
|
||||||
// entry in the list. If the end of the list has been reached,
|
// entry in the list. If the end of the list has been
|
||||||
// reuse the size of the last Block.
|
// reached, reuse the size and filters of the last Block.
|
||||||
if (opt_block_list[*list_pos + 1] != 0)
|
if (opt_block_list[*list_pos + 1].size != 0) {
|
||||||
++*list_pos;
|
++*list_pos;
|
||||||
|
|
||||||
*block_remaining = opt_block_list[*list_pos];
|
// Update the filters if needed.
|
||||||
|
if (opt_block_list[*list_pos - 1].filters_index
|
||||||
|
!= opt_block_list[*list_pos].filters_index) {
|
||||||
|
const uint32_t filter_idx = opt_block_list
|
||||||
|
[*list_pos].filters_index;
|
||||||
|
const lzma_filter *next = filters[filter_idx];
|
||||||
|
const lzma_ret ret = lzma_filters_update(
|
||||||
|
&strm, next);
|
||||||
|
|
||||||
|
if (ret != LZMA_OK) {
|
||||||
|
// This message is only possible if
|
||||||
|
// the filter chain has unsupported
|
||||||
|
// options since the filter chain is
|
||||||
|
// validated using
|
||||||
|
// lzma_raw_encoder_memusage() or
|
||||||
|
// lzma_stream_encoder_mt_memusage().
|
||||||
|
// Some options are not validated until
|
||||||
|
// the encoders are initialized.
|
||||||
|
message_fatal(
|
||||||
|
_("Error changing to "
|
||||||
|
"filter chain %u: %s"),
|
||||||
|
(unsigned)filter_idx,
|
||||||
|
message_strm(ret));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
*block_remaining = opt_block_list[*list_pos].size;
|
||||||
|
|
||||||
// If in single-threaded mode, split up the Block if needed.
|
// If in single-threaded mode, split up the Block if needed.
|
||||||
// This is not needed in multi-threaded mode because liblzma
|
// This is not needed in multi-threaded mode because liblzma
|
||||||
|
@ -883,12 +969,14 @@ coder_normal(file_pair *pair)
|
||||||
// output is still not identical because in single-threaded
|
// output is still not identical because in single-threaded
|
||||||
// mode the size info isn't written into Block Headers.
|
// mode the size info isn't written into Block Headers.
|
||||||
if (opt_block_list != NULL) {
|
if (opt_block_list != NULL) {
|
||||||
if (block_remaining < opt_block_list[list_pos]) {
|
if (block_remaining < opt_block_list[list_pos].size) {
|
||||||
assert(!hardware_threads_is_mt());
|
assert(!hardware_threads_is_mt());
|
||||||
next_block_remaining = opt_block_list[list_pos]
|
next_block_remaining =
|
||||||
|
opt_block_list[list_pos].size
|
||||||
- block_remaining;
|
- block_remaining;
|
||||||
} else {
|
} else {
|
||||||
block_remaining = opt_block_list[list_pos];
|
block_remaining =
|
||||||
|
opt_block_list[list_pos].size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,6 +30,18 @@ enum format_type {
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
/// Simple struct to track Block metadata specified through the
|
||||||
|
/// --block-list option.
|
||||||
|
typedef struct {
|
||||||
|
/// Uncompressed size of the Block
|
||||||
|
uint64_t size;
|
||||||
|
|
||||||
|
/// Index into the filters[] representing the filter chain to use
|
||||||
|
/// for this Block.
|
||||||
|
uint32_t filters_index;
|
||||||
|
} block_list_entry;
|
||||||
|
|
||||||
|
|
||||||
/// Operation mode of the command line tool. This is set in args.c and read
|
/// Operation mode of the command line tool. This is set in args.c and read
|
||||||
/// in several files.
|
/// in several files.
|
||||||
extern enum operation_mode opt_mode;
|
extern enum operation_mode opt_mode;
|
||||||
|
@ -50,9 +62,8 @@ extern bool opt_single_stream;
|
||||||
/// of input. This has an effect only when compressing to the .xz format.
|
/// of input. This has an effect only when compressing to the .xz format.
|
||||||
extern uint64_t opt_block_size;
|
extern uint64_t opt_block_size;
|
||||||
|
|
||||||
/// This is non-NULL if --block-list was used. This contains the Block sizes
|
/// List of block size and filter chain pointer pairs.
|
||||||
/// as an array that is terminated with 0.
|
extern block_list_entry *opt_block_list;
|
||||||
extern uint64_t *opt_block_list;
|
|
||||||
|
|
||||||
/// Set the integrity check type used when compressing
|
/// Set the integrity check type used when compressing
|
||||||
extern void coder_set_check(lzma_check check);
|
extern void coder_set_check(lzma_check check);
|
||||||
|
@ -80,3 +91,6 @@ extern void coder_free(void);
|
||||||
|
|
||||||
/// Create filter chain from string
|
/// Create filter chain from string
|
||||||
extern void coder_add_filters_from_str(const char *filter_str);
|
extern void coder_add_filters_from_str(const char *filter_str);
|
||||||
|
|
||||||
|
/// Add or overwrite a filter that can be used by the block-list.
|
||||||
|
extern void coder_add_block_filters(const char *str, size_t slot);
|
||||||
|
|
Loading…
Reference in New Issue