Fixed wrong spelling "limitter" to "limiter". This affects
liblzma's API.
This commit is contained in:
parent
beeb810608
commit
3be21fb12f
|
@ -87,7 +87,7 @@ Using liblzma securely
|
||||||
The simplest solution is to use setrlimit() if the kernel supports
|
The simplest solution is to use setrlimit() if the kernel supports
|
||||||
RLIMIT_AS, which limits the memory usage of the whole process.
|
RLIMIT_AS, which limits the memory usage of the whole process.
|
||||||
For more portable and fine-grained limitting, you can use
|
For more portable and fine-grained limitting, you can use
|
||||||
memory limitter functions found from <lzma/memlimit.h>.
|
memory limiter functions found from <lzma/memlimit.h>.
|
||||||
|
|
||||||
|
|
||||||
1.2.1. Encoder
|
1.2.1. Encoder
|
||||||
|
@ -118,24 +118,24 @@ Using liblzma securely
|
||||||
|
|
||||||
1.2.2. Decoder
|
1.2.2. Decoder
|
||||||
|
|
||||||
A single-threaded decoder should simply use a memory limitter and
|
A single-threaded decoder should simply use a memory limiter and
|
||||||
indicate an error if it runs out of memory.
|
indicate an error if it runs out of memory.
|
||||||
|
|
||||||
Memory-limitting with multi-threaded decoding is tricky. The simple
|
Memory-limitting with multi-threaded decoding is tricky. The simple
|
||||||
solution is to divide the maximum allowed memory usage with the
|
solution is to divide the maximum allowed memory usage with the
|
||||||
maximum allowed threads, and give each Block decoder their own
|
maximum allowed threads, and give each Block decoder their own
|
||||||
independent lzma_memory_limitter. The drawback is that if one Block
|
independent lzma_memory_limiter. The drawback is that if one Block
|
||||||
needs notably more RAM than any other Block, the decoder will run out
|
needs notably more RAM than any other Block, the decoder will run out
|
||||||
of memory when in reality there would be plenty of free RAM.
|
of memory when in reality there would be plenty of free RAM.
|
||||||
|
|
||||||
An attractive alternative would be using shared lzma_memory_limitter.
|
An attractive alternative would be using shared lzma_memory_limiter.
|
||||||
Depending on the application and the expected type of input, this may
|
Depending on the application and the expected type of input, this may
|
||||||
either be the best solution or a source of hard-to-repeat problems.
|
either be the best solution or a source of hard-to-repeat problems.
|
||||||
Consider the following requirements:
|
Consider the following requirements:
|
||||||
- You use at maximum of n threads.
|
- You use at maximum of n threads.
|
||||||
- x(i) is the decoder memory requirements of the Block number i
|
- x(i) is the decoder memory requirements of the Block number i
|
||||||
in an expected input Stream.
|
in an expected input Stream.
|
||||||
- The memory limitter is set to higher value than the sum of n
|
- The memory limiter is set to higher value than the sum of n
|
||||||
highest values x(i).
|
highest values x(i).
|
||||||
|
|
||||||
(If you are better at explaining the above conditions, please
|
(If you are better at explaining the above conditions, please
|
||||||
|
@ -150,7 +150,7 @@ Using liblzma securely
|
||||||
Most .lzma files have all the Blocks encoded with identical settings,
|
Most .lzma files have all the Blocks encoded with identical settings,
|
||||||
or at least the memory usage won't vary dramatically. That's why most
|
or at least the memory usage won't vary dramatically. That's why most
|
||||||
multi-threaded decoders probably want to use the simple "separate
|
multi-threaded decoders probably want to use the simple "separate
|
||||||
lzma_memory_limitter for each thread" solution, possibly fallbacking
|
lzma_memory_limiter for each thread" solution, possibly fallbacking
|
||||||
to single-threaded mode in case the per-thread memory limits aren't
|
to single-threaded mode in case the per-thread memory limits aren't
|
||||||
enough in multi-threaded mode.
|
enough in multi-threaded mode.
|
||||||
|
|
||||||
|
@ -206,7 +206,7 @@ FIXME: Memory usage of Stream info.
|
||||||
creating a denial of service like piping decoded a Data Block to
|
creating a denial of service like piping decoded a Data Block to
|
||||||
another process would do.
|
another process would do.
|
||||||
|
|
||||||
At first it would seem that using a memory limitter would prevent
|
At first it would seem that using a memory limiter would prevent
|
||||||
this issue as a side effect. But it does so only if the application
|
this issue as a side effect. But it does so only if the application
|
||||||
requests liblzma to allocate the Extra Records and provide them to
|
requests liblzma to allocate the Extra Records and provide them to
|
||||||
the application. If Extra Records aren't requested, they aren't
|
the application. If Extra Records aren't requested, they aren't
|
||||||
|
|
|
@ -265,8 +265,8 @@ typedef struct {
|
||||||
* custom memory allocation functions for use with liblzma.
|
* custom memory allocation functions for use with liblzma.
|
||||||
*
|
*
|
||||||
* When using lzma_memory_alloc() and lzma_memory_free(), opaque
|
* When using lzma_memory_alloc() and lzma_memory_free(), opaque
|
||||||
* must point to lzma_memory_limitter structure allocated and
|
* must point to lzma_memory_limiter structure allocated and
|
||||||
* initialized with lzma_memory_limitter_create().
|
* initialized with lzma_memory_limiter_create().
|
||||||
*
|
*
|
||||||
* If you don't need this, you should set it to NULL.
|
* If you don't need this, you should set it to NULL.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
/**
|
/**
|
||||||
* \file lzma/memlimit.h
|
* \file lzma/memlimit.h
|
||||||
* \brief Memory usage limitter
|
* \brief Memory usage limiter
|
||||||
*
|
*
|
||||||
* \author Copyright (C) 1999-2006 Igor Pavlov
|
* \author Copyright (C) 1999-2006 Igor Pavlov
|
||||||
* \author Copyright (C) 2007 Lasse Collin
|
* \author Copyright (C) 2007 Lasse Collin
|
||||||
|
@ -39,7 +39,7 @@ typedef struct lzma_memlimit_s lzma_memlimit;
|
||||||
* to these functions can be used in lzma_allocator structure, which makes
|
* to these functions can be used in lzma_allocator structure, which makes
|
||||||
* it easy to limit memory usage with liblzma.
|
* it easy to limit memory usage with liblzma.
|
||||||
*
|
*
|
||||||
* The memory limitter functions are not tied to limitting memory usage
|
* The memory limiter functions are not tied to limitting memory usage
|
||||||
* with liblzma itself. You can use them with anything you like.
|
* with liblzma itself. You can use them with anything you like.
|
||||||
*
|
*
|
||||||
* In multi-threaded applications, only one thread at once may use the same
|
* In multi-threaded applications, only one thread at once may use the same
|
||||||
|
@ -128,7 +128,7 @@ extern lzma_bool lzma_memlimit_reached(lzma_memlimit *mem, lzma_bool clear);
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief Gets the number of allocations owned by the memory limitter
|
* \brief Gets the number of allocations owned by the memory limiter
|
||||||
*
|
*
|
||||||
* The count does not include the helper structures; if no memory has
|
* The count does not include the helper structures; if no memory has
|
||||||
* been allocated with lzma_memlimit_alloc() or all memory allocated
|
* been allocated with lzma_memlimit_alloc() or all memory allocated
|
||||||
|
@ -189,9 +189,9 @@ extern void lzma_memlimit_free(lzma_memlimit *mem, void *ptr);
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief Frees the memory allocated for and by the memory usage limitter
|
* \brief Frees the memory allocated for and by the memory usage limiter
|
||||||
*
|
*
|
||||||
* \param mem Pointer to memory limitter
|
* \param mem Pointer to memory limiter
|
||||||
* \param free_allocated If this is non-zero, all the memory allocated
|
* \param free_allocated If this is non-zero, all the memory allocated
|
||||||
* by lzma_memlimit_alloc() using *mem is also
|
* by lzma_memlimit_alloc() using *mem is also
|
||||||
* freed if it hasn't already been freed with
|
* freed if it hasn't already been freed with
|
||||||
|
|
|
@ -169,8 +169,8 @@ extern lzma_ret lzma_stream_encoder_multi(
|
||||||
* completely when lzma_code() returns LZMA_STREAM_END. If *footer is still
|
* completely when lzma_code() returns LZMA_STREAM_END. If *footer is still
|
||||||
* NULL, there was no Extra field in the Footer Metadata Block.
|
* NULL, there was no Extra field in the Footer Metadata Block.
|
||||||
*
|
*
|
||||||
* \note If you use lzma_memory_limitter, the Extra Records will be
|
* \note If you use lzma_memory_limiter, the Extra Records will be
|
||||||
* allocated with it, and thus remain in the lzma_memory_limitter
|
* allocated with it, and thus remain in the lzma_memory_limiter
|
||||||
* even after they get exported to the application via *header
|
* even after they get exported to the application via *header
|
||||||
* and *footer pointers.
|
* and *footer pointers.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -31,7 +31,7 @@ libcommon_la_SOURCES = \
|
||||||
index.c \
|
index.c \
|
||||||
info.c \
|
info.c \
|
||||||
init.c \
|
init.c \
|
||||||
memory_limitter.c \
|
memory_limiter.c \
|
||||||
memory_usage.c \
|
memory_usage.c \
|
||||||
next_coder.c \
|
next_coder.c \
|
||||||
raw_common.c \
|
raw_common.c \
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
///////////////////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////////////////
|
||||||
//
|
//
|
||||||
/// \file memory_limitter.c
|
/// \file memory_limiter.c
|
||||||
/// \brief Limitting memory usage
|
/// \brief Limitting memory usage
|
||||||
//
|
//
|
||||||
// Copyright (C) 2007 Lasse Collin
|
// Copyright (C) 2007 Lasse Collin
|
|
@ -328,9 +328,9 @@ parse_stream_tail(listing_handle *handle)
|
||||||
static void
|
static void
|
||||||
list_native(listing_handle *handle)
|
list_native(listing_handle *handle)
|
||||||
{
|
{
|
||||||
lzma_memory_limitter *limitter
|
lzma_memory_limiter *limiter
|
||||||
= lzma_memory_limitter_create(opt_memory);
|
= lzma_memory_limiter_create(opt_memory);
|
||||||
if (limitter == NULL) {
|
if (limiter == NULL) {
|
||||||
errmsg(V_ERROR,
|
errmsg(V_ERROR,
|
||||||
}
|
}
|
||||||
lzma_info *info =
|
lzma_info *info =
|
||||||
|
|
|
@ -482,15 +482,15 @@ main(int argc, char **argv)
|
||||||
|
|
||||||
lzma_init_decoder();
|
lzma_init_decoder();
|
||||||
|
|
||||||
lzma_memlimit *mem_limitter = lzma_memlimit_create(mem_limit);
|
lzma_memlimit *mem_limiter = lzma_memlimit_create(mem_limit);
|
||||||
if (mem_limitter == NULL) {
|
if (mem_limiter == NULL) {
|
||||||
fprintf(stderr, "%s: %s\n", argv0, strerror(ENOMEM));
|
fprintf(stderr, "%s: %s\n", argv0, strerror(ENOMEM));
|
||||||
exit(ERROR);
|
exit(ERROR);
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(lzma_memlimit_count(mem_limitter) == 0);
|
assert(lzma_memlimit_count(mem_limiter) == 0);
|
||||||
|
|
||||||
allocator.opaque = mem_limitter;
|
allocator.opaque = mem_limiter;
|
||||||
strm.allocator = &allocator;
|
strm.allocator = &allocator;
|
||||||
|
|
||||||
#ifdef WIN32
|
#ifdef WIN32
|
||||||
|
@ -528,8 +528,8 @@ main(int argc, char **argv)
|
||||||
// Free the memory only when debugging. Freeing wastes some time,
|
// Free the memory only when debugging. Freeing wastes some time,
|
||||||
// but allows detecting possible memory leaks with Valgrind.
|
// but allows detecting possible memory leaks with Valgrind.
|
||||||
lzma_end(&strm);
|
lzma_end(&strm);
|
||||||
assert(lzma_memlimit_count(mem_limitter) == 0);
|
assert(lzma_memlimit_count(mem_limiter) == 0);
|
||||||
lzma_memlimit_end(mem_limitter, false);
|
lzma_memlimit_end(mem_limiter, false);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
return exit_status;
|
return exit_status;
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
///////////////////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////////////////
|
||||||
//
|
//
|
||||||
/// \file test_memlimit.c
|
/// \file test_memlimit.c
|
||||||
/// \brief Tests the memory usage limitter
|
/// \brief Tests the memory usage limiter
|
||||||
///
|
///
|
||||||
/// \note These tests cannot be done at exact byte count accuracy,
|
/// \note These tests cannot be done at exact byte count accuracy,
|
||||||
/// because memory limitter takes into account the memory wasted
|
/// because memory limiter takes into account the memory wasted
|
||||||
/// by bookkeeping structures and alignment (padding).
|
/// by bookkeeping structures and alignment (padding).
|
||||||
//
|
//
|
||||||
// Copyright (C) 2008 Lasse Collin
|
// Copyright (C) 2008 Lasse Collin
|
||||||
|
|
Loading…
Reference in New Issue