d6
/
xz-analysis-mirror
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleanups to the code that detects the amount of RAM and 

the number of CPU cores. Added support for using sysinfo()
on Linux systems whose libc lacks appropriate sysconf()
support (at least dietlibc). The Autoconf macros were
split into separate files, and CPU core count detection
was moved from hardware.c to cpucores.h. The core count
isn't used for anything real for now, so a problematic
part in process.c was commented out.
This commit is contained in:
Lasse Collin 2009-02-14 00:45:29 +02:00
parent 9c62371eab
commit 3084d662d2
10 changed files with 235 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
configure.ac
View File

@ -564,93 +564,8 @@ gl_GETOPT
# Find the best function to set timestamps.
AC_CHECK_FUNCS([futimens futimes futimesat utimes utime], [break])
# Check how to find out the amount of physical memory in the system. The
# xz command line tool uses this to automatically limit its memory usage.
# - sysconf() gives all the needed info on GNU+Linux and Solaris.
# - BSDs use sysctl().
AC_MSG_CHECKING([how to detect the amount of physical memory])
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <unistd.h>
int
main()
{
long i;
i = sysconf(_SC_PAGESIZE);
i = sysconf(_SC_PHYS_PAGES);
return 0;
}
]])], [
AC_DEFINE([HAVE_PHYSMEM_SYSCONF], [1],
[Define to 1 if the amount of physical memory can be detected
with sysconf(_SC_PAGESIZE) and sysconf(_SC_PHYS_PAGES).])
AC_MSG_RESULT([sysconf])
], [
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <sys/types.h>
#ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
#endif
#include <sys/sysctl.h>
int
main()
{
int name[2] = { CTL_HW, HW_PHYSMEM };
unsigned long mem;
size_t mem_ptr_size = sizeof(mem);
sysctl(name, 2, &mem, &mem_ptr_size, NULL, NULL);
return 0;
}
]])], [
AC_DEFINE([HAVE_PHYSMEM_SYSCTL], [1],
[Define to 1 if the amount of physical memory can be detected
with sysctl().])
AC_MSG_RESULT([sysctl])
], [
AC_MSG_RESULT([unknown])
])])
# Check how to find out the number of available CPU cores in the system.
# sysconf(_SC_NPROCESSORS_ONLN) works on most systems, except that BSDs
# use sysctl().
AC_MSG_CHECKING([how to detect the number of available CPU cores])
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <unistd.h>
int
main()
{
long i;
i = sysconf(_SC_NPROCESSORS_ONLN);
return 0;
}
]])], [
AC_DEFINE([HAVE_NCPU_SYSCONF], [1],
[Define to 1 if the number of available CPU cores can be
detected with sysconf(_SC_NPROCESSORS_ONLN).])
AC_MSG_RESULT([sysconf])
], [
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <sys/types.h>
#ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
#endif
#include <sys/sysctl.h>
int
main()
{
int name[2] = { CTL_HW, HW_NCPU };
int cpus;
size_t cpus_size = sizeof(cpus);
sysctl(name, 2, &cpus, &cpus_size, NULL, NULL);
return 0;
}
]])], [
AC_DEFINE([HAVE_NCPU_SYSCTL], [1],
[Define to 1 if the number of available CPU cores can be
detected with sysctl().])
AC_MSG_RESULT([sysctl])
], [
AC_MSG_RESULT([unknown])
])])
lc_PHYSMEM
lc_CPUCORES
###############################################################################

57
m4/lc_cpucores.m4 Normal file
View File

@ -0,0 +1,57 @@
dnl ###########################################################################
dnl
dnl lc_CPUCORES - Check how to find out the number of online CPU cores
dnl
dnl Check how to find out the number of available CPU cores in the system.
dnl sysconf(_SC_NPROCESSORS_ONLN) works on most systems, except that BSDs
dnl use sysctl().
dnl
dnl ###########################################################################
dnl
dnl Author: Lasse Collin
dnl
dnl This file has been put into the public domain.
dnl You can do whatever you want with this file.
dnl
dnl ###########################################################################
AC_DEFUN([lc_CPUCORES], [
AC_MSG_CHECKING([how to detect the number of available CPU cores])
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <unistd.h>
int
main(void)
{
long i;
i = sysconf(_SC_NPROCESSORS_ONLN);
return 0;
}
]])], [
AC_DEFINE([HAVE_NCPU_SYSCONF], [1],
[Define to 1 if the number of available CPU cores can be
detected with sysconf(_SC_NPROCESSORS_ONLN).])
AC_MSG_RESULT([sysconf])
], [
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <sys/types.h>
#ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
#endif
#include <sys/sysctl.h>
int
main(void)
{
int name[2] = { CTL_HW, HW_NCPU };
int cpus;
size_t cpus_size = sizeof(cpus);
sysctl(name, 2, &cpus, &cpus_size, NULL, NULL);
return 0;
}
]])], [
AC_DEFINE([HAVE_NCPU_SYSCTL], [1],
[Define to 1 if the number of available CPU cores can be
detected with sysctl().])
AC_MSG_RESULT([sysctl])
], [
AC_MSG_RESULT([unknown])
])])
])dnl lc_CPUCORES

74
m4/lc_physmem.m4 Normal file
View File

@ -0,0 +1,74 @@
dnl ###########################################################################
dnl
dnl lc_PHYSMEM - Check how to find out the amount of physical memory
dnl
dnl - sysconf() gives all the needed info on GNU+Linux and Solaris.
dnl - BSDs use sysctl().
dnl - sysinfo() works on Linux/dietlibc and probably on other Linux systems
dnl whose libc may lack sysconf().
dnl
dnl ###########################################################################
dnl
dnl Author: Lasse Collin
dnl
dnl This file has been put into the public domain.
dnl You can do whatever you want with this file.
dnl
dnl ###########################################################################
AC_DEFUN([lc_PHYSMEM], [
AC_MSG_CHECKING([how to detect the amount of physical memory])
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <unistd.h>
int
main(void)
{
long i;
i = sysconf(_SC_PAGESIZE);
i = sysconf(_SC_PHYS_PAGES);
return 0;
}
]])], [
AC_DEFINE([HAVE_PHYSMEM_SYSCONF], [1],
[Define to 1 if the amount of physical memory can be detected
with sysconf(_SC_PAGESIZE) and sysconf(_SC_PHYS_PAGES).])
AC_MSG_RESULT([sysconf])
], [
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <sys/types.h>
#ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
#endif
#include <sys/sysctl.h>
int
main(void)
{
int name[2] = { CTL_HW, HW_PHYSMEM };
unsigned long mem;
size_t mem_ptr_size = sizeof(mem);
sysctl(name, 2, &mem, &mem_ptr_size, NULL, NULL);
return 0;
}
]])], [
AC_DEFINE([HAVE_PHYSMEM_SYSCTL], [1],
[Define to 1 if the amount of physical memory can be detected
with sysctl().])
AC_MSG_RESULT([sysctl])
], [
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
#include <sys/sysinfo.h>
int
main(void)
{
struct sysinfo si;
sysinfo(&si);
return 0;
}
]])], [
AC_DEFINE([HAVE_PHYSMEM_SYSINFO], [1],
[Define to 1 if the amount of physical memory can be detected
with sysinfo().])
AC_MSG_RESULT([sysinfo])
], [
AC_MSG_RESULT([unknown])
])])])
])dnl lc_PHYSMEM

52
src/common/cpucores.h Normal file
View File

@ -0,0 +1,52 @@
///////////////////////////////////////////////////////////////////////////////
//
/// \file cpucores.h
/// \brief Get the number of online CPU cores
//
// This code has been put into the public domain.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
///////////////////////////////////////////////////////////////////////////////
#ifndef CPUCORES_H
#define CPUCORES_H
#if defined(HAVE_NCPU_SYSCONF)
# include <unistd.h>
#elif defined(HAVE_NCPU_SYSCTL)
# ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
# endif
# ifdef HAVE_SYS_SYSCTL_H
# include <sys/sysctl.h>
# endif
#endif
static inline uint32_t
cpucores(void)
{
uint32_t ret = 0;
#if defined(HAVE_CPUCORES_SYSCONF)
const long cpus = sysconf(_SC_NPROCESSORS_ONLN);
if (cpus > 0)
ret = (uint32_t)(cpus);
#elif defined(HAVE_CPUCORES_SYSCTL)
int name[2] = { CTL_HW, HW_NCPU };
int cpus;
size_t cpus_size = sizeof(cpus);
if (!sysctl(name, &cpus, &cpus_size, NULL, NULL)
&& cpus_size == sizeof(cpus) && cpus > 0)
ret = (uint32_t)(cpus);
#endif
return ret;
}
#endif

21
src/common/physmem.h
View File

@ -14,27 +14,27 @@
#ifndef PHYSMEM_H
#define PHYSMEM_H
#if defined(HAVE_PHYSMEM_SYSCTL) || defined(HAVE_NCPU_SYSCTL)
#if defined(HAVE_PHYSMEM_SYSCONF)
# include <unistd.h>
#elif defined(HAVE_PHYSMEM_SYSCTL)
# ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
# endif
# ifdef HAVE_SYS_SYSCTL_H
# include <sys/sysctl.h>
# endif
#endif
#if defined(HAVE_PHYSMEM_SYSCONF) || defined(HAVE_NCPU_SYSCONF)
# include <unistd.h>
#endif
#elif defined(HAVE_PHYSMEM_SYSINFO)
# include <sys/sysinfo.h>
#ifdef _WIN32
#elif defined(_WIN32)
# ifndef _WIN32_WINNT
# define _WIN32_WINNT 0x0500
# endif
# include <windows.h>
#endif
#ifdef __DJGPP__
#elif defined(__DJGPP__)
# include <dpmi.h>
#endif
@ -75,6 +75,11 @@ physmem(void)
ret = mem.ui;
}
#elif defined(HAVE_PHYSMEM_SYSINFO)
struct sysinfo si;
if (sysinfo(&si) == 0)
ret = (uint64_t)(si.totalram) * si.mem_unit;
#elif defined(_WIN32)
MEMORYSTATUSEX meminfo;
meminfo.dwLength = sizeof(meminfo);

4
src/xz/args.c
View File

@ -135,8 +135,8 @@ parse_real(args_info *args, int argc, char **argv)
break;
case 'T':
opt_threads = str_to_uint64("threads", optarg,
1, SIZE_MAX);
hardware_threadlimit_set(str_to_uint64(
"threads", optarg, 1, SIZE_MAX));
break;
// --version

50
src/xz/hardware.c
View File

@ -19,11 +19,12 @@
#include "private.h"
#include "physmem.h"
#include "cpucores.h"
/// Maximum number of free *coder* threads. This can be set with
/// the --threads=NUM command line option.
size_t opt_threads = 1;
static uint32_t threads_max;
/// Memory usage limit for encoding
@ -40,38 +41,31 @@ static uint64_t memlimit_custom = 0;
/// Get the number of CPU cores, and set opt_threads to default to that value.
/// User can then override this with --threads command line option.
static void
hardware_cores(void)
hardware_threadlimit_init(void)
{
#if defined(HAVE_NUM_PROCESSORS_SYSCONF)
const long cpus = sysconf(_SC_NPROCESSORS_ONLN);
if (cpus > 0)
opt_threads = (size_t)(cpus);
#elif defined(HAVE_NUM_PROCESSORS_SYSCTL)
int name[2] = { CTL_HW, HW_NCPU };
int cpus;
size_t cpus_size = sizeof(cpus);
if (!sysctl(name, &cpus, &cpus_size, NULL, NULL)
&& cpus_size == sizeof(cpus) && cpus > 0)
opt_threads = (size_t)(cpus);
#endif
// Limit opt_threads so that maximum number of threads doesn't exceed.
#if defined(_SC_THREAD_THREADS_MAX)
const long threads_max = sysconf(_SC_THREAD_THREADS_MAX);
if (threads_max > 0 && (size_t)(threads_max) < opt_threads)
opt_threads = (size_t)(threads_max);
#elif defined(PTHREAD_THREADS_MAX)
if (opt_threads > PTHREAD_THREADS_MAX)
opt_threads = PTHREAD_THREADS_MAX;
#endif
threads_max = cpucores();
if (threads_max == 0)
threads_max = 1;
return;
}
extern void
hardware_threadlimit_set(uint32_t threadlimit)
{
threads_max = threadlimit;
return;
}
extern uint32_t
hardware_threadlimit_get(void)
{
return threads_max;
}
static void
hardware_memlimit_init(void)
{
@ -117,6 +111,6 @@ extern void
hardware_init(void)
{
hardware_memlimit_init();
hardware_cores();
hardware_threadlimit_init();
return;
}

11
src/xz/hardware.h
View File

@ -17,14 +17,19 @@
//
///////////////////////////////////////////////////////////////////////////////
extern size_t opt_threads;
/// Initialize some hardware-specific variables, which are needed by other
/// hardware_* functions.
extern void hardware_init(void);
/// Set custom value for maximum number of coder threads.
extern void hardware_threadlimit_set(uint32_t threadlimit);
/// Get the maximum number of coder threads. Some additional helper threads
/// are allowed on top of this).
extern uint32_t hardware_threadlimit_get(void);
/// Set custom memory usage limit. This is used for both encoding and
/// decoding. Zero indicates resetting the limit back to defaults.
extern void hardware_memlimit_set(uint64_t memlimit);

5
src/xz/message.c
View File

@ -1029,8 +1029,9 @@ message_help(bool long_help)
hardware_memlimit_encoder() / (1024 * 1024),
hardware_memlimit_decoder() / (1024 * 1024));
printf(N_(" * one thread for (de)compression.\n\n",
" * %'" PRIu64 " threads for (de)compression.\n\n",
(uint64_t)(opt_threads)), (uint64_t)(opt_threads));
" * %'" PRIu32 " threads for (de)compression.\n\n",
hardware_threadlimit_get()),
hardware_threadlimit_get());
}
printf(_("Report bugs to <%s> (in English or Finnish).\n"),

2
src/xz/process.c
View File

@ -246,6 +246,7 @@ coder_set_compression_settings(void)
memory_limit >> 20);
}
/*
// Limit the number of worker threads so that memory usage
// limit isn't exceeded.
assert(memory_usage > 0);
@ -255,6 +256,7 @@ coder_set_compression_settings(void)
if (opt_threads > thread_limit)
opt_threads = thread_limit;
*/
return;
}