1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# @ECLASS: multiprocessing.eclass
# @MAINTAINER:
# base-system@gentoo.org
# @AUTHOR:
# Brian Harring <ferringb@gentoo.org>
# Mike Frysinger <vapier@gentoo.org>
# @SUPPORTED_EAPIS: 5 6 7 8
# @BLURB: multiprocessing helper functions
# @DESCRIPTION:
# The multiprocessing eclass contains a suite of utility functions
# that could be helpful to controlling parallel multiple job execution.
# The most common use is processing MAKEOPTS in order to obtain job
# count.
#
# @EXAMPLE:
#
# @CODE
# src_compile() {
# # custom build system that does not support most of MAKEOPTS
# ./mybs -j$(makeopts_jobs)
# }
# @CODE
case ${EAPI:-0} in
[5678]) ;;
*) die "${ECLASS}: EAPI ${EAPI:-0} not supported" ;;
esac
if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then
_MULTIPROCESSING_ECLASS=1
# @FUNCTION: get_nproc
# @USAGE: [${fallback:-1}]
# @DESCRIPTION:
# Attempt to figure out the number of processing units available.
# If the value can not be determined, prints the provided fallback
# instead. If no fallback is provided, defaults to 1.
get_nproc() {
local nproc
# GNU
if type -P nproc &>/dev/null; then
nproc=$(nproc)
fi
# BSD
if [[ -z ${nproc} ]] && type -P sysctl &>/dev/null; then
nproc=$(sysctl -n hw.ncpu 2>/dev/null)
fi
# fallback to python2.6+
# note: this may fail (raise NotImplementedError)
if [[ -z ${nproc} ]] && type -P python &>/dev/null; then
nproc=$(python -c 'import multiprocessing; print(multiprocessing.cpu_count());' 2>/dev/null)
fi
if [[ -n ${nproc} ]]; then
echo "${nproc}"
else
echo "${1:-1}"
fi
}
# @FUNCTION: _get_all_makeopts
# @INTERNAL
# @DESCRIPTION:
# Returns ${MAKEOPTS} ${GNUMAKEFLAGS} ${MAKEFLAGS}.
_get_all_makeopts() {
echo "${MAKEOPTS} ${GNUMAKEFLAGS} ${MAKEFLAGS}"
}
# @FUNCTION: get_makeopts_jobs
# @USAGE: [default-jobs]
# @DESCRIPTION:
# Return the number of jobs extracted from the make options (MAKEOPTS,
# GNUMAKEFLAGS, MAKEFLAGS). If the make options do not specify a number,
# then either the provided default is returned, or 1.
get_makeopts_jobs() {
makeopts_jobs "$(_get_all_makeopts)" "${1:-1}"
}
# @FUNCTION: makeopts_jobs
# @USAGE: [${MAKEOPTS}] [${inf:-$(( $(get_nproc) + 1 ))}]
# @DESCRIPTION:
# Searches the arguments (or sensible defaults) and extracts the jobs number
# specified therein. Useful for running non-make tools in parallel too.
# i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
# number as bash normalizes it to [0, 255]. If the flags haven't specified a
# -j flag, then "1" is shown as that is the default `make` uses. If the flags
# specify -j without a number, ${inf} is returned (defaults to nproc).
makeopts_jobs() {
[[ $# -eq 0 ]] && set -- "$(_get_all_makeopts)"
# This assumes the first .* will be more greedy than the second .*
# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
local jobs=$(echo " $* " | sed -r -n \
-e 's:.*[[:space:]](-[a-z]*j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
-e "s:.*[[:space:]](-[a-z]*j|--jobs)[[:space:]].*:${2:-$(( $(get_nproc) + 1 ))}:p")
echo ${jobs:-1}
}
# @FUNCTION: get_makeopts_loadavg
# @USAGE: [default-loadavg]
# @DESCRIPTION:
# Return the value for the load-average extracted from the make options (MAKEOPTS,
# GNUMAKEFLAGS, MAKEFLAGS). If the make options do not specify a value, then
# either the optional provided default is returned, or 999.
get_makeopts_loadavg() {
makeopts_loadavg "$(_get_all_makeopts)" "${1:-999}"
}
# @FUNCTION: makeopts_loadavg
# @USAGE: [${MAKEOPTS}] [${inf:-999}]
# @DESCRIPTION:
# Searches the arguments (or sensible defaults) and extracts the value set
# for load-average. For make and ninja based builds this will mean new jobs are
# not only limited by the jobs-value, but also by the current load - which might
# get excessive due to I/O and not just due to CPU load.
# Be aware that the returned number might be a floating-point number. Test
# whether your software supports that.
# If no limit is specified or --load-average is used without a number, ${inf}
# (defaults to 999) is returned.
makeopts_loadavg() {
[[ $# -eq 0 ]] && set -- "$(_get_all_makeopts)"
# This assumes the first .* will be more greedy than the second .*
# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
local lavg=$(echo " $* " | sed -r -n \
-e 's:.*[[:space:]](-[a-z]*l|--(load-average|max-load)[=[:space:]])[[:space:]]*([0-9]+(\.[0-9]+)?)[[:space:]].*:\3:p' \
-e "s:.*[[:space:]](-[a-z]*l|--(load-average|max-load))[[:space:]].*:${2:-999}:p")
# Default to ${inf} since the default is to not use a load limit.
echo ${lavg:-${2:-999}}
}
fi
|
