blob: c5a9a8ac3d1c2ddbf47021b0da89c3d464e7587e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
#!/usr/bin/env sh
# 'local' isn't POSIX-compliant, but in practice is available in
# implementations of sh that we care about.
# shellcheck disable=SC3043
set -eu
SCRIPT_NAME=$(basename "$0")
usage ()
{
cat <<EOF
${SCRIPT_NAME}: Find parallelism limit based on memory and CPUs available.
Requires some upper bound (i.e. reasonable guesstimate) of how much RAM each job needs.
Takes into account the physical memory available on the machine, and cgroup limits.
Reports but doesn't take into account how much memory appears to be currently
available.
The result is printed as a single integer to stdout. Additional information is
printed to stderr.
Usage:
$SCRIPT_NAME MAX_MB_RAM_per_job : Calculate max parallelism for given max-MB-RAM-per-job
Options:
-h: Print this message.
EOF
}
# echo to stderr
eecho () {
>&2 echo "$*"
}
min () {
local lhs="$1"
local rhs="$2"
echo $(( lhs < rhs ? lhs : rhs ))
}
max () {
local lhs="$1"
local rhs="$2"
echo $(( lhs > rhs ? lhs : rhs ))
}
is_numeric () {
case "$1" in
''|*[!0-9]*) return 1;;
*) return 0;;
esac
}
while getopts "h" opt ; do
case "$opt" in
h) usage
exit 0
;;
*) echo "Unknown option."
exit 1
;;
esac
done
# Remove the flags we parsed.
shift $((OPTIND-1))
if [ $# -ne 1 ] || ! is_numeric "$1";
then
usage
exit 1
fi
max_mb_per_job="$1"
# Follow the cgroup hierarchy and return the effective memory limit,
# which is the minimum of the limits at each level of the hierarchy.
get_cgroup_memmax_MB () {
local cgroup_root
cgroup_root=/sys/fs/cgroup
local cgroup
cgroup=$(cut -d : -f3 < /proc/self/cgroup)
local next_cgroup_dir
next_cgroup_dir="$cgroup_root$cgroup"
local res="unlimited"
while true
do
case "$next_cgroup_dir" in
"$cgroup_root"*) : ;;
*) break;;
esac
local cgroup_dir="$next_cgroup_dir"
next_cgroup_dir=$(dirname "$next_cgroup_dir")
local f="${cgroup_dir}memory.max"
if [ ! -f "$f" ]
then
continue
fi
local this_max
this_max=$(cat "$f")
if ! is_numeric "$this_max"
then
continue
fi
eecho "found cgroup limit $cgroup_dir: $this_max"
if ! is_numeric "$res"
then
res=$(( this_max / 1024 / 1024 ))
fi
res=$(min "$this_max" "$res")
done
echo "$res"
}
get_meminfo_available_MB () {
local x
x="$(grep ^MemAvailable /proc/meminfo | awk '{ print $2; }')"
echo $(( x / 1024 ))
}
get_meminfo_total_MB () {
local x
x="$(grep ^MemTotal /proc/meminfo | awk '{ print $2; }')"
echo $(( x / 1024 ))
}
get_max_mem_MB () {
# Report available, but for now we don't use it since it may give
# inconsistent results. e.g. we don't want to severely limit parallelism if
# something happens to be briefly using a lot of memory at the moment.
local meminfo_available
meminfo_available=$(get_meminfo_available_MB)
eecho "MemAvailable: $meminfo_available MB"
local meminfo_total
meminfo_total=$(get_meminfo_total_MB)
eecho "MemTotal: $meminfo_total MB"
local cgroup_memmax
cgroup_memmax=$(get_cgroup_memmax_MB)
eecho "cgroup limit: $cgroup_memmax MB"
if ! is_numeric "$cgroup_memmax"
then
# No cgroup limit; fall back to total
echo "$meminfo_total"
else
# Report smaller of total or cgroup limit
min "${meminfo_total}" "${cgroup_memmax}"
fi
}
get_parallelism_limit() {
local mb_per_job="$1"
local max_mem_MB
max_mem_MB=$(get_max_mem_MB)
local mem_based_limit=$(( max_mem_MB / mb_per_job ))
eecho "memory based parallelism limit: $mem_based_limit"
local cpu_based_limit
cpu_based_limit=$(nproc)
eecho "cpu based parallelism limit: $cpu_based_limit"
max 1 "$(min "$mem_based_limit" "$cpu_based_limit")"
}
get_parallelism_limit "$max_mb_per_job"
|
