1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
|
#!/usr/bin/env python3
"""
Run integration-e2e inside the shadow network simulator. This can be helpful
vs running it directly for several reasons.
* shadow simulates time, and can collapse idle time. This speeds up the
network bootstrapping step in particular.
* shadow tries to be deterministic. There are some gaps, but in general
there *should* be less nondeterministic flakiness under shadow
than when running natively.
"""
import atexit
import argparse
import shadowtools.config as scfg
import shadowtools.shadow_exec as shadow_exec
import os
import pathlib
import subprocess
import sys
import yaml
from pathlib import Path
from chutney import TorNet
from typing import Final
import common
from config import Config
assert __name__ == "__main__", "Can't determine _SCRIPT_DIR"
_SCRIPT_DIR = Path(sys.argv[0]).parent.resolve()
def _shadow_config_path(n: TorNet.Network) -> Path:
return n.dir.joinpath("shadow.yaml")
def _integration_e2e(args: argparse.Namespace) -> None:
"""Run the test (from inside shadow)"""
config = Config.load_json(Path(_SCRIPT_DIR).joinpath("arti.run.json"))
config.export_env()
# Try to ensure we tear down the network on exit, including failure, ctrl-c, etc.
atexit.register(lambda: subprocess.check_call([_SCRIPT_DIR.joinpath("teardown")]))
# bootstrap the network
subprocess.check_call([config.chutney, "bootstrap"])
# test the network
subprocess.check_call([_SCRIPT_DIR.joinpath("test"), "-v"])
def gen_shadow_config(*, seed: int, controller_hostname: str) -> scfg.Config:
"""
Generate a shadow config file, as a string, for the given parameters.
"""
current_script = os.path.abspath(__file__)
env = {
common.RUNNING_IN_SHADOW_ENV: "yes",
# re-export PATH. The test scripts assume that
# usual shell utilities are on it.
"PATH": os.getenv("PATH", ""),
}
return scfg.Config(
general=scfg.General(
stop_time="10m",
model_unblocked_syscall_latency=True,
seed=seed,
),
network=scfg.Network(
graph=scfg.Graph(type="1_gbit_switch"),
),
experimental=scfg.Experimental(
# shadow only actually increments simulated time (and potentially
# switches threads) if this much time would have been consumed by an
# unbroken sequence of unblocked syscalls. Using a relatively large
# value here (vs the default 1us) makes the simulation scheduling
# more stable and predictable; e.g. adding additional logging to
# debug an issue is less likely to make the issue disappear.
#
# The primary tradeoffs are:
# * Larger values can result in managed processes measuring elapsed
# time where not much happens as *zero*, which may not be handled
# gracefully. e.g. in c-tor, using values of 1 ms or more here can
# result in a flood of warnings "compute_drain_rate(): Bug:
# Computing stream drain rate with zero time delta".
# * Time will move forward at a larger granularity when unblocked syscall
# latency is applied. 10ms is still small enough though that this
# shouldn't be terribly strange; e.g. larger time jumps are likely
# to be observed on over-loaded systems with normal preemptive
# scheduling.
# * when the simulation does hit a
# busy loop, it may spend a bit longer "spinning" before moving
# time forward, potentially causing the simulation to take a bit
# longer to run. (if it would have otherwise timed out earlier than 10ms)
max_unapplied_cpu_latency="100us",
# In CI, attempting to pin to particular CPU cores may result in
# conflicts with other instances of shadow trying to do the same
# thing.
use_cpu_pinning=False,
# Likewise, shadow's default behavior of spin-looping is bad
# behavior in a shared environment.
use_worker_spinning=False,
),
hosts={
controller_hostname: scfg.Host(
network_node_id=0,
processes=[
scfg.Process(
path="sh",
args=f"-c '{current_script} integration-e2e 2>&1'",
environment=env,
# Give the web server below a little time to start.
start_time="5s",
)
],
),
common.TEST_DOMAIN: scfg.Host(
network_node_id=0,
processes=[
scfg.Process(
path="python3",
args="-m http.server 80",
start_time=0,
expected_final_state="running",
)
],
),
},
)
def _configure_and_run_shadow(args: argparse.Namespace) -> None:
toplevel = pathlib.Path(
os.fsdecode(
subprocess.check_output("git rev-parse --show-toplevel", shell=True)
).strip()
)
os.chdir(toplevel)
# configure this test
subprocess.check_call([_SCRIPT_DIR.joinpath("setup")])
config = Config.load_json(Path(_SCRIPT_DIR).joinpath("arti.run.json"))
config.export_env()
# initialize the network
subprocess.check_call(
[_SCRIPT_DIR.joinpath("init")],
env=(
os.environ
| dict(
# AF_UNIX sockets aren't supported in shadow
CHUTNEY_ENABLE_CONTROLSOCKET="no",
# ipv6 isn't supported in shadow
CHUTNEY_DISABLE_IPV6="yes",
# sandboxing isn't supported in shadow
CHUTNEY_TOR_SANDBOX="no",
)
),
)
# Load the network we just initialized
network = TorNet.Network.from_data_dir()
# Write out shadow config. We could just pipe it directly to the shadow
# process below, but writing it out is useful for debugging.
controller_hostname: Final = "host"
shadow_config = gen_shadow_config(
seed=args.seed, controller_hostname=controller_hostname
)
with _shadow_config_path(network).open("w") as f:
f.write(yaml.safe_dump(shadow_config))
shadow_exec.run_shadow_watching_process(
watch_host=controller_hostname,
shadow_bin=Path("shadow"),
dstdir=network.dir,
shadow_config_path=_shadow_config_path(network),
)
def main() -> None:
parser = argparse.ArgumentParser(
prog="integration-e2e-shadow",
description="Runs integration-e2e inside a shadow simulation",
)
parser.add_argument(
"-s", "--seed", type=int, default=1, help="Simulation PRNG seed"
)
parser.set_defaults(func=_configure_and_run_shadow)
subparsers = parser.add_subparsers()
integration_e2e_parser = subparsers.add_parser(
"integration-e2e",
help=(
"Run the e2e integration test, from inside shadow."
" Intended only for this script to recursively launch itself inside shadow."
),
)
integration_e2e_parser.set_defaults(func=_integration_e2e)
args = parser.parse_args()
args.func(args)
if __name__ == "__main__":
main()
|
