line |
stmt |
code |
1
|
|
#!/usr/bin/perl -w |
2
|
|
# Copyright 2009-2013 Bernhard M. Wiedemann |
3
|
|
# Copyright 2012-2020 SUSE LLC |
4
|
|
# SPDX-License-Identifier: GPL-2.0-or-later |
5
|
|
# |
6
|
|
|
7
|
|
=head1 SYNOPSIS |
8
|
|
|
9
|
|
isotovideo [OPTIONS] [TEST PARAMETER] |
10
|
|
|
11
|
|
Parses command line parameters, vars.json and tests the given assets/ISOs. |
12
|
|
|
13
|
|
=head1 OPTIONS |
14
|
|
|
15
|
|
=over 4 |
16
|
|
|
17
|
|
=item B<-d, --debug> |
18
|
|
|
19
|
|
Enable direct output to STDERR instead of autoinst-log.txt |
20
|
|
|
21
|
|
=item B<--workdir=> |
22
|
|
|
23
|
|
isotovideo will chdir to that directory on startup |
24
|
|
|
25
|
|
=item B<-v, --version> |
26
|
|
|
27
|
|
Show the current program version and test API version |
28
|
|
|
29
|
|
=item B<-h, -?, --help> |
30
|
|
|
31
|
|
Show this help. |
32
|
|
|
33
|
|
=head1 TEST PARAMETER |
34
|
|
|
35
|
|
All additional command line arguments specified in the C<key=value> format are |
36
|
|
parsed as test parameters which take precedence over the settings in the |
37
|
|
vars.json file. Lower case key names are transformed into upper case |
38
|
|
automatically for convenience. |
39
|
|
|
40
|
|
=cut |
41
|
|
|
42
|
28
|
use Mojo::Base -strict, -signatures; |
|
28
|
|
|
28
|
|
43
|
28
|
use autodie ':all'; |
|
28
|
|
|
28
|
|
44
|
28
|
no autodie 'kill'; |
|
28
|
|
|
28
|
|
45
|
|
|
46
|
|
# Avoid "Subroutine JSON::PP::Boolean::(0+ redefined" warnings |
47
|
|
# Details: https://progress.opensuse.org/issues/90371 |
48
|
28
|
use JSON::PP; |
|
28
|
|
|
28
|
|
49
|
|
|
50
|
28
|
my $installprefix; # $bmwqemu::scriptdir |
51
|
|
my $fatal_error; # the last error message caught by the die handler |
52
|
|
|
53
|
|
BEGIN { |
54
|
|
# the following line is modified during make install |
55
|
28
|
$installprefix = undef; |
56
|
|
|
57
|
28
|
my ($wd) = $0 =~ m-(.*)/-; |
58
|
28
|
$wd ||= '.'; |
59
|
28
|
$installprefix ||= $wd; |
60
|
28
|
unshift @INC, "$installprefix"; |
61
|
|
} |
62
|
|
|
63
|
28
|
use backend::driver; |
|
28
|
|
|
28
|
|
64
|
28
|
use log qw(diag fctwarn); |
|
28
|
|
|
28
|
|
65
|
28
|
use needle; |
|
28
|
|
|
28
|
|
66
|
28
|
use autotest (); |
|
28
|
|
|
28
|
|
67
|
28
|
use commands; |
|
28
|
|
|
28
|
|
68
|
28
|
use distribution; |
|
28
|
|
|
28
|
|
69
|
28
|
use testapi (); |
|
28
|
|
|
28
|
|
70
|
28
|
use Getopt::Long; |
|
28
|
|
|
28
|
|
71
|
28
|
require IPC::System::Simple; |
72
|
28
|
use POSIX qw(:sys_wait_h _exit); |
|
28
|
|
|
28
|
|
73
|
28
|
use Time::HiRes qw(gettimeofday tv_interval sleep time); |
|
28
|
|
|
28
|
|
74
|
28
|
use Try::Tiny; |
|
28
|
|
|
28
|
|
75
|
28
|
use IO::Select; |
|
28
|
|
|
28
|
|
76
|
28
|
use Mojo::File qw(curfile path); |
|
28
|
|
|
28
|
|
77
|
28
|
use Mojo::UserAgent; |
|
28
|
|
|
28
|
|
78
|
28
|
use Mojo::IOLoop::ReadWriteProcess 'process'; |
|
28
|
|
|
28
|
|
79
|
28
|
use Mojo::IOLoop::ReadWriteProcess::Session 'session'; |
|
28
|
|
|
28
|
|
80
|
28
|
Getopt::Long::Configure("no_ignore_case"); |
81
|
28
|
use OpenQA::Isotovideo::CommandHandler; |
|
28
|
|
|
28
|
|
82
|
28
|
use OpenQA::Isotovideo::Interface; |
|
28
|
|
|
28
|
|
83
|
28
|
use OpenQA::Isotovideo::Utils qw(checkout_git_repo_and_branch |
84
|
28
|
checkout_git_refspec handle_generated_assets load_test_schedule); |
|
28
|
|
85
|
|
|
86
|
28
|
session->enable; |
87
|
28
|
session->enable_subreaper; |
88
|
|
|
89
|
28
|
my %options; |
90
|
|
|
91
|
1
|
eval { require Pod::Usage; Pod::Usage::pod2usage($r) }; |
|
1
|
|
|
1
|
|
92
|
1
|
die "cannot display help, install perl(Pod::Usage)\n" if $@; |
|
1
|
|
|
1
|
|
93
|
0
|
} |
94
|
|
|
95
|
|
my $dirname = curfile->dirname; |
96
|
27
|
my $thisversion = qx{git -C $dirname rev-parse HEAD}; |
|
27
|
|
97
|
27
|
chomp $thisversion; |
98
|
27
|
die 'Could not parse version' unless $thisversion; |
99
|
27
|
return "Current version is $thisversion [interface v$OpenQA::Isotovideo::Interface::version]"; |
100
|
27
|
} |
101
|
27
|
|
102
|
|
print _get_version_string() . "\n"; |
103
|
|
exit 0; |
104
|
1
|
} |
|
1
|
|
105
|
1
|
|
106
|
1
|
GetOptions(\%options, 'debug|d', 'workdir=s', 'help|h|?', 'version|v') or usage(1); |
107
|
|
usage(0) if $options{help}; |
108
|
|
version() if $options{version}; |
109
|
28
|
|
110
|
28
|
chdir $options{workdir} if $options{workdir}; |
111
|
27
|
|
112
|
|
# global exit status |
113
|
26
|
my $return_code = 1; |
114
|
|
|
115
|
|
# record the last die message |
116
|
26
|
# note: It might *not* be a fatal error so we don't call bmwqemu::serialize_state here |
117
|
|
# immediately but only in the END block. |
118
|
|
$SIG{__DIE__} = sub ($e) { $fatal_error = $e }; |
119
|
|
|
120
|
|
diag(_get_version_string()); |
121
|
26
|
|
|
29
|
|
|
29
|
|
|
29
|
|
|
29
|
|
122
|
|
# enable debug default when started from a tty |
123
|
26
|
$log::direct_output = $options{debug}; |
124
|
|
|
125
|
|
select(STDERR); |
126
|
26
|
$| = 1; |
127
|
|
select(STDOUT); # default |
128
|
26
|
$| = 1; |
129
|
26
|
|
130
|
26
|
$bmwqemu::scriptdir = $installprefix; |
131
|
26
|
bmwqemu::init(); |
132
|
|
for my $arg (@ARGV) { |
133
|
26
|
if ($arg =~ /^([[:alnum:]_\[\]\.]+)=(.+)/) { |
134
|
26
|
my $key = uc $1; |
135
|
26
|
$bmwqemu::vars{$key} = $2; |
136
|
78
|
diag("Setting forced test parameter $key -> $2"); |
137
|
78
|
} |
138
|
78
|
} |
139
|
78
|
|
140
|
|
my $cmd_srv_process; |
141
|
|
my $command_handler; |
142
|
|
my $testprocess; |
143
|
26
|
my $cmd_srv_fd; |
144
|
|
my $cmd_srv_port; |
145
|
26
|
my $backend_process; |
146
|
26
|
my $loop = 1; |
147
|
26
|
|
148
|
26
|
# note: The subsequently defined stop_* functions are used to tear down the process tree. |
149
|
26
|
# However, the worker also ensures that all processes are being terminated (and |
150
|
|
# eventually killed). |
151
|
|
|
152
|
|
return unless defined $cmd_srv_process; |
153
|
|
return unless $cmd_srv_process->is_running; |
154
|
|
|
155
|
17
|
my $pid = $cmd_srv_process->pid; |
|
17
|
|
|
17
|
|
156
|
17
|
diag("stopping command server $pid because $reason"); |
157
|
5
|
|
158
|
|
if ($cmd_srv_port && $reason && $reason eq 'test execution ended') { |
159
|
5
|
my $job_token = $bmwqemu::vars{JOBTOKEN}; |
160
|
5
|
my $url = "http://127.0.0.1:$cmd_srv_port/$job_token/broadcast"; |
161
|
|
diag('isotovideo: informing websocket clients before stopping command server: ' . $url); |
162
|
5
|
|
163
|
3
|
# note: If the job is stopped by the worker because it has been |
164
|
3
|
# aborted, the worker will send this command on its own to the command |
165
|
3
|
# server and also stop the command server. So this is only done in the |
166
|
|
# case the test execution just ends. |
167
|
|
|
168
|
|
my $timeout = 15; |
169
|
|
# The command server might have already been stopped by the worker |
170
|
|
# after the user has aborted the job or the job timeout has been |
171
|
|
# exceeded so no checks for failure done. |
172
|
3
|
Mojo::UserAgent->new(request_timeout => $timeout)->post($url, json => {stopping_test_execution => $reason}); |
173
|
|
} |
174
|
|
|
175
|
|
$cmd_srv_process->stop(); |
176
|
3
|
$cmd_srv_process = undef; |
177
|
|
diag('done with command server'); |
178
|
|
} |
179
|
5
|
|
180
|
5
|
return unless defined $testprocess; |
181
|
5
|
|
182
|
|
diag('stopping autotest process ' . $testprocess->pid); |
183
|
|
$testprocess->stop() if $testprocess->is_running; |
184
|
17
|
$testprocess = undef; |
|
17
|
|
185
|
17
|
diag('done with autotest process'); |
186
|
|
} |
187
|
5
|
|
188
|
5
|
return unless defined $bmwqemu::backend && $backend_process; |
189
|
5
|
|
190
|
5
|
diag('stopping backend process ' . $backend_process->pid); |
191
|
|
$backend_process->stop if $backend_process->is_running; |
192
|
|
$backend_process = undef; |
193
|
14
|
diag('done with backend process'); |
|
14
|
|
194
|
14
|
} |
195
|
|
|
196
|
4
|
bmwqemu::serialize_state(component => 'isotovideo', msg => "isotovideo received signal $sig", log => 1); |
197
|
4
|
return $loop = 0 if $loop; |
198
|
4
|
stop_backend; |
199
|
4
|
stop_commands("received signal $sig"); |
200
|
|
stop_autotest; |
201
|
|
_exit(1); |
202
|
0
|
} |
|
0
|
|
|
0
|
|
203
|
0
|
|
204
|
0
|
$bmwqemu::vars{BACKEND} ||= "qemu"; |
205
|
0
|
$bmwqemu::backend = backend::driver->new($bmwqemu::vars{BACKEND}); |
206
|
0
|
return $bmwqemu::backend; |
207
|
0
|
} |
208
|
0
|
|
209
|
|
$SIG{TERM} = \&signalhandler; |
210
|
|
$SIG{INT} = \&signalhandler; |
211
|
14
|
$SIG{HUP} = \&signalhandler; |
|
14
|
|
212
|
14
|
|
213
|
14
|
# make sure all commands coming from the backend will not be in the |
214
|
5
|
# developers's locale - but a defined english one. This is SUSE's |
215
|
|
# default locale |
216
|
|
$ENV{LC_ALL} = 'en_US.UTF-8'; |
217
|
26
|
$ENV{LANG} = 'en_US.UTF-8'; |
218
|
26
|
|
219
|
26
|
checkout_git_repo_and_branch('CASEDIR'); |
220
|
|
|
221
|
|
# Try to load the main.pm from one of the following in this order: |
222
|
|
# - product dir |
223
|
|
# - casedir |
224
|
26
|
# |
225
|
26
|
# This allows further structuring the test distribution collections with |
226
|
|
# multiple distributions or flavors in one repository. |
227
|
26
|
$bmwqemu::vars{PRODUCTDIR} ||= $bmwqemu::vars{CASEDIR}; |
228
|
|
|
229
|
|
# checkout Git repo NEEDLES_DIR refers to (if it is a URL) and re-assign NEEDLES_DIR to contain the checkout path |
230
|
|
checkout_git_repo_and_branch('NEEDLES_DIR'); |
231
|
|
|
232
|
|
bmwqemu::ensure_valid_vars(); |
233
|
|
|
234
|
|
# as we are about to load the test modules checkout the specified git refspec, |
235
|
25
|
# if specified, or simply store the git hash that has been used. If it is not a |
236
|
|
# git repo fail silently, i.e. store an empty variable |
237
|
|
|
238
|
25
|
$bmwqemu::vars{TEST_GIT_HASH} = checkout_git_refspec($bmwqemu::vars{CASEDIR} => 'TEST_GIT_REFSPEC'); |
239
|
|
|
240
|
25
|
# set a default distribution if the tests don't have one |
241
|
|
$testapi::distri = distribution->new; |
242
|
|
|
243
|
|
load_test_schedule; |
244
|
|
|
245
|
|
# start the command fork before we get into the backend, the command child |
246
|
25
|
# is not supposed to talk to the backend directly |
247
|
|
($cmd_srv_process, $cmd_srv_fd) = commands::start_server($cmd_srv_port = $bmwqemu::vars{QEMUPORT} + 1); |
248
|
|
|
249
|
24
|
testapi::init(); |
250
|
|
needle::init(); |
251
|
24
|
bmwqemu::save_vars(); |
252
|
|
|
253
|
|
my $testfd; |
254
|
|
($testprocess, $testfd) = autotest::start_process(); |
255
|
19
|
|
256
|
|
init_backend(); |
257
|
14
|
path('os-autoinst.pid')->spurt("$$"); |
258
|
14
|
|
259
|
14
|
if (!$bmwqemu::backend->_send_json({cmd => 'alive'})) { |
260
|
|
# might throw an exception |
261
|
14
|
$bmwqemu::backend->start_vm(); |
262
|
14
|
} |
263
|
|
|
264
|
14
|
# launch debugging tools |
265
|
5
|
my %debugging_tools; |
266
|
|
$debugging_tools{vncviewer} = ['vncviewer', '-viewonly', '-shared', "localhost:$bmwqemu::vars{VNC}"] if $ENV{RUN_VNCVIEWER}; |
267
|
5
|
$debugging_tools{debugviewer} = ["$bmwqemu::scriptdir/debugviewer/debugviewer", 'qemuscreenshot/last.png'] if $ENV{RUN_DEBUGVIEWER}; |
268
|
|
for my $tool (keys %debugging_tools) { |
269
|
5
|
next if fork() != 0; |
270
|
|
no autodie 'exec'; |
271
|
|
{ exec(@{$debugging_tools{$tool}}) }; |
272
|
|
exit -1; # don't continue in any case (exec returns if it fails to spawn the process printing an error message on its own) |
273
|
4
|
} |
274
|
4
|
|
275
|
4
|
$backend_process = $bmwqemu::backend->{backend_process}; |
276
|
4
|
my $io_select = IO::Select->new(); |
277
|
0
|
$io_select->add($testfd); |
278
|
28
|
$io_select->add($cmd_srv_fd); |
|
28
|
|
|
28
|
|
279
|
0
|
$io_select->add($backend_process->channel_out); |
|
0
|
|
|
0
|
|
280
|
0
|
|
281
|
|
# stop main loop as soon as one of the child processes terminates |
282
|
|
my $stop_loop = sub (@) { $loop = 0 if $loop; }; |
283
|
4
|
$testprocess->once(collected => $stop_loop); |
284
|
4
|
$backend_process->once(collected => $stop_loop); |
285
|
4
|
$cmd_srv_process->once(collected => $stop_loop); |
286
|
4
|
|
287
|
4
|
# now we have everything, give the tests a go |
288
|
|
$testfd->write("GO\n"); |
289
|
|
|
290
|
4
|
$command_handler = OpenQA::Isotovideo::CommandHandler->new( |
|
9
|
|
|
9
|
|
|
9
|
|
291
|
4
|
cmd_srv_fd => $cmd_srv_fd, |
292
|
4
|
backend_fd => $backend_process->channel_in, |
293
|
4
|
); |
294
|
|
$command_handler->on(tests_done => sub (@) { |
295
|
|
CORE::close($testfd); |
296
|
4
|
$testfd = undef; |
297
|
|
stop_autotest; |
298
|
4
|
$loop = 0; |
299
|
|
}); |
300
|
|
|
301
|
|
my ($last_check_seconds, $last_check_microseconds) = gettimeofday; |
302
|
3
|
# an estimate of eternity |
|
3
|
|
303
|
3
|
my $delta = $last_check_seconds ? tv_interval([$last_check_seconds, $last_check_microseconds], [gettimeofday]) : 100; |
304
|
3
|
# sleep the remains of one second if $delta > 0 |
305
|
3
|
my $timeout = $delta > 0 ? 1 - $delta : 0; |
306
|
3
|
$command_handler->timeout($timeout < 0 ? 0 : $timeout); |
307
|
4
|
return $delta; |
308
|
|
} |
309
|
4
|
|
310
|
40
|
if ($no_wait) { |
|
40
|
|
311
|
|
# prevent CPU overload by waiting at least a little bit |
312
|
40
|
$command_handler->timeout(0.1); |
313
|
|
} |
314
|
40
|
else { |
315
|
40
|
_calc_check_delta; |
316
|
40
|
# come back later, avoid too often called function |
317
|
|
return if $command_handler->timeout > 0.05; |
318
|
|
} |
319
|
49
|
($last_check_seconds, $last_check_microseconds) = gettimeofday; |
|
49
|
|
|
49
|
|
320
|
49
|
my $rsp = $bmwqemu::backend->_send_json({cmd => 'check_asserted_screen'}) || {}; |
321
|
|
# the test needs that information |
322
|
24
|
$rsp->{tags} = $command_handler->tags; |
323
|
|
if ($rsp->{found} || $rsp->{timeout}) { |
324
|
|
myjsonrpc::send_json($testfd, {ret => $rsp}); |
325
|
25
|
$command_handler->clear_tags_and_timeout(); |
326
|
|
} |
327
|
25
|
else { |
328
|
|
_calc_check_delta unless $no_wait; |
329
|
44
|
} |
330
|
44
|
} |
331
|
|
|
332
|
44
|
$return_code = 0; |
333
|
44
|
|
334
|
9
|
# enter the main loop: process messages from autotest, command server and backend |
335
|
9
|
while ($loop) { |
336
|
|
my ($ready_for_read, $ready_for_write, $exceptions) = IO::Select::select($io_select, undef, $io_select, $command_handler->timeout); |
337
|
|
for my $readable (@$ready_for_read) { |
338
|
35
|
my $rsp = myjsonrpc::read_json($readable); |
339
|
|
if (!defined $rsp) { |
340
|
|
fctwarn sprintf("THERE IS NOTHING TO READ %d %d %d", fileno($readable), fileno($testfd), fileno($cmd_srv_fd)); |
341
|
|
$readable = 1; |
342
|
4
|
$loop = 0; |
343
|
|
last; |
344
|
|
} |
345
|
4
|
if ($readable == $backend_process->channel_out) { |
346
|
289
|
$command_handler->send_to_backend_requester({ret => $rsp->{rsp}}); |
347
|
289
|
next; |
348
|
249
|
} |
349
|
249
|
$command_handler->process_command($readable, $rsp); |
350
|
0
|
} |
351
|
0
|
if (defined($command_handler->tags)) { |
352
|
0
|
check_asserted_screen($command_handler->no_wait); |
353
|
0
|
} |
354
|
|
} |
355
|
249
|
|
356
|
93
|
# tell the command server that it should no longer process isotovideo commands since we've |
357
|
93
|
# just left the loop which would handle such commands (otherwise the command server would just |
358
|
|
# hang on the next isotovideo command) |
359
|
156
|
$command_handler->stop_command_processing; |
360
|
|
|
361
|
288
|
# terminate/kill the command server and let it inform its websocket clients before |
362
|
49
|
stop_commands('test execution ended'); |
363
|
|
|
364
|
|
if ($testfd) { |
365
|
|
$return_code = 1; # unusual shutdown |
366
|
|
CORE::close $testfd; |
367
|
|
stop_autotest; |
368
|
|
} |
369
|
3
|
|
370
|
|
diag 'isotovideo ' . ($return_code ? 'failed' : 'done'); |
371
|
|
|
372
|
3
|
my $clean_shutdown; |
373
|
|
if (!$return_code) { |
374
|
3
|
eval { |
375
|
0
|
$clean_shutdown = $bmwqemu::backend->_send_json({cmd => 'is_shutdown'}); |
376
|
0
|
diag('backend shutdown state: ' . ($clean_shutdown // '?')); |
377
|
0
|
}; |
378
|
|
|
379
|
|
# don't rely on the backend in a sane state if we failed - just stop it later |
380
|
3
|
eval { bmwqemu::stop_vm(); }; |
381
|
|
if ($@) { |
382
|
3
|
bmwqemu::serialize_state(component => 'backend', msg => "unable to stop VM: $@", error => 1); |
383
|
3
|
$return_code = 1; |
384
|
3
|
} |
385
|
3
|
} |
386
|
3
|
|
387
|
|
# read calculated variables from backend and tests |
388
|
|
bmwqemu::load_vars(); |
389
|
|
|
390
|
3
|
$return_code = handle_generated_assets($command_handler, $clean_shutdown) unless $return_code; |
|
3
|
|
391
|
3
|
|
392
|
0
|
# clear any previously recorded die message; it was not fatal after all if the execution came this far |
393
|
0
|
$fatal_error = undef; |
394
|
|
|
395
|
|
END { |
396
|
|
stop_backend; |
397
|
|
stop_commands('test execution ended through exception'); |
398
|
3
|
stop_autotest; |
399
|
|
|
400
|
3
|
# in case of early exit, e.g. help display |
401
|
|
$return_code //= 0; |
402
|
|
|
403
|
3
|
bmwqemu::serialize_state(component => 'isotovideo', msg => $fatal_error) if $fatal_error; |
404
|
|
print "$$: EXIT $return_code\n"; |
405
|
|
$? = $return_code; |
406
|
14
|
} |