File Coverage

basetest.pm
Criterion Covered Total %
statement 397 477 83.2
total 397 477 83.2


line stmt code
1   # Copyright 2009-2013 Bernhard M. Wiedemann
2   # Copyright 2012-2021 SUSE LLC
3   # SPDX-License-Identifier: GPL-2.0-or-later
4    
5    
6   use Mojo::Base -strict, -signatures;
7 29 use autodie ':all';
  29  
  29  
8 29  
  29  
  29  
9   use bmwqemu ();
10 29 use ocr;
  29  
  29  
11 29 use testapi ();
  29  
  29  
12 29 use autotest ();
  29  
  29  
13 29 use MIME::Base64 'decode_base64';
  29  
  29  
14 29 use Mojo::File 'path';
  29  
  29  
15 29  
  29  
  29  
16   my $serial_file_pos = 0;
17   my $autoinst_log_pos = 0;
18    
19   # enable strictures and warnings in all tests globally but allow signatures
20   strict->import;
21 3 warnings->import;
  3  
  3  
22 3 warnings->unimport('experimental::signatures');
23 3 }
24 3  
25   $category ||= 'unknown';
26   my $self = {class => $class};
27 273 $self->{lastscreenshot} = undef;
  273  
  273  
  273  
28 273 $self->{details} = [];
29 273 $self->{result} = undef;
30 273 $self->{running} = 0;
31 273 $self->{category} = $category;
32 273 $self->{test_count} = 0;
33 273 $self->{screen_count} = 0;
34 273 $self->{wav_fn} = undef;
35 273 $self->{dents} = 0;
36 273 $self->{post_fail_hook_running} = 0;
37 273 $self->{timeoutcounter} = 0;
38 273 $self->{activated_consoles} = [];
39 273 $self->{name} = $class;
40 273 $self->{serial_failures} = [];
41 273 $self->{autoinst_failures} = [];
42 273 $self->{fatal_failure} = 0;
43 273 $self->{execution_time} = 0;
44 273 $self->{test_start_time} = 0;
45 273 return bless $self, $class;
46 273 }
47 273  
48 273 =head1 Methods
49    
50   =head2 run
51    
52   Body of the test to be implemented by child classes.
53   This code is run during test.
54    
55   =head2 is_applicable
56    
57   Return false if the test should be skipped.
58    
59   By default it checks the test name and fullname against a comma-separated
60   blocklist in C<EXCLUDE_MODULES> variable and returns false if it is found there.
61    
62   If C<INCLUDE_MODULES> is set it will only return true for modules matching the
63   passlist specified in a comma-separated list in C<EXCLUDE_MODULES> matching
64   either test name or fullname.
65    
66   C<EXCLUDE_MODULES> has precedence over C<INCLUDE_MODULES> and can be combined
67   to blocklist test modules from the passlist specified in C<INCLUDE_MODULES>.
68    
69   Can eg. check vars{BIGTEST}, vars{LIVETEST}
70    
71   =cut
72    
73   if ($bmwqemu::vars{EXCLUDE_MODULES}) {
74   my %excluded = map { $_ => 1 } split(/\s*,\s*/, $bmwqemu::vars{EXCLUDE_MODULES});
75    
76 267 return 0 if $excluded{$self->{class}};
  267  
  267  
77 267 return 0 if $excluded{$self->{fullname}};
78 2 }
  3  
79   if ($bmwqemu::vars{INCLUDE_MODULES}) {
80 2 my %included = map { $_ => 1 } split(/\s*,\s*/, $bmwqemu::vars{INCLUDE_MODULES});
81 0  
82   return 0 unless ($included{$self->{class}} || $included{$self->{fullname}});
83 265 }
84 145 return 1;
  148  
85   }
86 145  
87   =head2 test_flags
88 121  
89   Return a hash of flags that are either there or not
90    
91   'fatal' - abort whole test suite if this fails (and set overall state 'failed')
92   'ignore_failure' - if this module fails, it will not affect the overall result at all
93   'milestone' - after this test succeeds, update 'lastgood'
94   'no_rollback' - don't roll back to 'lastgood' snapshot if this fails
95   'always_rollback' - roll back to 'lastgood' snapshot even if this does not fail
96    
97   =cut
98    
99    
100   =head2 post_fail_hook
101    
102   Function is run after test has failed to e.g. recover log files
103 199  
  199  
  199  
  199  
104   =cut
105    
106    
107   =head2 _framenumber_to_timerange
108    
109   Create a media fragment time from a given framenumber
110    
111 17 =cut
  17  
  17  
  17  
112    
113   return [sprintf("%.2f", $frame / 24.0), sprintf("%.2f", ($frame + 1) / 24.0)];
114   }
115    
116   my $serialized_match = $self->_serialize_match($match);
117   my $properties = $match->{needle}->{properties} || [];
118   my $result = {
119 5 needle => $serialized_match->{name},
  5  
  5  
120 5 area => $serialized_match->{area},
121   error => $serialized_match->{error},
122   json => $serialized_match->{json},
123 2 tags => [@$tags], # make a copy
  2  
  2  
  2  
  2  
  2  
  2  
  2  
124 2 properties => [@$properties], # make a copy
125 2 frametime => _framenumber_to_timerange($frame),
126   screenshot => $self->next_resultname('png'),
127   result => 'ok',
128   };
129    
130   # make sure needle is blessed
131 2 my $foundneedle = bless $match->{needle}, "needle";
132    
133   # When the needle has the workaround property,
134   # mark the result as dent and increase the dents
135   if (my $workaround = $foundneedle->has_property('workaround')) {
136   $result->{dent} = 1;
137   $result->{result} = "softfail";
138    
139 2 # write a test result file
140   my $reason = $foundneedle->get_property_value('workaround');
141   $self->record_soft_failure_result($reason);
142    
143 2 bmwqemu::diag("needle '$serialized_match->{name}' is a workaround. The reason is $reason");
144 1 }
145 1  
146   # also include the not matched needles
147   my $candidates;
148 1 for my $cand (@{$failed_needles || []}) {
149 1 push @$candidates, $self->_serialize_match($cand);
150   }
151 1 $result->{needles} = $candidates if $candidates;
152    
153   my $fn = join('/', bmwqemu::result_dir(), $result->{screenshot});
154   $img->write_with_thumbnail($fn);
155 2  
156 2 $self->{result} ||= 'ok';
  2  
157 1  
158   push @{$self->{details}}, $result;
159 2 return $result;
160   }
161 2  
162 2 =head2
163    
164 2 serialize a match result from needle::search
165    
166 2 =cut
  2  
167 2  
168   my $name = $candidate->{needle}->{name};
169   my $jsonfile = $candidate->{needle}->{file};
170   my %match = (
171   name => $name,
172   error => $candidate->{error},
173   area => [],
174   json => $jsonfile
175   );
176 3  
  3  
  3  
  3  
177 3 if (my $unregistered = $candidate->{needle}->{unregistered}) {
178 3 $match{unregistered} = $unregistered;
179   }
180   for my $area (@{$candidate->{area}}) {
181   my $na = {};
182 3 for my $i (qw(x y w h result)) {
183   $na->{$i} = $area->{$i};
184   }
185   $na->{similarity} = int($area->{similarity} * 100);
186 3 $na->{click_point} = $area->{click_point} if exists $area->{click_point};
187 0 push @{$match{area}}, $na;
188   }
189 3  
  3  
190 3 return \%match;
191 3 }
192 15  
193   my $img = $args{img};
194 3 my $needles = $args{needles} || [];
195 3 my $tags = $args{tags} || [];
196 3 my $status = $args{result} || 'fail';
  3  
197   my $overall = $args{overall}; # whether and how to set global test result
198   my $frame = $args{frame};
199 3  
200   my $candidates;
201   for my $cand (@{$needles || []}) {
202 3 push @$candidates, $self->_serialize_match($cand);
  3  
  3  
  3  
203 3 }
204 3  
205 3 my $result = {
206 3 screenshot => $self->next_resultname('png'),
207 3 result => $status,
208 3 frametime => _framenumber_to_timerange($frame),
209   };
210 3  
211 3 $result->{needles} = $candidates if $candidates;
  3  
212 0 $result->{tags} = [@$tags] if $tags; # make a copy
213    
214   my $fn = join('/', bmwqemu::result_dir(), $result->{screenshot});
215 3 $img->write_with_thumbnail($fn);
216    
217   $self->{result} = $overall if $overall;
218    
219   push @{$self->{details}}, $result;
220   return $result;
221 3 }
222 3  
223   --$self->{test_count};
224 3 return pop @{$self->{details}};
225 3 }
226    
227 3 return $self->{details};
228   }
229 3  
  3  
230 3 $self->{result} = $result if $result;
231   return $self->{result} || 'na';
232   }
233 125  
  125  
  125  
234 125 $self->{running} = 1;
235 125 autotest::set_current_test($self);
  125  
236   }
237    
238 58 $self->{running} = 0;
  58  
  58  
239 58 $self->{result} ||= 'ok';
240   unless ($self->{test_count}) {
241   $self->take_screenshot();
242 58 }
  58  
  58  
  58  
243 58 autotest::set_current_test(undef);
244 58 }
245    
246   $self->{result} = 'fail' if $self->{result};
247 58 autotest::set_current_test(undef);
  58  
  58  
248 58 }
249 58  
250   $self->{result} = 'skip' if !$self->{result};
251   autotest::set_current_test(undef);
252 28 }
  28  
  28  
253 28  
254 28  
255 28 my $n = ++$self->{timeoutcounter};
256 28 $self->take_screenshot(sprintf("timeout-%02i", $n));
257   }
258 28  
259   # you should overload that in test classes
260   return;
261 18 }
  18  
  18  
262 18  
263 18 # you should overload that in test classes
264   return;
265   }
266 0  
  0  
  0  
267 0 my $post_fail_hook_start_time = time;
268 0 unless ($bmwqemu::vars{_SKIP_POST_FAIL_HOOKS}) {
269   $self->{post_fail_hook_running} = 1;
270   eval { $self->post_fail_hook; };
271   bmwqemu::diag("post_fail_hook failed: $@") if $@;
272 0 $self->{post_fail_hook_running} = 0;
  0  
  0  
273 0  
274 0 # There might be more messages on serial now.
275   # Read them now to not stumble upon them in the next module.
276   $self->get_new_serial_output();
277 47 }
  47  
  47  
278    
279 47 $self->fail_if_running();
280   $self->compute_test_execution_time();
281   my $post_fail_hook_execution_time = execution_time($post_fail_hook_start_time);
282 41 bmwqemu::modstate("post fail hooks runtime: $post_fail_hook_execution_time s");
  41  
  41  
283   die $msg . "\n";
284 41 }
285    
286    
287 18 # Set the execution time for a general time spent
  18  
  18  
  18  
288 18 $self->{execution_time} = execution_time($self->{test_start_time});
289 18 bmwqemu::modstate(sprintf("finished %s %s (runtime: %d s)", $self->{name}, $self->{category}, $self->{execution_time}));
290 17 }
291 17  
  17  
292 17 $self->{test_start_time} = time;
293 17  
294   my $died;
295   my $name = $self->{name};
296   # Set flags to the field value
297 17 $self->{flags} = $self->test_flags();
298   eval {
299   $self->pre_run_hook();
300 18 if (defined $self->{run_args}) {
301 18 $self->run($self->{run_args});
302 18 }
303 18 else {
304 18 $self->run();
305   }
306   $self->post_run_hook();
307 64 };
  64  
  64  
  64  
308   if ($@) {
309 47 # copy the exception early
  47  
  47  
310   my $internal = Exception::Class->caught('OpenQA::Exception::InternalException');
311 47  
312 47 $self->{result} = 'fail';
313   # add a fail screenshot in case there is none
314   if (!@{$self->{details}} || ($self->{details}->[-1]->{result} || '') ne 'fail') {
315 47 $self->take_screenshot();
  47  
  47  
316 47 }
317   # show a text result with the die message unless the die was internally generated
318 47 if (!$internal) {
319 47 my $msg = "# Test died: $@";
320   bmwqemu::fctinfo($msg);
321 47 $self->record_resultfile('Failed', $msg, result => 'fail');
322 47 $died = 1;
323 47 }
324 47 }
325 4  
326   eval { $self->search_for_expected_serial_failures(); };
327   # Process serial detection failure
328 43 if ($@) {
329   bmwqemu::diag($@);
330 41 $self->record_resultfile('Failed', $@, result => 'fail');
331   $died = 1;
332 47 }
333    
334 6 $self->run_post_fail("test $name died") if ($died);
335    
336 6 if (($self->{result} || '') eq 'fail') {
337   # fatal
338 6 $self->run_post_fail("test $name failed");
  6  
339 4 }
340    
341   $self->compute_test_execution_time();
342 6 $self->done();
343 6 return;
344 6 }
345 6  
346 6 my $result = {
347   details => $self->details(),
348   result => $self->result(),
349   dents => $self->{dents},
350 47 execution_time => $self->{execution_time},
  47  
351   };
352 47 $result->{extra_test_results} = $self->{extra_test_results} if $self->{extra_test_results};
353 14  
354 14 # be aware that $name has to be unique within one job (also assumed in several other places)
355 14 my $fn = bmwqemu::result_dir() . sprintf("/result-%s.json", $self->{name});
356   bmwqemu::save_json_file($result, $fn);
357   return $result;
358 47 }
359    
360 29 my $testname = $self->{name};
361   my $count = ++$self->{test_count};
362 0 if ($name) {
363   return "$testname-$count.$name.$type";
364   }
365 29 else {
366 29 return "$testname-$count.$type";
367 29 }
368   }
369    
370 58 path(bmwqemu::result_dir(), $filename)->spurt($output);
  58  
  58  
371   }
372    
373   =head2 record_resultfile
374    
375   $self->record_resultfile($title, $output [, result => $result] [, resultname => $name]);
376 58  
377 58 Record result file to be parsed when evaluating test results, for example
378   within the openQA web interface.
379   =cut
380 58 my $filename = $self->next_resultname('txt', $nargs{resultname});
381 58 my $detail = {
382 58 title => $title,
383   result => $nargs{result},
384   text => $filename,
385 34 };
  34  
  34  
  34  
  34  
386 34 push @{$self->{details}}, $detail;
387 34 $self->write_resultfile($filename, $output);
388 34 }
389 1  
390   $string //= '';
391   # take screenshot for documentation (screenshot does not represent fail itself)
392 33 $self->take_screenshot() unless (testapi::is_serial_terminal);
393    
394   my $output = "# wait_serial expected: $ref\n";
395   $output .= "# Result:\n";
396 29 $output .= "$string\n";
  29  
  29  
  29  
  29  
397 29 $self->record_resultfile('wait_serial', $output, result => $res);
398   return undef;
399   }
400    
401   $reason //= '(no reason specified)';
402   my $result = $self->record_testresult('softfail', %args);
403   my $filename = $self->next_resultname('txt');
404   $result->{title} = 'Soft Failed';
405   $result->{text} = $filename;
406   $self->write_resultfile($filename, "# Soft Failure:\n$reason\n");
407 26 $self->{dents}++;
  26  
  26  
  26  
  26  
  26  
408 26 return undef;
409   }
410    
411   $self->{extra_test_results} //= [];
412 26 foreach my $t (@{$tests}) {
413   $t->{script} = $self->{script} if (!defined($t->{script}) || $t->{script} eq 'unk');
414 26 push @{$self->{extra_test_results}}, $t;
  26  
415 26 }
416   return undef;
417   }
418 10  
  10  
  10  
  10  
  10  
  10  
419 10 =head2 record_testresult
420    
421 10 Makes a new test detail with the specified $result, adds it to the
422   test details and returns it.
423 10  
424 10 =cut
425 10  
426 10 $result //= 'unk';
427 10 # assign result as overall result unless it is already worse
428   my $current_result = \$self->{result};
429   if ($result eq 'fail') {
430 3 $$current_result = 'fail';
  3  
  3  
  3  
  3  
431 3 }
432 3 elsif ($result eq 'softfail') {
433 3 if (!$$current_result || $$current_result ne 'fail' || $args{force_status}) {
434 3 $$current_result = 'softfail';
435 3 }
436 3 }
437 3 elsif ($result && $result eq 'ok') {
438 3 $$current_result //= 'ok';
439   }
440   else {
441 1 # set $result to 'unk' if an invalid value has been specified
  1  
  1  
  1  
442 1 $result = 'unk';
443 1 }
  1  
444 3  
445 3 # add detail
  3  
446   my $detail = {result => $result};
447 1 push(@{$self->{details}}, $detail);
448   ++$self->{test_count};
449   return $detail;
450   }
451    
452   =head2 _result_add_screenshot
453    
454   internal function to add a screenshot to an existing result structure
455    
456   =cut
457 137  
  137  
  137  
  137  
  137  
458 137 my $rsp = autotest::query_isotovideo('backend_last_screenshot_data');
459   my $img = $rsp->{image};
460 137 return $result unless $img;
461 137  
462 2 $img = tinycv::from_ppm(decode_base64($img));
463   return $result unless $img;
464    
465 6 $result->{screenshot} = $self->next_resultname('png');
466 5 $result->{frametime} = _framenumber_to_timerange($rsp->{frame});
467    
468   my $fn = join('/', bmwqemu::result_dir(), $result->{screenshot});
469   $img->write_with_thumbnail($fn);
470 5  
471   return $result;
472   }
473    
474 124 =head2 take_screenshot
475    
476   add screenshot with 'unk' result if an image is available
477    
478 137 =cut
479 137  
  137  
480 137 $res //= 'unk';
481 137 my $result = $self->record_testresult($res);
482   $self->_result_add_screenshot($result);
483    
484   # prevent adding incomplete result to details in case not image was available
485   $self->remove_last_result() unless ($result->{screenshot});
486    
487   return $result;
488   }
489    
490 0 my $fn = $self->{name} . "-captured.wav";
  0  
  0  
  0  
491 0 die "audio capture already in progress. Stop it first!\n" if ($self->{wav_fn});
492 0 $self->{wav_fn} = $fn;
493 0 return $fn;
494   }
495 0  
496 0 bmwqemu::log_call();
497   autotest::query_isotovideo('backend_stop_audiocapture');
498 0  
499 0 my $result = {
500   audio => $self->{wav_fn},
501 0 result => 'unk',
502 0 };
503    
504 0 push @{$self->{details}}, $result;
505    
506   return $result;
507   }
508    
509   my $rsp = autotest::query_isotovideo('backend_verify_image', {imgpath => $imgpath, mustmatch => $mustmatch});
510    
511   my $img = tinycv::read($imgpath);
512   if ($rsp->{found}) {
513 125 my $foundneedle = $rsp->{found};
  125  
  125  
  125  
514 125 $self->record_screenmatch($img, $foundneedle, [$mustmatch], $rsp->{candidates});
515 125 my $lastarea = $foundneedle->{area}->[-1];
516 125 bmwqemu::fctres(sprintf("found %s, similarity %.2f @ %d/%d", $foundneedle->{needle}->{name}, $lastarea->{similarity}, $lastarea->{x}, $lastarea->{y}));
517   return $foundneedle;
518   }
519 125 bmwqemu::fctres(sprintf("failed to find %s", $mustmatch));
520   my @needles_params = (img => $img, needles => $rsp->{candidates}, tags => [$mustmatch]);
521 125 if ($check) {
522   $self->record_screenfail(@needles_params, result => 'unk');
523   }
524 0 else {
  0  
  0  
525 0 $self->record_screenfail(@needles_params, result => 'fail', overall => 'fail');
526 0 }
527 0 return;
528 0 }
529    
530   =head2 ocr_checklist
531 0  
  0  
  0  
532 0 Optical Character Recognition matching.
533 0  
534   Return a listref containing hashrefs like this:
535    
536   {
537 0 screenshot=>2, # nr of screenshot for the test to OCR
538   x=>104, y=>201, # position
539   xs=>380, ys=>150, # size
540 0 pattern=>"H ?ello", # regex to match the OCR result
  0  
541   result=>"OK" # or "fail"
542 0 }
543    
544   =cut
545 0  
  0  
  0  
  0  
  0  
  0  
546 0  
547   $self->record_screenfail(
548 0 img => $lastscreenshot,
549 0 result => 'fail',
550 0 overall => 'fail'
551 0 );
552 0  
553 0 testapi::send_key('alt-sysrq-w');
554 0 testapi::send_key('alt-sysrq-l');
555   testapi::send_key('alt-sysrq-d'); # only available with CONFIG_LOCKDEP
556 0 return;
557 0 }
558 0  
559 0 # this is called if the test failed and the framework loaded a VM
560   # snapshot - all consoles activated in the test's run function loose their
561   # state
562 0 for my $console (@{$self->{activated_consoles}}) {
563   # the backend will only reset its state, and call activate
564 0 # the next time - the console itself might actually not be
565   # able to activate a 2nd time, but that's up to the console class
566   autotest::query_isotovideo('backend_reset_console', {testapi_console => $console});
567   }
568   $self->{activated_consoles} = [];
569    
570   if (defined($autotest::last_milestone_console)) {
571   my $ret = autotest::query_isotovideo('backend_select_console',
572   {testapi_console => $autotest::last_milestone_console});
573   die $ret->{error} if $ret->{error};
574   }
575    
576   return;
577   }
578    
579   if (defined $bmwqemu::vars{BACKEND} && $bmwqemu::vars{BACKEND} eq 'qemu') {
580   $self->parse_serial_output_qemu();
581   }
582   }
583 0  
  0  
  0  
584   myjsonrpc::send_json($autotest::isotovideo, {cmd => 'read_serial', position => $serial_file_pos});
585 0 my $json = myjsonrpc::read_json($autotest::isotovideo);
  0  
  0  
  0  
586 0 $serial_file_pos = $json->{position};
587   return $json->{serial};
588   }
589    
590   # serial failures defined in distri (test can override them)
591   my $failures = $self->{serial_failures};
592 0  
593 0 my $serial = $self->get_new_serial_output();
594 0 my $die = 0;
595 0 my %regexp_matched;
596   # loop line by line
597   for my $line (split(/^/, $serial)) {
598   chomp $line;
599   for my $regexp_table (@{$failures}) {
600   my $regexp = $regexp_table->{pattern};
601 15 my $message = $regexp_table->{message};
  15  
  15  
602 15 my $type = $regexp_table->{type};
  15  
603    
604   # Input parameters validation
605   die "Wrong type defined for serial failure. Only 'info', 'soft', 'hard' or 'fatal' allowed. Got: $type" if $type !~ /^info|soft|hard|fatal$/;
606 0 die "Message not defined for serial failure for the pattern: '$regexp', type: $type" if !defined $message;
607    
608 15 # If you want to match a simple string please be sure that you create it with quotemeta
609   if (!exists $regexp_matched{$regexp} and $line =~ /$regexp/) {
610 15 $regexp_matched{$regexp} = 1;
611 0 my $fail_type = 'softfail';
612   if ($type eq 'info') {
613 0 $fail_type = 'ok';
614   }
615   elsif ($type =~ 'hard|fatal') {
616 15 $die = 1;
617   $fail_type = 'fail';
618   $self->{fatal_failure} = $type eq 'fatal';
619 33 }
  33  
  33  
620 33 $self->record_resultfile($message, $message . " - Serial error: $line", result => $fail_type);
621 0 $self->{result} = $fail_type;
622   }
623   }
624   }
625 20 die "Got serial hard failure" if $die;
  20  
  20  
626 20 return;
627 20 }
628 20  
629 20 1;