line |
stmt |
code |
1
|
|
# Copyright 2009-2013 Bernhard M. Wiedemann |
2
|
|
# Copyright 2012-2021 SUSE LLC |
3
|
|
# SPDX-License-Identifier: GPL-2.0-or-later |
4
|
|
|
5
|
|
use Mojo::Base 'backend::baseclass', -signatures; |
6
|
2
|
use autodie ':all'; |
|
2
|
|
|
2
|
|
7
|
2
|
use Time::HiRes qw(sleep); |
|
2
|
|
|
2
|
|
8
|
2
|
use Time::Seconds; |
|
2
|
|
|
2
|
|
9
|
2
|
use IPC::Run (); |
|
2
|
|
|
2
|
|
10
|
2
|
require IPC::System::Simple; |
|
2
|
|
|
2
|
|
11
|
|
|
12
|
|
$bmwqemu::vars{WORKER_HOSTNAME} or die 'Need variable WORKER_HOSTNAME'; |
13
|
2
|
return $class->SUPER::new; |
|
2
|
|
|
2
|
|
14
|
2
|
} |
15
|
2
|
|
16
|
|
$bmwqemu::vars{"IPMI_$_"} or die 'Need variable IPMI_$_' foreach qw(HOSTNAME USER PASSWORD); |
17
|
|
return ('ipmitool', '-I', 'lanplus', '-H', $bmwqemu::vars{IPMI_HOSTNAME}, '-U', $bmwqemu::vars{IPMI_USER}, '-P', $bmwqemu::vars{IPMI_PASSWORD}); |
18
|
2
|
} |
|
2
|
|
|
2
|
|
19
|
2
|
|
20
|
1
|
$args{tries} //= 1; |
21
|
|
|
22
|
|
my @cmd = $self->ipmi_cmdline(); |
23
|
3
|
push(@cmd, split(/ /, $cmd)); |
|
3
|
|
|
3
|
|
|
3
|
|
|
3
|
|
24
|
3
|
|
25
|
|
my ($stdin, $stdout, $stderr, $ret); |
26
|
3
|
# Mitigate occasional failures of impitool due to self-tests/self-reboots of |
27
|
2
|
# the IPMI controller by a simple sleep/retry mechanism. |
28
|
|
my @tries = (1 .. $args{tries}); |
29
|
2
|
for (@tries) { |
30
|
|
$ret = IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr); |
31
|
|
if ($ret == 0) { |
32
|
2
|
$self->dell_sleep; |
33
|
2
|
last; |
34
|
4
|
} else { |
35
|
4
|
sleep 4; |
36
|
0
|
} |
37
|
0
|
} |
38
|
|
chomp $stdout; |
39
|
4
|
chomp $stderr; |
40
|
|
|
41
|
|
die join(' ', @cmd) . ": $stderr" unless ($ret); |
42
|
2
|
bmwqemu::diag("IPMI: $stdout"); |
43
|
2
|
return $stdout; |
44
|
|
} |
45
|
2
|
|
46
|
2
|
# DELL BMCs are touchy |
47
|
2
|
return unless ($bmwqemu::vars{IPMI_HW} || '') eq 'dell'; |
48
|
|
sleep 4; |
49
|
|
} |
50
|
|
|
51
|
1
|
my $tries = 3; # arbitrary selection of tries |
|
1
|
|
|
1
|
|
52
|
1
|
|
53
|
0
|
my $stdout = $self->ipmitool('chassis power status', tries => $tries); |
54
|
|
if ($stdout !~ m/is off/) { |
55
|
|
$self->ipmitool("chassis power off"); |
56
|
2
|
for (0 .. $tries) { |
|
2
|
|
|
2
|
|
57
|
2
|
sleep(3); |
58
|
|
my $stdout = $self->ipmitool('chassis power status', tries => $tries); |
59
|
2
|
last if $stdout =~ m/is off/; |
60
|
2
|
$self->ipmitool('chassis power off'); |
61
|
2
|
} |
62
|
2
|
} |
63
|
7
|
|
64
|
7
|
$self->ipmitool("chassis power on"); |
65
|
7
|
for (0 .. $tries) { |
66
|
6
|
sleep(3); |
67
|
|
my $ret = $self->ipmitool('chassis power status', tries => $tries); |
68
|
|
last if $ret =~ m/is on/; |
69
|
|
$self->ipmitool('chassis power on'); |
70
|
2
|
} |
71
|
2
|
1; |
72
|
7
|
} |
73
|
7
|
|
74
|
7
|
# reset ipmi main board if switch on |
75
|
6
|
# We may need this IPMI_BACKEND_MC_RESET setting to tune differently |
76
|
|
# on different ipmi workers according to different ipmi machines' behavior. |
77
|
2
|
# It is expected generally that ipmi machine's stability is higher with this mc reset. |
78
|
|
# However there maybe exceptions on machines from different vendors. |
79
|
|
# So keep it for flexibility. |
80
|
1
|
$self->do_mc_reset if $bmwqemu::vars{IPMI_BACKEND_MC_RESET}; |
|
1
|
|
|
1
|
|
81
|
|
$self->get_mc_status; |
82
|
|
$self->restart_host unless $bmwqemu::vars{IPMI_DO_NOT_RESTART_HOST}; |
83
|
|
$self->truncate_serial_file; |
84
|
|
my $sol = $testapi::distri->add_console( |
85
|
|
'sol', 'ipmi-xterm', |
86
|
|
{ |
87
|
1
|
persistent => $bmwqemu::vars{IPMI_SOL_PERSISTENT_CONSOLE} // 1, |
88
|
1
|
log => $bmwqemu::vars{HARDWARE_CONSOLE_LOG} // 0}); |
89
|
1
|
$sol->backend($self); |
90
|
1
|
return {}; |
91
|
|
} |
92
|
|
|
93
|
|
$self->ipmitool("chassis power off") unless $bmwqemu::vars{IPMI_DO_NOT_POWER_OFF}; |
94
|
|
$self->deactivate_console({testapi_console => 'sol'}) if defined $testapi::distri->{consoles}->{sol}; |
95
|
1
|
return {}; |
96
|
1
|
} |
97
|
1
|
|
98
|
|
my $ret = $self->ipmitool('chassis power status', tries => 3); |
99
|
|
return $ret =~ m/is off/; |
100
|
1
|
} |
|
1
|
|
|
1
|
|
101
|
1
|
|
102
|
1
|
|
103
|
1
|
$self->ipmitool("mc guid"); |
104
|
|
$self->ipmitool("mc info"); |
105
|
|
$self->ipmitool("mc selftest") unless $bmwqemu::vars{IPMI_SKIP_SELFTEST}; |
106
|
1
|
} |
|
1
|
|
|
1
|
|
107
|
1
|
|
108
|
1
|
# deactivate sol console before doing mc reset because it breaks sol connection |
109
|
|
if (defined $testapi::distri->{consoles}->{sol}) { |
110
|
|
bmwqemu::diag("Before doing mc reset, sol console exists, so cleanup it"); |
111
|
1
|
$testapi::distri->{consoles}->{sol}->reset(); |
|
1
|
|
|
1
|
|
|
1
|
|
|
1
|
|
|
1
|
|
112
|
|
bmwqemu::diag("sol console reset done"); |
113
|
3
|
$self->deactivate_console({testapi_console => 'sol'}); |
|
3
|
|
|
3
|
|
114
|
3
|
bmwqemu::diag("deactivate console sol done"); |
115
|
2
|
} |
116
|
2
|
|
117
|
|
# during the eval execution of following commands, SIG{__DIE__} will definitely be triggered, let it go |
118
|
|
local $SIG{__DIE__} = {}; |
119
|
1
|
|
|
1
|
|
|
1
|
|
120
|
|
# mc reset cmd should return immediately, try maximum 5 times to ensure cmd executed |
121
|
1
|
my $max_tries = $bmwqemu::vars{IPMI_MC_RESET_MAX_TRIES} // 5; |
122
|
1
|
for (1 .. $max_tries) { |
123
|
1
|
eval { $self->ipmitool("mc reset cold"); }; |
124
|
1
|
if (my $E = $@) { |
125
|
1
|
bmwqemu::diag("IPMI mc reset failure: $E"); |
126
|
1
|
} |
127
|
|
else { |
128
|
|
bmwqemu::diag('IPMI mc reset success, waiting some seconds before trying to connect again'); |
129
|
|
sleep $bmwqemu::vars{IPMI_MC_RESET_SLEEP_TIME_S} // 10; |
130
|
1
|
bmwqemu::diag('sleep period ends, probing connection with ping'); |
131
|
|
# check until mc reset is done and ipmi recovered |
132
|
|
my $count = 0; |
133
|
1
|
my $timeout = $bmwqemu::vars{IPMI_MC_RESET_TIMEOUT} // ONE_MINUTE; |
134
|
1
|
my $ping_count = $bmwqemu::vars{IPMI_MC_RESET_PING_COUNT} // 1; |
135
|
1
|
my $ping_cmd = "ping -c$ping_count '$bmwqemu::vars{IPMI_HOSTNAME}'"; |
|
1
|
|
136
|
1
|
my $ipmi_tries = $bmwqemu::vars{IPMI_MC_RESET_IPMI_TRIES} // 3; |
137
|
0
|
while ($count++ < $timeout) { |
138
|
|
eval { system($ping_cmd); }; |
139
|
|
if (!$@) { |
140
|
1
|
# ping pass, check ipmitool function normally |
141
|
1
|
eval { $self->ipmitool('chassis power status', tries => $ipmi_tries); }; |
142
|
1
|
if (!$@) { |
143
|
|
bmwqemu::diag("IPMI: ipmitool is recovered after mc reset"); |
144
|
1
|
return; |
145
|
1
|
} |
146
|
1
|
} |
147
|
1
|
sleep 3; |
148
|
1
|
} |
149
|
1
|
} |
150
|
1
|
sleep 3; |
|
1
|
|
151
|
1
|
} |
152
|
|
|
153
|
1
|
die "IPMI mc reset failure after $max_tries tries! Exit..."; |
|
1
|
|
154
|
1
|
} |
155
|
1
|
|
156
|
1
|
1; |