QA » openQA Project » openQA Infrastructure

Almost all openqa handler (/usr/bin/perl /usr/share/openqa/script/openqa prefork […]) currently hang in the D-state on OSD.

openqa:~ # ps aux | grep "/usr/share/openqa/script/openqa prefor[k]"
geekote+  2941  8.7  1.0 384532 210240 ?       D    14:49   2:13 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+  2946  5.6  1.0 381996 206696 ?       D    14:49   1:26 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 10284  2.8  0.9 367816 193356 ?       D    14:56   0:32 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 10663  6.0  1.0 381388 205032 ?       R    14:57   1:05 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 12040  5.3  1.0 383656 208780 ?       D    15:00   0:47 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 13269  5.2  0.9 381168 203344 ?       D    15:01   0:42 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14161  0.8  0.8 355488 177000 ?       D    15:03   0:05 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14718  1.7  0.8 360132 181644 ?       D    15:03   0:11 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14729  5.7  0.9 375812 197020 ?       D    15:03   0:38 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14837  1.1  0.8 357032 177848 ?       D    15:04   0:07 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14840  7.2  0.9 375060 197156 ?       D    15:04   0:48 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14845  7.9  1.0 383264 208816 ?       D    15:04   0:53 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14849  4.4  0.9 366744 191416 ?       D    15:04   0:29 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 14979  6.9  1.0 387076 210412 ?       D    14:35   2:47 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 15484  1.8  0.9 360192 185432 ?       D    15:05   0:10 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 15794 11.6  1.0 393520 215320 ?       D    15:06   1:03 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 16235  3.7  0.9 371104 192780 ?       D    15:07   0:17 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 16597  0.3  0.8 349292 170632 ?       D    15:08   0:01 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 16610  0.1  0.8 350916 170608 ?       D    15:08   0:00 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 16611  0.8  0.9 363432 188572 ?       D    15:08   0:03 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 20520 27.0  0.8 357136 177796 ?       R    15:14   0:03 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 20521 64.0  0.9 371536 192352 ?       S    15:14   0:07 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800
geekote+ 24565  0.1  0.8 345716 164836 ?       Ss   Sep23   3:58 /usr/bin/perl /usr/share/openqa/script/openqa prefork -m production --proxy -i 100 -H 400 -w 20 -c 1 -G 800

They all seem to wait for XFS:

openqa:~ # cat /proc/2941/stack
[<ffffffff811d4236>] balance_dirty_pages+0x286/0xd90
[<ffffffff811d5002>] balance_dirty_pages_ratelimited+0x2c2/0x3a0
[<ffffffff812c6ecb>] iomap_write_actor+0xeb/0x140
[<ffffffff812c74c4>] iomap_apply+0xa4/0x110
[<ffffffff812c7592>] iomap_file_buffered_write+0x62/0x90
[<ffffffffa08be8ce>] xfs_file_buffered_aio_write+0xae/0x2c0 [xfs]
[<ffffffff8125d0ae>] __vfs_write+0xde/0x150
[<ffffffff8125e05d>] vfs_write+0xad/0x1a0
[<ffffffff8125f552>] SyS_write+0x42/0x90
[<ffffffff81004aeb>] do_syscall_64+0x7b/0x160
[<ffffffff818000b6>] entry_SYSCALL_64_after_hwframe+0x59/0xbe
[<ffffffffffffffff>] 0xffffffffffffffff

These are the currently opened files:

openqa:~ # lsof -p 2941
COMMAND  PID      USER   FD   TYPE             DEVICE SIZE/OFF       NODE NAME
openqa  2941 geekotest  cwd    DIR              253,1     4096          2 /
openqa  2941 geekotest  rtd    DIR              253,1     4096          2 /
openqa  2941 geekotest  txt    REG              253,1  2099808      81338 /usr/bin/perl
openqa  2941 geekotest  mem    REG              253,1    26408      40882 /lib64/libnss_dns-2.26.so
openqa  2941 geekotest  mem    REG              253,1    18624     146367 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/BSD/Resource/Resource.so
openqa  2941 geekotest  mem    REG              253,1    60000     199100 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/JSON/XS/XS.so
openqa  2941 geekotest  mem    REG              253,1    51856     199241 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/HTML/Parser/Parser.so
openqa  2941 geekotest  mem    REG              253,1   615352      82961 /usr/lib64/libgmp.so.10.3.2
openqa  2941 geekotest  mem    REG              253,1    23032     198885 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Crypt/DH/GMP/GMP.so
openqa  2941 geekotest  mem    REG              253,1   235576      81507 /usr/lib64/liblzma.so.5.2.3
openqa  2941 geekotest  mem    REG              253,1  1469280      81602 /usr/lib64/libxml2.so.2.9.7
openqa  2941 geekotest  mem    REG              253,1   435720     199102 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/XML/LibXML/LibXML.so
openqa  2941 geekotest  mem    REG              253,1   236808      40959 /lib64/libtirpc.so.3.0.0
openqa  2941 geekotest  mem    REG              253,1   108632      85646 /usr/lib64/libnsl.so.2.0.0
openqa  2941 geekotest  mem    REG              253,1    55728      41269 /lib64/libnss_nis.so.2.0.0
openqa  2941 geekotest  mem    REG              253,1    41960      40769 /lib64/libnss_compat-2.26.so
openqa  2941 geekotest  mem    REG              253,1   575808      82142 /usr/lib64/libpcre.so.1.2.9
openqa  2941 geekotest  mem    REG              253,1   159456      40533 /lib64/libselinux.so.1
openqa  2941 geekotest  mem    REG              253,1    14648      84946 /usr/lib64/libkeyutils.so.1.6
openqa  2941 geekotest  mem    REG              253,1   118512      80019 /usr/lib64/libsasl2.so.3.0.0
openqa  2941 geekotest  mem    REG              253,1    96624      40972 /lib64/libresolv-2.26.so
openqa  2941 geekotest  mem    REG              253,1    60472      80064 /usr/lib64/liblber-2.4.so.2.10.9
openqa  2941 geekotest  mem    REG              253,1    52440      84055 /usr/lib64/libkrb5support.so.0.1
openqa  2941 geekotest  mem    REG              253,1    14720      83981 /usr/lib64/libcom_err.so.2.1
openqa  2941 geekotest  mem    REG              253,1   199576      84019 /usr/lib64/libk5crypto.so.3.1
openqa  2941 geekotest  mem    REG              253,1   900864      80707 /usr/lib64/libkrb5.so.3.3
openqa  2941 geekotest  mem    REG              253,1   335352      84621 /usr/lib64/libldap_r-2.4.so.2.10.9
openqa  2941 geekotest  mem    REG              253,1   309784      84013 /usr/lib64/libgssapi_krb5.so.2.2
openqa  2941 geekotest  mem    REG              253,1   319696      83674 /usr/lib64/libpq.so.5.12
openqa  2941 geekotest  mem    REG              253,1   277400     198098 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/DBD/Pg/Pg.so
openqa  2941 geekotest  mem    REG              253,1    51688     199117 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Params/Validate/XS/XS.so
openqa  2941 geekotest  mem    REG              253,1    18760     178240 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Time/Piece/Piece.so
openqa  2941 geekotest  mem    REG              253,1    14576     177724 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/I18N/Langinfo/Langinfo.so
openqa  2941 geekotest  mem    REG              253,1   169224     201127 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/YAML/XS/LibYAML/LibYAML.so
openqa  2941 geekotest  mem    REG              253,1   305168      85105 /usr/lib64/libcmark.so.0.28.3
openqa  2941 geekotest  mem    REG              253,1    38512     200950 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/CommonMark/CommonMark.so
openqa  2941 geekotest  mem    REG              253,1    18536     199832 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/DateTime/DateTime.so
openqa  2941 geekotest  mem    REG              253,1    10416     177739 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Tie/Hash/NamedCapture/NamedCapture.so
openqa  2941 geekotest  mem    REG              253,1     6208     178236 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Sys/Hostname/Hostname.so
openqa  2941 geekotest  mem    REG              253,1   146040     198863 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/DBI/DBI.so
openqa  2941 geekotest  mem    REG              253,1   117328     198599 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/List/MoreUtils/XS/XS.so
openqa  2941 geekotest  mem    REG              253,1    32280     199830 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Moose/Moose.so
openqa  2941 geekotest  mem    REG              253,1    10320     198138 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Sub/Identify/Identify.so
openqa  2941 geekotest  mem    REG              253,1    14528     199251 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Class/Load/XS/XS.so
openqa  2941 geekotest  mem    REG              253,1    18616     198222 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Params/Util/Util.so
openqa  2941 geekotest  mem    REG              253,1    55864     198897 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Class/XSAccessor/XSAccessor.so
openqa  2941 geekotest  mem    REG              253,1    10400     198135 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Sub/Name/Name.so
openqa  2941 geekotest  mem    REG              253,1    31120     198229 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Package/Stash/XS/XS.so
openqa  2941 geekotest  mem    REG              253,1    35648     193872 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Variable/Magic/Magic.so
openqa  2941 geekotest  mem    REG              253,1    10432     177745 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/attributes/attributes.so
openqa  2941 geekotest  mem    REG              253,1    88984     201379 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Cpanel/JSON/XS/XS.so
openqa  2941 geekotest  mem    REG              253,1    88800     177705 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/B/B.so
openqa  2941 geekotest  mem    REG              253,1  2681240      84318 /usr/lib64/libcrypto.so.1.1
openqa  2941 geekotest  mem    REG              253,1   442648      85368 /usr/lib64/libssl.so.1.1
openqa  2941 geekotest  mem    REG              253,1   502432     198238 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Net/SSLeay/SSLeay.so
openqa  2941 geekotest  mem    REG              253,1    60784      40897 /lib64/libnss_files-2.26.so
openqa  2941 geekotest  mem    REG              253,1   106088     138559 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/POSIX/POSIX.so
openqa  2941 geekotest  mem    REG              253,1    43488     139167 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Socket/Socket.so
openqa  2941 geekotest  mem    REG              253,1    18744     177722 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Hash/Util/FieldHash/FieldHash.so
openqa  2941 geekotest  mem    REG              253,1   548400     178244 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/re/re.so
openqa  2941 geekotest  mem    REG              253,1   579912     177743 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Unicode/Normalize/Normalize.so
openqa  2941 geekotest  mem    REG              253,1    41592      40978 /lib64/librt-2.26.so
openqa  2941 geekotest  mem    REG              253,1    26976     178239 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Time/HiRes/HiRes.so
openqa  2941 geekotest  mem    REG              253,1    14624     135933 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Cwd/Cwd.so
openqa  2941 geekotest  mem    REG              253,1    14480     177726 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/MIME/Base64/Base64.so
openqa  2941 geekotest  mem    REG              253,1    92216      40043 /lib64/libz.so.1.2.11
openqa  2941 geekotest  mem    REG              253,1    59944     177707 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Compress/Raw/Zlib/Zlib.so
openqa  2941 geekotest  mem    REG              253,1    18800     139165 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/IO/IO.so
openqa  2941 geekotest  mem    REG              253,1    27200     138557 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/File/Glob/Glob.so
openqa  2941 geekotest  mem    REG              253,1    89000     177736 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Storable/Storable.so
openqa  2941 geekotest  mem    REG              253,1    39432     127327 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/Encode/Encode.so
openqa  2941 geekotest  mem    REG              253,1    43216     139162 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Digest/SHA/SHA.so
openqa  2941 geekotest  mem    REG              253,1    18712     139163 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Fcntl/Fcntl.so
openqa  2941 geekotest  mem    REG              253,1    18728     139161 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Digest/MD5/MD5.so
openqa  2941 geekotest  mem    REG              253,1    39520     139160 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/Data/Dumper/Dumper.so
openqa  2941 geekotest  mem    REG              253,1    51872     146356 /usr/lib/perl5/vendor_perl/5.26.1/x86_64-linux-thread-multi/auto/List/Util/Util.so
openqa  2941 geekotest  mem    REG              253,1    18904     178243 /usr/lib/perl5/5.26.1/x86_64-linux-thread-multi/auto/mro/mro.so
openqa  2941 geekotest  mem    REG              253,1  2038456      39871 /lib64/libc-2.26.so
openqa  2941 geekotest  mem    REG              253,1   144048      40951 /lib64/libpthread-2.26.so
openqa  2941 geekotest  mem    REG              253,1    57448      40462 /lib64/libcrypt-2.26.so
openqa  2941 geekotest  mem    REG              253,1    18400      40529 /lib64/libdl-2.26.so
openqa  2941 geekotest  mem    REG              253,1  1355760      40535 /lib64/libm-2.26.so
openqa  2941 geekotest  mem    REG              253,1   180056      40288 /lib64/ld-2.26.so
openqa  2941 geekotest  mem    REG              253,1   330604     182205 /usr/lib/locale/en_US.utf8/LC_CTYPE
openqa  2941 geekotest  mem    REG              253,1    26244      27665 /usr/lib64/gconv/gconv-modules.cache
openqa  2941 geekotest    0r   CHR                1,3      0t0       1029 /dev/null
openqa  2941 geekotest    1u  unix 0xffff88018f5a4000      0t0  220143201 type=STREAM
openqa  2941 geekotest    2u  unix 0xffff88018f5a4000      0t0  220143201 type=STREAM
openqa  2941 geekotest    3u  IPv6          306583711      0t0        TCP localhost:9526->localhost:47984 (ESTABLISHED)
openqa  2941 geekotest    4u  unix 0xffff880100c19000      0t0  220142243 type=STREAM
openqa  2941 geekotest    5r   REG              253,1    14590     199991 /usr/lib/perl5/vendor_perl/5.26.1/Mojolicious/Plugin/AssetPack/Store.pm
openqa  2941 geekotest    6r   REG              253,1    39061     175849 /usr/share/openqa/assets/images/logo-not_complete.svg
openqa  2941 geekotest    7r   REG              253,1      690     175375 /usr/share/openqa/assets/images/logo-blocked-16.png
openqa  2941 geekotest    8r   REG              253,1    15448     175858 /usr/share/openqa/assets/images/suse.png
openqa  2941 geekotest    9w  FIFO               0,12      0t0  220140890 pipe
openqa  2941 geekotest   10u  IPv4          220140892      0t0        TCP localhost:9526 (LISTEN)
openqa  2941 geekotest   11r   REG              253,1      578     175373 /usr/share/openqa/assets/images/logo-16.png
openqa  2941 geekotest   12r   REG              253,1    38864     175382 /usr/share/openqa/assets/images/logo-cancelled.svg
openqa  2941 geekotest   13u  IPv6          220140894      0t0        TCP localhost:9526 (LISTEN)
openqa  2941 geekotest   14r   REG              253,1      695     175850 /usr/share/openqa/assets/images/logo-passed-16.png
openqa  2941 geekotest   15u  unix 0xffff88012ba0a400      0t0  306350390 type=STREAM
openqa  2941 geekotest   16u  unix 0xffff88031bad6c00      0t0  306360877 type=STREAM
openqa  2941 geekotest   17r   REG              253,1    38862     175378 /usr/share/openqa/assets/images/logo-blocked.svg
openqa  2941 geekotest   18r   REG              253,1      741     175854 /usr/share/openqa/assets/images/logo-softfailed-16.png
openqa  2941 geekotest   19u  IPv4          306574803      0t0        TCP 149.44.176.58:53238->149.44.176.6:amqps (ESTABLISHED)
openqa  2941 geekotest   20u  IPv4          306576774      0t0        TCP 149.44.176.58:54292->149.44.176.6:amqps (ESTABLISHED)
openqa  2941 geekotest   21r   REG             253,32 23597816 8589934864 /var/lib/openqa/share/factory/tmp/mojo.tmp.2EfuDNclUFYwonIY
openqa  2941 geekotest   22w   REG             253,48  4038656 4641774464 /var/lib/openqa/testresults/04737/04737242-sle-12-SP3-Server-DVD-Updates-x86_64-Build20200925-2-mru-qa_kernel_multipath_supportserver@64bit/video.ogv
openqa  2941 geekotest   40r   REG              253,1    39057     175388 /usr/share/openqa/assets/images/logo-failed.svg
openqa  2941 geekotest   41r   REG              253,1    38864     175853 /usr/share/openqa/assets/images/logo-scheduled.svg
openqa  2941 geekotest   42r   REG              253,1      578     175373 /usr/share/openqa/assets/images/logo-16.png
openqa  2941 geekotest   47r   REG              253,1      987     175859 /usr/share/openqa/assets/images/terminal.svg
openqa  2941 geekotest   48r   REG              253,1     1356     175370 /usr/share/openqa/assets/images/audio.svg
openqa  2941 geekotest   49r   REG              253,1    38952     175851 /usr/share/openqa/assets/images/logo-passed.svg
openqa  2941 geekotest   51r   REG              253,1      725     175386 /usr/share/openqa/assets/images/logo-failed-16.png
openqa  2941 geekotest   52r   REG              253,1    38862     175385 /usr/share/openqa/assets/images/logo-execution.svg
openqa  2941 geekotest   53r   REG              253,1    38688     175856 /usr/share/openqa/assets/images/logo.svg
openqa  2941 geekotest   55r   REG              253,1    99704     207584 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-brands-400.eot
openqa  2941 geekotest   68r   REG              253,1    99468     207590 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-brands-400.ttf
openqa  2941 geekotest   69r   REG              253,1    55144     207591 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-brands-400.woff2
openqa  2941 geekotest   70r   REG              253,1   511707     207587 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-brands-400.svg
openqa  2941 geekotest  152r   REG              253,1    30788     207592 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-regular-400.eot
openqa  2941 geekotest  153r   REG              253,1    30560     207595 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-regular-400.ttf
openqa  2941 geekotest  154r   REG              253,1    12216     207596 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-regular-400.woff2
openqa  2941 geekotest  155r   REG              253,1   103937     207593 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-regular-400.svg
openqa  2941 geekotest  156r   REG              253,1   115148     208252 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-solid-900.eot
openqa  2941 geekotest  157r   REG              253,1   114928     208254 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-solid-900.ttf
openqa  2941 geekotest  158r   REG              253,1    44068     208255 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-solid-900.woff2
openqa  2941 geekotest  159r   REG              253,1   420283     208253 /usr/share/openqa/assets/cache/use.fontawesome.com/releases/v5.0.10/webfonts/fa-solid-900.svg
openqa  2941 geekotest  160r   REG              253,1      538     112208 /usr/share/openqa/assets/cache/cdnjs.cloudflare.com/ajax/libs/chosen/1.7.0/chosen-sprite.png
openqa  2941 geekotest  161r   REG              253,1      738     112493 /usr/share/openqa/assets/cache/cdnjs.cloudflare.com/ajax/libs/chosen/1.7.0/chosen-sprite@2x.png
openqa  2941 geekotest  162r   REG              253,1      677     175383 /usr/share/openqa/assets/images/logo-execution-16.png
openqa  2941 geekotest  164r   REG              253,1      723     175848 /usr/share/openqa/assets/images/logo-not_complete-16.png
openqa  2941 geekotest  165r   REG              253,1    38688     175856 /usr/share/openqa/assets/images/logo.svg
openqa  2941 geekotest  166r   REG              253,1    39061     175855 /usr/share/openqa/assets/images/logo-softfailed.svg
openqa  2941 geekotest  167r   REG              253,1      690     175852 /usr/share/openqa/assets/images/logo-scheduled-16.png
openqa  2941 geekotest  168r   REG              253,1    38864     175382 /usr/share/openqa/assets/images/logo-cancelled.svg
openqa  2941 geekotest  169r   REG              253,1      683     175381 /usr/share/openqa/assets/images/logo-cancelled-16.png
openqa  2941 geekotest  170r   REG              253,1      683     175381 /usr/share/openqa/assets/images/logo-cancelled-16.png

openqa.suse.de is also kind of unresponsive. Do we maybe need faster (NetApp storage)?