line |
stmt |
code |
1
|
|
# Copyright 2018 SUSE LLC |
2
|
|
# SPDX-License-Identifier: GPL-2.0-or-later |
3
|
|
|
4
|
|
use Mojo::Base -base, -signatures; |
5
|
55
|
|
|
55
|
|
|
55
|
|
6
|
|
use Mojo::UserAgent; |
7
|
55
|
use Mojo::URL; |
|
55
|
|
|
55
|
|
8
|
55
|
use Mojo::File qw(path); |
|
55
|
|
|
55
|
|
9
|
55
|
use File::stat; |
|
55
|
|
|
55
|
|
10
|
55
|
use Try::Tiny; |
|
55
|
|
|
55
|
|
11
|
55
|
use POSIX 'strftime'; |
|
55
|
|
|
55
|
|
12
|
55
|
use bmwqemu; |
|
55
|
|
|
55
|
|
13
|
55
|
use needle; |
|
55
|
|
|
55
|
|
14
|
55
|
|
|
55
|
|
|
55
|
|
15
|
|
has files_to_download => sub { [] }; |
16
|
|
has openqa_url => sub { |
17
|
|
# deduce the default openQA URL from OPENQA_URL/OPENQA_HOSTNAME |
18
|
|
# note: OPENQA_URL is sometimes just the hostname (eg. e212.suse.de) but might be a proper URL |
19
|
|
# as well (eg. http://openqa1-opensuse). |
20
|
|
my $url = Mojo::URL->new($bmwqemu::vars{OPENQA_URL}); |
21
|
|
my $host = $bmwqemu::vars{OPENQA_HOSTNAME}; |
22
|
|
|
23
|
|
# determine host if not present in OPENQA_URL |
24
|
|
if (!$url->host) { |
25
|
|
my $path_parts = $url->path->parts; |
26
|
|
if (scalar @$path_parts == 1) { |
27
|
|
if ($url->scheme) { |
28
|
|
# treat URLs like 'localhost:9526' in the way that 'localhost' is the hostname (and not the protocol) |
29
|
|
$url->host($url->scheme); |
30
|
|
$url->port($path_parts->[0]); |
31
|
|
$url->scheme(''); |
32
|
|
} |
33
|
|
else { |
34
|
|
# treat URLs like just 'e212.suse.de' in the way that 'e212.suse.de' is the hostname (and not a relative path) |
35
|
|
$url->host($path_parts->[0]); |
36
|
|
} |
37
|
|
$url->path(Mojo::Path->new); |
38
|
|
} |
39
|
|
elsif ($host) { |
40
|
|
# build a default URL from OPENQA_HOSTNAME if no host in OPENQA_URL is missing |
41
|
|
$url = Mojo::URL->new(); |
42
|
|
$url->scheme(''); |
43
|
|
$url->host($host); |
44
|
|
} |
45
|
|
} |
46
|
|
|
47
|
|
# assume 'http' if scheme is missing |
48
|
|
if (!$url->scheme) { |
49
|
|
$url->scheme('http'); |
50
|
|
} |
51
|
|
|
52
|
|
return $url; |
53
|
|
}; |
54
|
|
has ua => sub { Mojo::UserAgent->new }; |
55
|
|
has download_limit => 150; |
56
|
|
|
57
|
|
my $needle_name = $needle->{name}; |
58
|
4
|
my $latest_update = $needle->{t_updated}; |
|
4
|
|
|
4
|
|
|
4
|
|
|
4
|
|
|
4
|
|
59
|
4
|
my $needles_dir = needle::needles_dir(); |
60
|
4
|
my $download_target = "$needles_dir/$needle_name.$extension"; |
61
|
4
|
|
62
|
4
|
if (my $target_stat = stat($download_target)) { |
63
|
|
if (my $target_last_modified = $target_stat->[9] // $target_stat->[8]) { |
64
|
4
|
$target_last_modified = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($target_last_modified)); |
65
|
2
|
if ($target_last_modified ge $latest_update) { |
66
|
2
|
bmwqemu::diag("skipping downloading new needle: $download_target seems already up-to-date (last update: $target_last_modified > $latest_update)"); |
67
|
2
|
return; |
68
|
1
|
} |
69
|
1
|
} |
70
|
|
} |
71
|
|
|
72
|
|
push(@{$self->files_to_download}, { |
73
|
|
target => $download_target, |
74
|
3
|
url => Mojo::URL->new($self->openqa_url . $needle->{$path_param}), |
75
|
|
}); |
76
|
3
|
} |
77
|
|
|
78
|
|
my $download_url = $download->{url}; |
79
|
|
my $download_target = $download->{target}; |
80
|
3
|
bmwqemu::diag("download new needle: $download_url => $download_target"); |
|
3
|
|
|
3
|
|
|
3
|
|
81
|
3
|
|
82
|
3
|
# download the file |
83
|
3
|
my $download_res; |
84
|
|
try { |
85
|
|
$download_res = $self->ua->get($download_url)->result; |
86
|
3
|
if (!$download_res->is_success) { |
87
|
|
my $return_code = $download_res->code; |
88
|
3
|
bmwqemu::diag("failed to download $download_url, server returned $return_code"); |
89
|
0
|
$download_res = undef; |
90
|
0
|
} |
91
|
0
|
} |
92
|
0
|
catch { |
93
|
|
bmwqemu::fctinfo("internal error occurred when downloading $download_url: $_"); |
94
|
|
}; |
95
|
|
|
96
|
3
|
# store the file on disk |
97
|
3
|
return unless ($download_res); |
98
|
|
try { |
99
|
|
unlink($download_target); |
100
|
3
|
path($download_target)->spurt($download_res->body); |
101
|
|
} |
102
|
0
|
catch { |
103
|
0
|
bmwqemu::diag("unable to store download under $download_target"); |
104
|
|
}; |
105
|
|
} |
106
|
0
|
|
107
|
0
|
# adds downloads for the specified $new_needles if those are missing/outdated locally |
108
|
|
my $download_limit = $self->download_limit; |
109
|
|
my $added_downloads = $self->files_to_download; |
110
|
|
for my $needle (@$new_needles) { |
111
|
6
|
last if (scalar @$added_downloads >= $download_limit); |
|
6
|
|
|
6
|
|
|
6
|
|
112
|
6
|
$self->_add_download($needle, 'json', 'json_path'); |
113
|
6
|
$self->_add_download($needle, 'png', 'image_path'); |
114
|
6
|
} |
115
|
3
|
} |
116
|
2
|
|
117
|
2
|
# downloads previously added downloads |
118
|
|
|
119
|
|
# downloads missing needles considering $new_needles |
120
|
|
# (see t/21-needle-downloader.t for an example of $new_needles) |
121
|
|
$self->add_relevant_downloads($new_needles); |
122
|
5
|
$self->download(); |
|
5
|
|
|
5
|
|
|
5
|
|
|
5
|
|
123
|
|
} |
124
|
|
|
125
|
|
1; |