QA » openQA Project

Example how it looks like when in a successful run (despite the connection being interrupted):

# Subtest: resume test execution and 2nd tab
    ok 1 - 2 browser tabs open (live view and tab from previous subtest)
    ok 2 - found connection opened at 135
    # waiting for resume, developer console contains:
    # status: Connecting to ws:/localhost:49280/liveviewhandler/tests/1/developer/ws-proxy
    # status: Using proxy: yes
    # status: Connection opened
    # <== {"data":null,"type":"info","what":"connecting to os-autoinst command server at ws:\/\/127.0.0.1:20013\/1wTUiBKtudqe3ExH\/ws"}
    # <== {"data":null,"type":"info","what":"reusing previous connection to os-autoinst command server at ws:\/\/127.0.0.1:20013\/1wTUiBKtudqe3ExH\/ws"}
    # <== {"data":{"outstanding_files":0,"outstanding_images":0,"upload_up_to":null,"upload_up_to_current_module":null},"type":"info","what":"upload progress"}
    # <== {"data":{"current_api_function":"last_screenshot_data","current_test_full_name":"tinycore\/tests-shutdown","devel_mode_major_version":1,"devel_mode_minor_version":1,"developer_id":2,"developer_name":"Demo","developer_session_started_at":"2019-07-25T14:13:43","developer_session_tab_count":2,"outstanding_files":0,"outstanding_images":0,"pause_on_next_command":0,"pause_on_screen_mismatch":"assert_screen","pause_test_name":null,"running":"shutdown","tags":null,"test_execution_paused":"reached module shutdown","upload_up_to_current_module":null},"type":"info","what":"cmdsrvmsg"}
    # <== {"data":{"current_api_function":"last_screenshot_data","current_test_full_name":"tinycore\/tests-shutdown","devel_mode_major_version":1,"devel_mode_minor_version":1,"developer_id":2,"developer_name":"Demo","developer_session_started_at":"2019-07-25T14:13:43","developer_session_tab_count":2,"outstanding_files":0,"outstanding_images":0,"pause_on_next_command":0,"pause_on_screen_mismatch":"assert_screen","pause_test_name":null,"running":"shutdown","tags":null,"test_execution_paused":"reached module shutdown","upload_up_to_current_module":null},"type":"info","what":"cmdsrvmsg"}
    # status: Can't send command, no ws connection opened! Will try to send when connection has been restored.
    # waiting for resume, developer console contains:
    # status: Connecting to ws:/localhost:49280/liveviewhandler/tests/1/developer/ws-proxy
    # status: Using proxy: yes
    # status: Connection opened
    # <== {"data":null,"type":"info","what":"connecting to os-autoinst command server at ws:\/\/127.0.0.1:20013\/1wTUiBKtudqe3ExH\/ws"}
    # <== {"data":null,"type":"info","what":"reusing previous connection to os-autoinst command server at ws:\/\/127.0.0.1:20013\/1wTUiBKtudqe3ExH\/ws"}
    # <== {"data":{"outstanding_files":0,"outstanding_images":0,"upload_up_to":null,"upload_up_to_current_module":null},"type":"info","what":"upload progress"}
    # <== {"data":{"current_api_function":"last_screenshot_data","current_test_full_name":"tinycore\/tests-shutdown","devel_mode_major_version":1,"devel_mode_minor_version":1,"developer_id":2,"developer_name":"Demo","developer_session_started_at":"2019-07-25T14:13:43","developer_session_tab_count":2,"outstanding_files":0,"outstanding_images":0,"pause_on_next_command":0,"pause_on_screen_mismatch":"assert_screen","pause_test_name":null,"running":"shutdown","tags":null,"test_execution_paused":"reached module shutdown","upload_up_to_current_module":null},"type":"info","what":"cmdsrvmsg"}
    # <== {"data":{"current_api_function":"last_screenshot_data","current_test_full_name":"tinycore\/tests-shutdown","devel_mode_major_version":1,"devel_mode_minor_version":1,"developer_id":2,"developer_name":"Demo","developer_session_started_at":"2019-07-25T14:13:43","developer_session_tab_count":2,"outstanding_files":0,"outstanding_images":0,"pause_on_next_command":0,"pause_on_screen_mismatch":"assert_screen","pause_test_name":null,"running":"shutdown","tags":null,"test_execution_paused":"reached module shutdown","upload_up_to_current_module":null},"type":"info","what":"cmdsrvmsg"}
    # status: Can't send command, no ws connection opened! Will try to send when connection has been restored.
    # <== {"data":{"outstanding_files":0,"outstanding_images":0,"upload_up_to":"shutdown","upload_up_to_current_module":1},"type":"info","what":"upload progress"}
    # waiting for resume, developer console contains:
    # status: Connecting to ws:/localhost:49280/liveviewhandler/tests/1/developer/ws-proxy
    # status: Using proxy: yes
    # status: Connection opened
    # <== {"data":null,"type":"info","what":"connecting to os-autoinst command server at ws:\/\/127.0.0.1:20013\/1wTUiBKtudqe3ExH\/ws"}
    # <== {"data":null,"type":"info","what":"reusing previous connection to os-autoinst command server at ws:\/\/127.0.0.1:20013\/1wTUiBKtudqe3ExH\/ws"}
    # <== {"data":{"outstanding_files":0,"outstanding_images":0,"upload_up_to":null,"upload_up_to_current_module":null},"type":"info","what":"upload progress"}
    # <== {"data":{"current_api_function":"last_screenshot_data","current_test_full_name":"tinycore\/tests-shutdown","devel_mode_major_version":1,"devel_mode_minor_version":1,"developer_id":2,"developer_name":"Demo","developer_session_started_at":"2019-07-25T14:13:43","developer_session_tab_count":2,"outstanding_files":0,"outstanding_images":0,"pause_on_next_command":0,"pause_on_screen_mismatch":"assert_screen","pause_test_name":null,"running":"shutdown","tags":null,"test_execution_paused":"reached module shutdown","upload_up_to_current_module":null},"type":"info","what":"cmdsrvmsg"}
    # <== {"data":{"current_api_function":"last_screenshot_data","current_test_full_name":"tinycore\/tests-shutdown","devel_mode_major_version":1,"devel_mode_minor_version":1,"developer_id":2,"developer_name":"Demo","developer_session_started_at":"2019-07-25T14:13:43","developer_session_tab_count":2,"outstanding_files":0,"outstanding_images":0,"pause_on_next_command":0,"pause_on_screen_mismatch":"assert_screen","pause_test_name":null,"running":"shutdown","tags":null,"test_execution_paused":"reached module shutdown","upload_up_to_current_module":null},"type":"info","what":"cmdsrvmsg"}
    # status: Can't send command, no ws connection opened! Will try to send when connection has been restored.
    # <== {"data":{"outstanding_files":0,"outstanding_images":0,"upload_up_to":"shutdown","upload_up_to_current_module":1},"type":"info","what":"upload progress"}
    # <== {"data":{"category":"cmdsrv-connection","code":1006,"reason":null},"type":"error","what":"connection to os-autoinst command server lost"}
    # <== {"data":{"category":"cmdsrv-connection","code":1006,"reason":null},"type":"error","what":"connection to os-autoinst command server lost"}
    # status: Connection error: error (check JavaScript console for details)
    # status: Connection closed, trying to reconnect in 2 seconds
    ok 3 - found resume at 4399
    ok 4 - found resume (2nd tab) at 9198
    1..4
ok 25 - resume test execution and 2nd tab

The situation in the initial ticket description could be easily avoided by passing the current job ID to the web UI on worker registration if it is re-registering while a job is running.

Than this code which is executed on registration would be only conditionally executed:

    # in case the worker died ...
    # ... restart job assigned to this worker
    if (my $job = $worker->job) {
        $job->set_property('JOBTOKEN');
        $job->auto_duplicate;

        # .. set it incomplete
        $job->done(result => OpenQA::Jobs::Constants::INCOMPLETE);
        $worker->update({job_id => undef});
    }

So in case the worker still works on that job we wouldn't execute that if-block anymore. I guess that solves the problem without introducing any new ones like abandoned jobs stuck in running state forever (because if the worker really dies it would have lost the current job ID and we would still execute that if-block).